searcharray 0.0.25__tar.gz → 0.0.27__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {searcharray-0.0.25 → searcharray-0.0.27}/PKG-INFO +5 -5
- {searcharray-0.0.25 → searcharray-0.0.27}/README.md +4 -4
- {searcharray-0.0.25 → searcharray-0.0.27}/searcharray/phrase/middle_out.py +1 -1
- {searcharray-0.0.25 → searcharray-0.0.27}/searcharray/postings.py +14 -36
- searcharray-0.0.27/searcharray/similarity.py +30 -0
- {searcharray-0.0.25 → searcharray-0.0.27}/searcharray/solr.py +17 -8
- {searcharray-0.0.25 → searcharray-0.0.27}/searcharray.egg-info/PKG-INFO +5 -5
- {searcharray-0.0.25 → searcharray-0.0.27}/searcharray.egg-info/SOURCES.txt +1 -0
- {searcharray-0.0.25 → searcharray-0.0.27}/setup.py +1 -1
- {searcharray-0.0.25 → searcharray-0.0.27}/test/test_search.py +2 -4
- {searcharray-0.0.25 → searcharray-0.0.27}/test/test_solr.py +43 -29
- {searcharray-0.0.25 → searcharray-0.0.27}/test/test_tmdb.py +16 -3
- {searcharray-0.0.25 → searcharray-0.0.27}/pyproject.toml +0 -0
- {searcharray-0.0.25 → searcharray-0.0.27}/searcharray/__init__.py +0 -0
- {searcharray-0.0.25 → searcharray-0.0.27}/searcharray/phrase/__init__.py +0 -0
- {searcharray-0.0.25 → searcharray-0.0.27}/searcharray/phrase/posn_diffs.py +0 -0
- {searcharray-0.0.25 → searcharray-0.0.27}/searcharray/phrase/scan_merge.py +0 -0
- {searcharray-0.0.25 → searcharray-0.0.27}/searcharray/term_dict.py +0 -0
- {searcharray-0.0.25 → searcharray-0.0.27}/searcharray/utils/__init__.py +0 -0
- {searcharray-0.0.25 → searcharray-0.0.27}/searcharray/utils/bitcount.py +0 -0
- {searcharray-0.0.25 → searcharray-0.0.27}/searcharray/utils/mat_set.py +0 -0
- {searcharray-0.0.25 → searcharray-0.0.27}/searcharray/utils/roaringish.py +0 -0
- {searcharray-0.0.25 → searcharray-0.0.27}/searcharray/utils/row_viewable_matrix.py +0 -0
- {searcharray-0.0.25 → searcharray-0.0.27}/searcharray.egg-info/dependency_links.txt +0 -0
- {searcharray-0.0.25 → searcharray-0.0.27}/searcharray.egg-info/entry_points.txt +0 -0
- {searcharray-0.0.25 → searcharray-0.0.27}/searcharray.egg-info/requires.txt +0 -0
- {searcharray-0.0.25 → searcharray-0.0.27}/searcharray.egg-info/top_level.txt +0 -0
- {searcharray-0.0.25 → searcharray-0.0.27}/setup.cfg +0 -0
- {searcharray-0.0.25 → searcharray-0.0.27}/test/test_extension_array.py +0 -0
- {searcharray-0.0.25 → searcharray-0.0.27}/test/test_msmarco.py +0 -0
- {searcharray-0.0.25 → searcharray-0.0.27}/test/test_phrase_matches.py +0 -0
- {searcharray-0.0.25 → searcharray-0.0.27}/test/test_utils.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: searcharray
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.27
|
|
4
4
|
Summary: Searchable pandas text extension arrays for prototyping search
|
|
5
5
|
Home-page: https://github.com/softwaredoug/searcharray
|
|
6
6
|
Author: Doug Turnbull
|
|
@@ -37,7 +37,7 @@ Think Lucene, but as a Pandas column.
|
|
|
37
37
|
|
|
38
38
|
```python
|
|
39
39
|
In[3]: df['title_indexed'] = PostingsArray.index(df['title'])
|
|
40
|
-
np.sort(df['title_indexed'].array.
|
|
40
|
+
np.sort(df['title_indexed'].array.score('Cat'))
|
|
41
41
|
Out[3]: array([ 0. , 0. , 0. , ..., 15.84568033,
|
|
42
42
|
15.84568033, 15.84568033])
|
|
43
43
|
```
|
|
@@ -100,11 +100,11 @@ Out[2]:
|
|
|
100
100
|
Then search, getting top N with `Cat`
|
|
101
101
|
|
|
102
102
|
```
|
|
103
|
-
In[3]: np.sort(df['title_indexed'].array.
|
|
103
|
+
In[3]: np.sort(df['title_indexed'].array.score('Cat'))
|
|
104
104
|
Out[3]: array([ 0. , 0. , 0. , ..., 15.84568033,
|
|
105
105
|
15.84568033, 15.84568033])
|
|
106
106
|
|
|
107
|
-
In[4]: df['title_indexed'].
|
|
107
|
+
In[4]: df['title_indexed'].score('Cat').argsort()
|
|
108
108
|
Out[4]:
|
|
109
109
|
|
|
110
110
|
array([0, 18561, 18560, ..., 15038, 19012, 4392])
|
|
@@ -189,7 +189,7 @@ df['hrs_into_past'] = (now - df['timestamp']).dt.total_seconds() / 3600
|
|
|
189
189
|
Then multiply by BM25 if you want:
|
|
190
190
|
|
|
191
191
|
```
|
|
192
|
-
df['score'] = df['title_indexed'].
|
|
192
|
+
df['score'] = df['title_indexed'].score('Cat') * df['hrs_into_past']
|
|
193
193
|
```
|
|
194
194
|
|
|
195
195
|
## TODOs / Future Work / Known issues
|
|
@@ -8,7 +8,7 @@ Think Lucene, but as a Pandas column.
|
|
|
8
8
|
|
|
9
9
|
```python
|
|
10
10
|
In[3]: df['title_indexed'] = PostingsArray.index(df['title'])
|
|
11
|
-
np.sort(df['title_indexed'].array.
|
|
11
|
+
np.sort(df['title_indexed'].array.score('Cat'))
|
|
12
12
|
Out[3]: array([ 0. , 0. , 0. , ..., 15.84568033,
|
|
13
13
|
15.84568033, 15.84568033])
|
|
14
14
|
```
|
|
@@ -71,11 +71,11 @@ Out[2]:
|
|
|
71
71
|
Then search, getting top N with `Cat`
|
|
72
72
|
|
|
73
73
|
```
|
|
74
|
-
In[3]: np.sort(df['title_indexed'].array.
|
|
74
|
+
In[3]: np.sort(df['title_indexed'].array.score('Cat'))
|
|
75
75
|
Out[3]: array([ 0. , 0. , 0. , ..., 15.84568033,
|
|
76
76
|
15.84568033, 15.84568033])
|
|
77
77
|
|
|
78
|
-
In[4]: df['title_indexed'].
|
|
78
|
+
In[4]: df['title_indexed'].score('Cat').argsort()
|
|
79
79
|
Out[4]:
|
|
80
80
|
|
|
81
81
|
array([0, 18561, 18560, ..., 15038, 19012, 4392])
|
|
@@ -160,7 +160,7 @@ df['hrs_into_past'] = (now - df['timestamp']).dt.total_seconds() / 3600
|
|
|
160
160
|
Then multiply by BM25 if you want:
|
|
161
161
|
|
|
162
162
|
```
|
|
163
|
-
df['score'] = df['title_indexed'].
|
|
163
|
+
df['score'] = df['title_indexed'].score('Cat') * df['hrs_into_past']
|
|
164
164
|
```
|
|
165
165
|
|
|
166
166
|
## TODOs / Future Work / Known issues
|
|
@@ -316,7 +316,7 @@ class PosnBitArray:
|
|
|
316
316
|
doc_ids = encoder.keys(term_posns)
|
|
317
317
|
change_indices = np.nonzero(np.diff(doc_ids))[0]
|
|
318
318
|
change_indices = np.concatenate((np.asarray([0]), change_indices + 1))
|
|
319
|
-
posns =
|
|
319
|
+
posns = term_posns & encoder.payload_lsb_mask
|
|
320
320
|
bit_counts = bit_count64(posns)
|
|
321
321
|
|
|
322
322
|
term_freqs = np.add.reduceat(bit_counts, change_indices)
|
|
@@ -19,6 +19,7 @@ from searcharray.phrase.scan_merge import scan_merge_ins
|
|
|
19
19
|
from searcharray.phrase.posn_diffs import compute_phrase_freqs
|
|
20
20
|
from searcharray.phrase.middle_out import PosnBitArrayBuilder, PosnBitArrayAlreadyEncBuilder, PosnBitArray
|
|
21
21
|
from searcharray.utils.mat_set import SparseMatSetBuilder
|
|
22
|
+
from searcharray.similarity import Similarity, default_bm25
|
|
22
23
|
|
|
23
24
|
logger = logging.getLogger(__name__)
|
|
24
25
|
|
|
@@ -618,49 +619,26 @@ class PostingsArray(ExtensionArray):
|
|
|
618
619
|
term_freq = self.term_freq(token)
|
|
619
620
|
return term_freq > 0
|
|
620
621
|
|
|
621
|
-
def
|
|
622
|
-
"""
|
|
623
|
-
|
|
624
|
-
idf, computed as log(1 + (N - n + 0.5) / (n + 0.5))
|
|
625
|
-
"""
|
|
626
|
-
token = self._check_token_arg(token)
|
|
627
|
-
if isinstance(token, list):
|
|
628
|
-
return self.bm25_phrase_idf(token)
|
|
629
|
-
|
|
630
|
-
df = self.doc_freq(token)
|
|
631
|
-
num_docs = len(self)
|
|
632
|
-
return np.log(1 + (num_docs - df + 0.5) / (df + 0.5))
|
|
633
|
-
|
|
634
|
-
def bm25_phrase_idf(self, tokens: List[str]) -> float:
|
|
635
|
-
"""Calculate the idf for a phrase.
|
|
636
|
-
|
|
637
|
-
This is the sum of the idfs of the individual terms.
|
|
638
|
-
"""
|
|
639
|
-
idfs = [self.bm25_idf(term) for term in tokens]
|
|
640
|
-
return np.sum(idfs)
|
|
641
|
-
|
|
642
|
-
def bm25_tf(self, token: Union[List[str], str], k1=1.2, b=0.75, slop=1) -> float:
|
|
643
|
-
"""Calculate the (Lucene) BM25 tf for a term or phrase.
|
|
644
|
-
|
|
645
|
-
tf, computed as freq / (freq + k1 * (1 - b + b * dl / avgdl))
|
|
646
|
-
"""
|
|
647
|
-
tf = self.term_freq(token)
|
|
648
|
-
score = tf / (tf + k1 * (1 - b + b * self.doc_lengths() / self.avg_doc_length))
|
|
649
|
-
return score
|
|
650
|
-
|
|
651
|
-
def bm25(self, token: Union[str, List[str]], doc_stats=None, k1=1.2, b=0.75):
|
|
652
|
-
"""Score each doc using BM25.
|
|
622
|
+
def score(self, token: Union[str, List[str]], similarity: Similarity = default_bm25) -> np.ndarray:
|
|
623
|
+
"""Score each doc using a similarity function.
|
|
653
624
|
|
|
654
625
|
Parameters
|
|
655
626
|
----------
|
|
656
627
|
token : str or list of str of what to search (already tokenized)
|
|
657
|
-
|
|
658
|
-
k1 : float, optional BM25 param. Defaults to 1.2.
|
|
659
|
-
b : float, optional BM25 param. Defaults to 0.75.
|
|
628
|
+
similarity : How to score the documents. Default is BM25.
|
|
660
629
|
"""
|
|
661
630
|
# Get term freqs per token
|
|
662
631
|
token = self._check_token_arg(token)
|
|
663
|
-
|
|
632
|
+
|
|
633
|
+
tfs = self.term_freq(token)
|
|
634
|
+
token = self._check_token_arg(token)
|
|
635
|
+
tokens_l = [token] if isinstance(token, str) else token
|
|
636
|
+
all_dfs = np.asarray([self.doc_freq(token) for token in tokens_l])
|
|
637
|
+
doc_lens = self.doc_lengths()
|
|
638
|
+
scores = similarity(term_freqs=tfs, doc_freqs=all_dfs,
|
|
639
|
+
doc_lens=doc_lens, avg_doc_lens=self.avg_doc_length,
|
|
640
|
+
num_docs=len(self))
|
|
641
|
+
return scores
|
|
664
642
|
|
|
665
643
|
def positions(self, token: str, key=None) -> List[np.ndarray]:
|
|
666
644
|
"""Return a list of lists of positions of the given term."""
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
"""Similarity functions given term stats."""
|
|
2
|
+
from typing import Protocol
|
|
3
|
+
import numpy as np
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class Similarity(Protocol):
|
|
7
|
+
"""Similarity function protocol."""
|
|
8
|
+
|
|
9
|
+
def __call__(self, term_freqs: np.ndarray, doc_freqs: np.ndarray, doc_lens: np.ndarray,
|
|
10
|
+
avg_doc_lens: int, num_docs: int) -> np.ndarray:
|
|
11
|
+
"""Calculate similarity scores."""
|
|
12
|
+
...
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def bm25_similarity(k1: float = 1.2, b: float = 0.75) -> Similarity:
|
|
16
|
+
def bm25(term_freqs: np.ndarray, doc_freqs: np.ndarray,
|
|
17
|
+
doc_lens: np.ndarray,
|
|
18
|
+
avg_doc_lens: int, num_docs: int) -> np.ndarray:
|
|
19
|
+
"""Calculate BM25 scores."""
|
|
20
|
+
# Sum doc freqs
|
|
21
|
+
sum_dfs = np.sum(doc_freqs, axis=0)
|
|
22
|
+
# Calculate idf
|
|
23
|
+
idf = np.log(1 + (num_docs - sum_dfs + 0.5) / (sum_dfs + 0.5))
|
|
24
|
+
# Calculate tf
|
|
25
|
+
tf = term_freqs / (term_freqs + k1 * (1 - b + b * doc_lens / avg_doc_lens))
|
|
26
|
+
return idf * tf
|
|
27
|
+
return bm25
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
default_bm25 = bm25_similarity()
|
|
@@ -4,6 +4,7 @@ import pandas as pd
|
|
|
4
4
|
import numpy as np
|
|
5
5
|
from typing import List, Optional, Dict, Tuple
|
|
6
6
|
from searcharray.postings import PostingsArray
|
|
7
|
+
from searcharray.similarity import Similarity, default_bm25
|
|
7
8
|
|
|
8
9
|
|
|
9
10
|
def parse_min_should_match(num_clauses: int, spec: str) -> int:
|
|
@@ -111,7 +112,8 @@ def _edismax_term_centric(frame: pd.DataFrame,
|
|
|
111
112
|
query_fields: Dict[str, float],
|
|
112
113
|
num_search_terms: int,
|
|
113
114
|
search_terms: Dict[str, List[str]],
|
|
114
|
-
mm: str
|
|
115
|
+
mm: str,
|
|
116
|
+
similarity: Similarity) -> Tuple[np.ndarray, str]:
|
|
115
117
|
explain = []
|
|
116
118
|
term_scores = []
|
|
117
119
|
for term_posn in range(num_search_terms):
|
|
@@ -120,7 +122,7 @@ def _edismax_term_centric(frame: pd.DataFrame,
|
|
|
120
122
|
for field, boost in query_fields.items():
|
|
121
123
|
term = search_terms[field][term_posn]
|
|
122
124
|
post_arr = get_field(frame, field)
|
|
123
|
-
field_term_score = post_arr.
|
|
125
|
+
field_term_score = post_arr.score(term, similarity=similarity) * (1 if boost is None else boost)
|
|
124
126
|
boost_exp = f"{boost}" if boost is not None else "1"
|
|
125
127
|
term_explain.append(f"{field}:{term}^{boost_exp}")
|
|
126
128
|
max_scores = np.maximum(max_scores, field_term_score)
|
|
@@ -139,12 +141,14 @@ def _edismax_field_centric(frame: pd.DataFrame,
|
|
|
139
141
|
query_fields: Dict[str, float],
|
|
140
142
|
num_search_terms: int,
|
|
141
143
|
search_terms: Dict[str, List[str]],
|
|
142
|
-
mm: str
|
|
144
|
+
mm: str,
|
|
145
|
+
similarity: Similarity = default_bm25) -> Tuple[np.ndarray, str]:
|
|
143
146
|
field_scores = []
|
|
144
147
|
explain = []
|
|
145
148
|
for field, boost in query_fields.items():
|
|
146
149
|
post_arr = get_field(frame, field)
|
|
147
|
-
term_scores = np.array([post_arr.
|
|
150
|
+
term_scores = np.array([post_arr.score(term, similarity=similarity)
|
|
151
|
+
for term in search_terms[field]])
|
|
148
152
|
min_should_match = parse_min_should_match(len(search_terms[field]), spec=mm)
|
|
149
153
|
exp = " ".join([f"{field}:{term}" for term in search_terms[field]])
|
|
150
154
|
boost_exp = f"{boost}" if boost is not None else "1"
|
|
@@ -169,7 +173,8 @@ def edismax(frame: pd.DataFrame,
|
|
|
169
173
|
pf: Optional[List[str]] = None,
|
|
170
174
|
pf2: Optional[List[str]] = None,
|
|
171
175
|
pf3: Optional[List[str]] = None,
|
|
172
|
-
q_op: str = "OR"
|
|
176
|
+
q_op: str = "OR",
|
|
177
|
+
similarity: Similarity = default_bm25) -> Tuple[np.ndarray, str]:
|
|
173
178
|
"""Run edismax search over dataframe with searcharray fields.
|
|
174
179
|
|
|
175
180
|
Parameters
|
|
@@ -209,15 +214,19 @@ def edismax(frame: pd.DataFrame,
|
|
|
209
214
|
|
|
210
215
|
num_search_terms, search_terms, term_centric = parse_query_terms(frame, q, list(query_fields.keys()))
|
|
211
216
|
if term_centric:
|
|
212
|
-
qf_scores, explain = _edismax_term_centric(frame, query_fields,
|
|
217
|
+
qf_scores, explain = _edismax_term_centric(frame, query_fields,
|
|
218
|
+
num_search_terms, search_terms, mm,
|
|
219
|
+
similarity=similarity)
|
|
213
220
|
else:
|
|
214
|
-
qf_scores, explain = _edismax_field_centric(frame, query_fields,
|
|
221
|
+
qf_scores, explain = _edismax_field_centric(frame, query_fields,
|
|
222
|
+
num_search_terms, search_terms, mm,
|
|
223
|
+
similarity=similarity)
|
|
215
224
|
|
|
216
225
|
phrase_scores = []
|
|
217
226
|
for field, boost in phrase_fields.items():
|
|
218
227
|
arr = get_field(frame, field)
|
|
219
228
|
terms = search_terms[field]
|
|
220
|
-
field_phrase_score = arr.
|
|
229
|
+
field_phrase_score = arr.score(terms, similarity=similarity) * (1 if boost is None else boost)
|
|
221
230
|
boost_exp = f"{boost}" if boost is not None else "1"
|
|
222
231
|
explain += f" ({field}:\"{' '.join(terms)}\")^{boost_exp}"
|
|
223
232
|
phrase_scores.append(field_phrase_score)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: searcharray
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.27
|
|
4
4
|
Summary: Searchable pandas text extension arrays for prototyping search
|
|
5
5
|
Home-page: https://github.com/softwaredoug/searcharray
|
|
6
6
|
Author: Doug Turnbull
|
|
@@ -37,7 +37,7 @@ Think Lucene, but as a Pandas column.
|
|
|
37
37
|
|
|
38
38
|
```python
|
|
39
39
|
In[3]: df['title_indexed'] = PostingsArray.index(df['title'])
|
|
40
|
-
np.sort(df['title_indexed'].array.
|
|
40
|
+
np.sort(df['title_indexed'].array.score('Cat'))
|
|
41
41
|
Out[3]: array([ 0. , 0. , 0. , ..., 15.84568033,
|
|
42
42
|
15.84568033, 15.84568033])
|
|
43
43
|
```
|
|
@@ -100,11 +100,11 @@ Out[2]:
|
|
|
100
100
|
Then search, getting top N with `Cat`
|
|
101
101
|
|
|
102
102
|
```
|
|
103
|
-
In[3]: np.sort(df['title_indexed'].array.
|
|
103
|
+
In[3]: np.sort(df['title_indexed'].array.score('Cat'))
|
|
104
104
|
Out[3]: array([ 0. , 0. , 0. , ..., 15.84568033,
|
|
105
105
|
15.84568033, 15.84568033])
|
|
106
106
|
|
|
107
|
-
In[4]: df['title_indexed'].
|
|
107
|
+
In[4]: df['title_indexed'].score('Cat').argsort()
|
|
108
108
|
Out[4]:
|
|
109
109
|
|
|
110
110
|
array([0, 18561, 18560, ..., 15038, 19012, 4392])
|
|
@@ -189,7 +189,7 @@ df['hrs_into_past'] = (now - df['timestamp']).dt.total_seconds() / 3600
|
|
|
189
189
|
Then multiply by BM25 if you want:
|
|
190
190
|
|
|
191
191
|
```
|
|
192
|
-
df['score'] = df['title_indexed'].
|
|
192
|
+
df['score'] = df['title_indexed'].score('Cat') * df['hrs_into_past']
|
|
193
193
|
```
|
|
194
194
|
|
|
195
195
|
## TODOs / Future Work / Known issues
|
|
@@ -36,7 +36,7 @@ setup(
|
|
|
36
36
|
# For a discussion on single-sourcing the version across setup.py and the
|
|
37
37
|
# project code, see
|
|
38
38
|
# https://packaging.python.org/guides/single-sourcing-package-version/
|
|
39
|
-
version="0.0.
|
|
39
|
+
version="0.0.27", # Required
|
|
40
40
|
# This is a one-line description or tagline of what your project does. This
|
|
41
41
|
# corresponds to the "Summary" metadata field:
|
|
42
42
|
# https://packaging.python.org/specifications/core-metadata/#summary
|
|
@@ -40,10 +40,8 @@ def test_doc_lengths(data):
|
|
|
40
40
|
assert data.avg_doc_length == 2.5
|
|
41
41
|
|
|
42
42
|
|
|
43
|
-
def
|
|
44
|
-
|
|
45
|
-
assert bm25_idf > 0.0
|
|
46
|
-
bm25 = data.bm25("bar")
|
|
43
|
+
def test_default_score_matches_lucene(data):
|
|
44
|
+
bm25 = data.score("bar")
|
|
47
45
|
assert bm25.shape == (100,)
|
|
48
46
|
assert np.isclose(bm25, [0.37066694, 0., 0.34314217, 0.] * 25).all()
|
|
49
47
|
|
|
@@ -85,11 +85,11 @@ edismax_scenarios = {
|
|
|
85
85
|
'title': lambda: PostingsArray.index(["foo bar bar baz", "data2", "data3 bar", "bunny funny wunny"]),
|
|
86
86
|
'body': lambda: PostingsArray.index(["buzz", "data2", "data3 bar", "bunny funny wunny"])
|
|
87
87
|
},
|
|
88
|
-
"expected": [lambda frame: sum([frame['title'].array.
|
|
89
|
-
frame['title'].array.
|
|
88
|
+
"expected": [lambda frame: sum([frame['title'].array.score("foo")[0],
|
|
89
|
+
frame['title'].array.score("bar")[0]]),
|
|
90
90
|
0,
|
|
91
|
-
lambda frame: max(frame['title'].array.
|
|
92
|
-
frame['body'].array.
|
|
91
|
+
lambda frame: max(frame['title'].array.score("bar")[2],
|
|
92
|
+
frame['body'].array.score("bar")[2]),
|
|
93
93
|
0],
|
|
94
94
|
"params": {'q': "foo bar", 'qf': ["title", "body"]},
|
|
95
95
|
},
|
|
@@ -99,11 +99,11 @@ edismax_scenarios = {
|
|
|
99
99
|
'body': lambda: PostingsArray.index(["foo bar", "data2", "data3 bar", "bunny funny wunny"],
|
|
100
100
|
tokenizer=just_lowercasing_tokenizer)
|
|
101
101
|
},
|
|
102
|
-
"expected": [lambda frame: max(sum([frame['title'].array.
|
|
103
|
-
frame['title'].array.
|
|
104
|
-
frame['body'].array.
|
|
102
|
+
"expected": [lambda frame: max(sum([frame['title'].array.score("foo")[0],
|
|
103
|
+
frame['title'].array.score("bar")[0]]),
|
|
104
|
+
frame['body'].array.score("foo bar")[0]),
|
|
105
105
|
0,
|
|
106
|
-
lambda frame: frame['title'].array.
|
|
106
|
+
lambda frame: frame['title'].array.score("bar")[2],
|
|
107
107
|
0],
|
|
108
108
|
"params": {'q': "foo bar", 'qf': ["title", "body"]},
|
|
109
109
|
},
|
|
@@ -113,9 +113,9 @@ edismax_scenarios = {
|
|
|
113
113
|
'body': lambda: PostingsArray.index(["foo bar", "data2", "data3 bar", "bunny funny wunny"],
|
|
114
114
|
tokenizer=just_lowercasing_tokenizer)
|
|
115
115
|
},
|
|
116
|
-
"expected": [lambda frame: max(sum([frame['title'].array.
|
|
117
|
-
frame['title'].array.
|
|
118
|
-
frame['body'].array.
|
|
116
|
+
"expected": [lambda frame: max(sum([frame['title'].array.score("foo")[0],
|
|
117
|
+
frame['title'].array.score("bar")[0]]),
|
|
118
|
+
frame['body'].array.score("foo bar")[0]),
|
|
119
119
|
0,
|
|
120
120
|
0,
|
|
121
121
|
0],
|
|
@@ -127,9 +127,9 @@ edismax_scenarios = {
|
|
|
127
127
|
'body': lambda: PostingsArray.index(["foo bar", "data2", "data3 bar", "bunny funny wunny"],
|
|
128
128
|
tokenizer=just_lowercasing_tokenizer)
|
|
129
129
|
},
|
|
130
|
-
"expected": [lambda frame: max(sum([frame['title'].array.
|
|
131
|
-
frame['title'].array.
|
|
132
|
-
frame['body'].array.
|
|
130
|
+
"expected": [lambda frame: max(sum([frame['title'].array.score("foo")[0],
|
|
131
|
+
frame['title'].array.score("bar")[0]]),
|
|
132
|
+
frame['body'].array.score("foo bar")[0]),
|
|
133
133
|
0,
|
|
134
134
|
0,
|
|
135
135
|
0],
|
|
@@ -140,11 +140,11 @@ edismax_scenarios = {
|
|
|
140
140
|
'title': lambda: PostingsArray.index(["foo bar bar baz", "data2", "data3 bar", "bunny funny wunny"]),
|
|
141
141
|
'body': lambda: PostingsArray.index(["buzz", "data2", "data3 bar", "bunny funny wunny"])
|
|
142
142
|
},
|
|
143
|
-
"expected": [lambda frame: sum([frame['title'].array.
|
|
144
|
-
frame['title'].array.
|
|
143
|
+
"expected": [lambda frame: sum([frame['title'].array.score("foo")[0] * 10,
|
|
144
|
+
frame['title'].array.score("bar")[0] * 10]),
|
|
145
145
|
0,
|
|
146
|
-
lambda frame: max(frame['title'].array.
|
|
147
|
-
frame['body'].array.
|
|
146
|
+
lambda frame: max(frame['title'].array.score("bar")[2] * 10,
|
|
147
|
+
frame['body'].array.score("bar")[2]),
|
|
148
148
|
0],
|
|
149
149
|
"params": {'q': "foo bar", 'qf': ["title^10", "body"]},
|
|
150
150
|
},
|
|
@@ -153,12 +153,12 @@ edismax_scenarios = {
|
|
|
153
153
|
'title': lambda: PostingsArray.index(["foo bar bar baz", "data2", "data3 bar", "bunny funny wunny"]),
|
|
154
154
|
'body': lambda: PostingsArray.index(["buzz", "data2", "data3 bar", "bunny funny wunny"])
|
|
155
155
|
},
|
|
156
|
-
"expected": [lambda frame: sum([frame['title'].array.
|
|
157
|
-
frame['title'].array.
|
|
158
|
-
frame['title'].array.
|
|
156
|
+
"expected": [lambda frame: sum([frame['title'].array.score(["foo", "bar"])[0],
|
|
157
|
+
frame['title'].array.score("foo")[0],
|
|
158
|
+
frame['title'].array.score("bar")[0]]),
|
|
159
159
|
0,
|
|
160
|
-
lambda frame: max(frame['title'].array.
|
|
161
|
-
frame['body'].array.
|
|
160
|
+
lambda frame: max(frame['title'].array.score("bar")[2],
|
|
161
|
+
frame['body'].array.score("bar")[2]),
|
|
162
162
|
0],
|
|
163
163
|
"params": {'q': "foo bar", 'qf': ["title", "body"],
|
|
164
164
|
'pf': ["title"]}
|
|
@@ -169,15 +169,15 @@ edismax_scenarios = {
|
|
|
169
169
|
'body': lambda: PostingsArray.index(["buzz", "data2", "data3 bar", "bunny funny wunny"],
|
|
170
170
|
tokenizer=everythings_a_b_tokenizer)
|
|
171
171
|
},
|
|
172
|
-
"expected": [lambda frame: max(frame['title'].array.
|
|
173
|
-
frame['body'].array.
|
|
172
|
+
"expected": [lambda frame: max(frame['title'].array.score("bar")[0],
|
|
173
|
+
frame['body'].array.score("b")[0]),
|
|
174
174
|
|
|
175
|
-
lambda frame: frame['body'].array.
|
|
175
|
+
lambda frame: frame['body'].array.score("b")[1],
|
|
176
176
|
|
|
177
|
-
lambda frame: max(frame['title'].array.
|
|
178
|
-
frame['body'].array.
|
|
177
|
+
lambda frame: max(frame['title'].array.score("bar")[2],
|
|
178
|
+
frame['body'].array.score("b")[2]),
|
|
179
179
|
|
|
180
|
-
lambda frame: frame['body'].array.
|
|
180
|
+
lambda frame: frame['body'].array.score("b")[3]],
|
|
181
181
|
"params": {'q': "bar", 'qf': ["title", "body"]},
|
|
182
182
|
},
|
|
183
183
|
}
|
|
@@ -206,3 +206,17 @@ def test_edismax(frame, expected, params):
|
|
|
206
206
|
expected = list(compute_expected(expected, frame))
|
|
207
207
|
scores, explain = edismax(frame, **params)
|
|
208
208
|
assert np.allclose(scores, expected)
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
def always_one_similarity(*args, **kwargs) -> np.ndarray:
|
|
212
|
+
term_freqs = kwargs['term_freqs']
|
|
213
|
+
return term_freqs > 0
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
@w_scenarios(edismax_scenarios)
|
|
217
|
+
def test_edismax_custom_similarity(frame, expected, params):
|
|
218
|
+
frame = build_df(frame)
|
|
219
|
+
expected = list(compute_expected(expected, frame))
|
|
220
|
+
params['similarity'] = always_one_similarity
|
|
221
|
+
scores, explain = edismax(frame, **params)
|
|
222
|
+
assert np.all(scores.astype(np.int64) == scores)
|
|
@@ -87,10 +87,23 @@ def test_tokenize_tmdb(tmdb_raw_data):
|
|
|
87
87
|
def test_slice_then_search(tmdb_data):
|
|
88
88
|
star_wars_in_title = tmdb_data['title_tokens'].array.match(["Star", "Wars"])
|
|
89
89
|
star_wars_in_title = tmdb_data[star_wars_in_title]
|
|
90
|
-
skywalker_bm25 = star_wars_in_title['overview_tokens'].array.
|
|
90
|
+
skywalker_bm25 = star_wars_in_title['overview_tokens'].array.score(["Skywalker"])
|
|
91
91
|
assert skywalker_bm25.shape[0] == 3
|
|
92
92
|
|
|
93
93
|
|
|
94
|
+
tmdb_term_matches = [
|
|
95
|
+
("Star", ['11', '330459', '76180']),
|
|
96
|
+
("Black", ['374430']),
|
|
97
|
+
]
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
@pytest.mark.parametrize("term,expected_matches", tmdb_term_matches)
|
|
101
|
+
def test_term_freqs(tmdb_data, term, expected_matches):
|
|
102
|
+
sliced = tmdb_data[tmdb_data['doc_id'].isin(expected_matches)]
|
|
103
|
+
term_freqs = sliced['title_tokens'].array.term_freq(term)
|
|
104
|
+
assert np.all(term_freqs == 1)
|
|
105
|
+
|
|
106
|
+
|
|
94
107
|
tmdb_phrase_matches = [
|
|
95
108
|
(["Star", "Wars"], ['11', '330459', '76180']),
|
|
96
109
|
(["Black", "Mirror:"], ['374430']),
|
|
@@ -151,8 +164,8 @@ def test_gather_results(benchmark, tmdb_data):
|
|
|
151
164
|
N = 10
|
|
152
165
|
all_results = []
|
|
153
166
|
for keywords in [['Star', 'Wars'], ['Black', 'Mirror:'], ['rambo']]:
|
|
154
|
-
score = tmdb_data['title_tokens'].array.
|
|
155
|
-
score += tmdb_data['overview_tokens'].array.
|
|
167
|
+
score = tmdb_data['title_tokens'].array.score(keywords)
|
|
168
|
+
score += tmdb_data['overview_tokens'].array.score(keywords)
|
|
156
169
|
tmdb_data['score'] = score
|
|
157
170
|
top_n = tmdb_data.sort_values('score', ascending=False)[:N].copy()
|
|
158
171
|
top_n.loc[:, 'doc_id'] = top_n['doc_id'].astype(int)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|