searcharray 0.0.25__tar.gz → 0.0.27__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (32) hide show
  1. {searcharray-0.0.25 → searcharray-0.0.27}/PKG-INFO +5 -5
  2. {searcharray-0.0.25 → searcharray-0.0.27}/README.md +4 -4
  3. {searcharray-0.0.25 → searcharray-0.0.27}/searcharray/phrase/middle_out.py +1 -1
  4. {searcharray-0.0.25 → searcharray-0.0.27}/searcharray/postings.py +14 -36
  5. searcharray-0.0.27/searcharray/similarity.py +30 -0
  6. {searcharray-0.0.25 → searcharray-0.0.27}/searcharray/solr.py +17 -8
  7. {searcharray-0.0.25 → searcharray-0.0.27}/searcharray.egg-info/PKG-INFO +5 -5
  8. {searcharray-0.0.25 → searcharray-0.0.27}/searcharray.egg-info/SOURCES.txt +1 -0
  9. {searcharray-0.0.25 → searcharray-0.0.27}/setup.py +1 -1
  10. {searcharray-0.0.25 → searcharray-0.0.27}/test/test_search.py +2 -4
  11. {searcharray-0.0.25 → searcharray-0.0.27}/test/test_solr.py +43 -29
  12. {searcharray-0.0.25 → searcharray-0.0.27}/test/test_tmdb.py +16 -3
  13. {searcharray-0.0.25 → searcharray-0.0.27}/pyproject.toml +0 -0
  14. {searcharray-0.0.25 → searcharray-0.0.27}/searcharray/__init__.py +0 -0
  15. {searcharray-0.0.25 → searcharray-0.0.27}/searcharray/phrase/__init__.py +0 -0
  16. {searcharray-0.0.25 → searcharray-0.0.27}/searcharray/phrase/posn_diffs.py +0 -0
  17. {searcharray-0.0.25 → searcharray-0.0.27}/searcharray/phrase/scan_merge.py +0 -0
  18. {searcharray-0.0.25 → searcharray-0.0.27}/searcharray/term_dict.py +0 -0
  19. {searcharray-0.0.25 → searcharray-0.0.27}/searcharray/utils/__init__.py +0 -0
  20. {searcharray-0.0.25 → searcharray-0.0.27}/searcharray/utils/bitcount.py +0 -0
  21. {searcharray-0.0.25 → searcharray-0.0.27}/searcharray/utils/mat_set.py +0 -0
  22. {searcharray-0.0.25 → searcharray-0.0.27}/searcharray/utils/roaringish.py +0 -0
  23. {searcharray-0.0.25 → searcharray-0.0.27}/searcharray/utils/row_viewable_matrix.py +0 -0
  24. {searcharray-0.0.25 → searcharray-0.0.27}/searcharray.egg-info/dependency_links.txt +0 -0
  25. {searcharray-0.0.25 → searcharray-0.0.27}/searcharray.egg-info/entry_points.txt +0 -0
  26. {searcharray-0.0.25 → searcharray-0.0.27}/searcharray.egg-info/requires.txt +0 -0
  27. {searcharray-0.0.25 → searcharray-0.0.27}/searcharray.egg-info/top_level.txt +0 -0
  28. {searcharray-0.0.25 → searcharray-0.0.27}/setup.cfg +0 -0
  29. {searcharray-0.0.25 → searcharray-0.0.27}/test/test_extension_array.py +0 -0
  30. {searcharray-0.0.25 → searcharray-0.0.27}/test/test_msmarco.py +0 -0
  31. {searcharray-0.0.25 → searcharray-0.0.27}/test/test_phrase_matches.py +0 -0
  32. {searcharray-0.0.25 → searcharray-0.0.27}/test/test_utils.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: searcharray
3
- Version: 0.0.25
3
+ Version: 0.0.27
4
4
  Summary: Searchable pandas text extension arrays for prototyping search
5
5
  Home-page: https://github.com/softwaredoug/searcharray
6
6
  Author: Doug Turnbull
@@ -37,7 +37,7 @@ Think Lucene, but as a Pandas column.
37
37
 
38
38
  ```python
39
39
  In[3]: df['title_indexed'] = PostingsArray.index(df['title'])
40
- np.sort(df['title_indexed'].array.bm25('Cat'))
40
+ np.sort(df['title_indexed'].array.score('Cat'))
41
41
  Out[3]: array([ 0. , 0. , 0. , ..., 15.84568033,
42
42
  15.84568033, 15.84568033])
43
43
  ```
@@ -100,11 +100,11 @@ Out[2]:
100
100
  Then search, getting top N with `Cat`
101
101
 
102
102
  ```
103
- In[3]: np.sort(df['title_indexed'].array.bm25('Cat'))
103
+ In[3]: np.sort(df['title_indexed'].array.score('Cat'))
104
104
  Out[3]: array([ 0. , 0. , 0. , ..., 15.84568033,
105
105
  15.84568033, 15.84568033])
106
106
 
107
- In[4]: df['title_indexed'].bm25('Cat').argsort()
107
+ In[4]: df['title_indexed'].score('Cat').argsort()
108
108
  Out[4]:
109
109
 
110
110
  array([0, 18561, 18560, ..., 15038, 19012, 4392])
@@ -189,7 +189,7 @@ df['hrs_into_past'] = (now - df['timestamp']).dt.total_seconds() / 3600
189
189
  Then multiply by BM25 if you want:
190
190
 
191
191
  ```
192
- df['score'] = df['title_indexed'].bm25('Cat') * df['hrs_into_past']
192
+ df['score'] = df['title_indexed'].score('Cat') * df['hrs_into_past']
193
193
  ```
194
194
 
195
195
  ## TODOs / Future Work / Known issues
@@ -8,7 +8,7 @@ Think Lucene, but as a Pandas column.
8
8
 
9
9
  ```python
10
10
  In[3]: df['title_indexed'] = PostingsArray.index(df['title'])
11
- np.sort(df['title_indexed'].array.bm25('Cat'))
11
+ np.sort(df['title_indexed'].array.score('Cat'))
12
12
  Out[3]: array([ 0. , 0. , 0. , ..., 15.84568033,
13
13
  15.84568033, 15.84568033])
14
14
  ```
@@ -71,11 +71,11 @@ Out[2]:
71
71
  Then search, getting top N with `Cat`
72
72
 
73
73
  ```
74
- In[3]: np.sort(df['title_indexed'].array.bm25('Cat'))
74
+ In[3]: np.sort(df['title_indexed'].array.score('Cat'))
75
75
  Out[3]: array([ 0. , 0. , 0. , ..., 15.84568033,
76
76
  15.84568033, 15.84568033])
77
77
 
78
- In[4]: df['title_indexed'].bm25('Cat').argsort()
78
+ In[4]: df['title_indexed'].score('Cat').argsort()
79
79
  Out[4]:
80
80
 
81
81
  array([0, 18561, 18560, ..., 15038, 19012, 4392])
@@ -160,7 +160,7 @@ df['hrs_into_past'] = (now - df['timestamp']).dt.total_seconds() / 3600
160
160
  Then multiply by BM25 if you want:
161
161
 
162
162
  ```
163
- df['score'] = df['title_indexed'].bm25('Cat') * df['hrs_into_past']
163
+ df['score'] = df['title_indexed'].score('Cat') * df['hrs_into_past']
164
164
  ```
165
165
 
166
166
  ## TODOs / Future Work / Known issues
@@ -316,7 +316,7 @@ class PosnBitArray:
316
316
  doc_ids = encoder.keys(term_posns)
317
317
  change_indices = np.nonzero(np.diff(doc_ids))[0]
318
318
  change_indices = np.concatenate((np.asarray([0]), change_indices + 1))
319
- posns = encoded & encoder.payload_lsb_mask
319
+ posns = term_posns & encoder.payload_lsb_mask
320
320
  bit_counts = bit_count64(posns)
321
321
 
322
322
  term_freqs = np.add.reduceat(bit_counts, change_indices)
@@ -19,6 +19,7 @@ from searcharray.phrase.scan_merge import scan_merge_ins
19
19
  from searcharray.phrase.posn_diffs import compute_phrase_freqs
20
20
  from searcharray.phrase.middle_out import PosnBitArrayBuilder, PosnBitArrayAlreadyEncBuilder, PosnBitArray
21
21
  from searcharray.utils.mat_set import SparseMatSetBuilder
22
+ from searcharray.similarity import Similarity, default_bm25
22
23
 
23
24
  logger = logging.getLogger(__name__)
24
25
 
@@ -618,49 +619,26 @@ class PostingsArray(ExtensionArray):
618
619
  term_freq = self.term_freq(token)
619
620
  return term_freq > 0
620
621
 
621
- def bm25_idf(self, token: Union[List[str], str], doc_stats=None) -> float:
622
- """Calculate the (Lucene) idf for a term.
623
-
624
- idf, computed as log(1 + (N - n + 0.5) / (n + 0.5))
625
- """
626
- token = self._check_token_arg(token)
627
- if isinstance(token, list):
628
- return self.bm25_phrase_idf(token)
629
-
630
- df = self.doc_freq(token)
631
- num_docs = len(self)
632
- return np.log(1 + (num_docs - df + 0.5) / (df + 0.5))
633
-
634
- def bm25_phrase_idf(self, tokens: List[str]) -> float:
635
- """Calculate the idf for a phrase.
636
-
637
- This is the sum of the idfs of the individual terms.
638
- """
639
- idfs = [self.bm25_idf(term) for term in tokens]
640
- return np.sum(idfs)
641
-
642
- def bm25_tf(self, token: Union[List[str], str], k1=1.2, b=0.75, slop=1) -> float:
643
- """Calculate the (Lucene) BM25 tf for a term or phrase.
644
-
645
- tf, computed as freq / (freq + k1 * (1 - b + b * dl / avgdl))
646
- """
647
- tf = self.term_freq(token)
648
- score = tf / (tf + k1 * (1 - b + b * self.doc_lengths() / self.avg_doc_length))
649
- return score
650
-
651
- def bm25(self, token: Union[str, List[str]], doc_stats=None, k1=1.2, b=0.75):
652
- """Score each doc using BM25.
622
+ def score(self, token: Union[str, List[str]], similarity: Similarity = default_bm25) -> np.ndarray:
623
+ """Score each doc using a similarity function.
653
624
 
654
625
  Parameters
655
626
  ----------
656
627
  token : str or list of str of what to search (already tokenized)
657
- doc_stats : tuple of doc stats to use (avg_doc_length, num_docs, doc_count). Defaults to index stats.
658
- k1 : float, optional BM25 param. Defaults to 1.2.
659
- b : float, optional BM25 param. Defaults to 0.75.
628
+ similarity : How to score the documents. Default is BM25.
660
629
  """
661
630
  # Get term freqs per token
662
631
  token = self._check_token_arg(token)
663
- return self.bm25_idf(token, doc_stats=doc_stats) * self.bm25_tf(token)
632
+
633
+ tfs = self.term_freq(token)
634
+ token = self._check_token_arg(token)
635
+ tokens_l = [token] if isinstance(token, str) else token
636
+ all_dfs = np.asarray([self.doc_freq(token) for token in tokens_l])
637
+ doc_lens = self.doc_lengths()
638
+ scores = similarity(term_freqs=tfs, doc_freqs=all_dfs,
639
+ doc_lens=doc_lens, avg_doc_lens=self.avg_doc_length,
640
+ num_docs=len(self))
641
+ return scores
664
642
 
665
643
  def positions(self, token: str, key=None) -> List[np.ndarray]:
666
644
  """Return a list of lists of positions of the given term."""
@@ -0,0 +1,30 @@
1
+ """Similarity functions given term stats."""
2
+ from typing import Protocol
3
+ import numpy as np
4
+
5
+
6
+ class Similarity(Protocol):
7
+ """Similarity function protocol."""
8
+
9
+ def __call__(self, term_freqs: np.ndarray, doc_freqs: np.ndarray, doc_lens: np.ndarray,
10
+ avg_doc_lens: int, num_docs: int) -> np.ndarray:
11
+ """Calculate similarity scores."""
12
+ ...
13
+
14
+
15
+ def bm25_similarity(k1: float = 1.2, b: float = 0.75) -> Similarity:
16
+ def bm25(term_freqs: np.ndarray, doc_freqs: np.ndarray,
17
+ doc_lens: np.ndarray,
18
+ avg_doc_lens: int, num_docs: int) -> np.ndarray:
19
+ """Calculate BM25 scores."""
20
+ # Sum doc freqs
21
+ sum_dfs = np.sum(doc_freqs, axis=0)
22
+ # Calculate idf
23
+ idf = np.log(1 + (num_docs - sum_dfs + 0.5) / (sum_dfs + 0.5))
24
+ # Calculate tf
25
+ tf = term_freqs / (term_freqs + k1 * (1 - b + b * doc_lens / avg_doc_lens))
26
+ return idf * tf
27
+ return bm25
28
+
29
+
30
+ default_bm25 = bm25_similarity()
@@ -4,6 +4,7 @@ import pandas as pd
4
4
  import numpy as np
5
5
  from typing import List, Optional, Dict, Tuple
6
6
  from searcharray.postings import PostingsArray
7
+ from searcharray.similarity import Similarity, default_bm25
7
8
 
8
9
 
9
10
  def parse_min_should_match(num_clauses: int, spec: str) -> int:
@@ -111,7 +112,8 @@ def _edismax_term_centric(frame: pd.DataFrame,
111
112
  query_fields: Dict[str, float],
112
113
  num_search_terms: int,
113
114
  search_terms: Dict[str, List[str]],
114
- mm: str) -> Tuple[np.ndarray, str]:
115
+ mm: str,
116
+ similarity: Similarity) -> Tuple[np.ndarray, str]:
115
117
  explain = []
116
118
  term_scores = []
117
119
  for term_posn in range(num_search_terms):
@@ -120,7 +122,7 @@ def _edismax_term_centric(frame: pd.DataFrame,
120
122
  for field, boost in query_fields.items():
121
123
  term = search_terms[field][term_posn]
122
124
  post_arr = get_field(frame, field)
123
- field_term_score = post_arr.bm25(term) * (1 if boost is None else boost)
125
+ field_term_score = post_arr.score(term, similarity=similarity) * (1 if boost is None else boost)
124
126
  boost_exp = f"{boost}" if boost is not None else "1"
125
127
  term_explain.append(f"{field}:{term}^{boost_exp}")
126
128
  max_scores = np.maximum(max_scores, field_term_score)
@@ -139,12 +141,14 @@ def _edismax_field_centric(frame: pd.DataFrame,
139
141
  query_fields: Dict[str, float],
140
142
  num_search_terms: int,
141
143
  search_terms: Dict[str, List[str]],
142
- mm: str) -> Tuple[np.ndarray, str]:
144
+ mm: str,
145
+ similarity: Similarity = default_bm25) -> Tuple[np.ndarray, str]:
143
146
  field_scores = []
144
147
  explain = []
145
148
  for field, boost in query_fields.items():
146
149
  post_arr = get_field(frame, field)
147
- term_scores = np.array([post_arr.bm25(term) for term in search_terms[field]])
150
+ term_scores = np.array([post_arr.score(term, similarity=similarity)
151
+ for term in search_terms[field]])
148
152
  min_should_match = parse_min_should_match(len(search_terms[field]), spec=mm)
149
153
  exp = " ".join([f"{field}:{term}" for term in search_terms[field]])
150
154
  boost_exp = f"{boost}" if boost is not None else "1"
@@ -169,7 +173,8 @@ def edismax(frame: pd.DataFrame,
169
173
  pf: Optional[List[str]] = None,
170
174
  pf2: Optional[List[str]] = None,
171
175
  pf3: Optional[List[str]] = None,
172
- q_op: str = "OR") -> Tuple[np.ndarray, str]:
176
+ q_op: str = "OR",
177
+ similarity: Similarity = default_bm25) -> Tuple[np.ndarray, str]:
173
178
  """Run edismax search over dataframe with searcharray fields.
174
179
 
175
180
  Parameters
@@ -209,15 +214,19 @@ def edismax(frame: pd.DataFrame,
209
214
 
210
215
  num_search_terms, search_terms, term_centric = parse_query_terms(frame, q, list(query_fields.keys()))
211
216
  if term_centric:
212
- qf_scores, explain = _edismax_term_centric(frame, query_fields, num_search_terms, search_terms, mm)
217
+ qf_scores, explain = _edismax_term_centric(frame, query_fields,
218
+ num_search_terms, search_terms, mm,
219
+ similarity=similarity)
213
220
  else:
214
- qf_scores, explain = _edismax_field_centric(frame, query_fields, num_search_terms, search_terms, mm)
221
+ qf_scores, explain = _edismax_field_centric(frame, query_fields,
222
+ num_search_terms, search_terms, mm,
223
+ similarity=similarity)
215
224
 
216
225
  phrase_scores = []
217
226
  for field, boost in phrase_fields.items():
218
227
  arr = get_field(frame, field)
219
228
  terms = search_terms[field]
220
- field_phrase_score = arr.bm25(terms) * (1 if boost is None else boost)
229
+ field_phrase_score = arr.score(terms, similarity=similarity) * (1 if boost is None else boost)
221
230
  boost_exp = f"{boost}" if boost is not None else "1"
222
231
  explain += f" ({field}:\"{' '.join(terms)}\")^{boost_exp}"
223
232
  phrase_scores.append(field_phrase_score)
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: searcharray
3
- Version: 0.0.25
3
+ Version: 0.0.27
4
4
  Summary: Searchable pandas text extension arrays for prototyping search
5
5
  Home-page: https://github.com/softwaredoug/searcharray
6
6
  Author: Doug Turnbull
@@ -37,7 +37,7 @@ Think Lucene, but as a Pandas column.
37
37
 
38
38
  ```python
39
39
  In[3]: df['title_indexed'] = PostingsArray.index(df['title'])
40
- np.sort(df['title_indexed'].array.bm25('Cat'))
40
+ np.sort(df['title_indexed'].array.score('Cat'))
41
41
  Out[3]: array([ 0. , 0. , 0. , ..., 15.84568033,
42
42
  15.84568033, 15.84568033])
43
43
  ```
@@ -100,11 +100,11 @@ Out[2]:
100
100
  Then search, getting top N with `Cat`
101
101
 
102
102
  ```
103
- In[3]: np.sort(df['title_indexed'].array.bm25('Cat'))
103
+ In[3]: np.sort(df['title_indexed'].array.score('Cat'))
104
104
  Out[3]: array([ 0. , 0. , 0. , ..., 15.84568033,
105
105
  15.84568033, 15.84568033])
106
106
 
107
- In[4]: df['title_indexed'].bm25('Cat').argsort()
107
+ In[4]: df['title_indexed'].score('Cat').argsort()
108
108
  Out[4]:
109
109
 
110
110
  array([0, 18561, 18560, ..., 15038, 19012, 4392])
@@ -189,7 +189,7 @@ df['hrs_into_past'] = (now - df['timestamp']).dt.total_seconds() / 3600
189
189
  Then multiply by BM25 if you want:
190
190
 
191
191
  ```
192
- df['score'] = df['title_indexed'].bm25('Cat') * df['hrs_into_past']
192
+ df['score'] = df['title_indexed'].score('Cat') * df['hrs_into_past']
193
193
  ```
194
194
 
195
195
  ## TODOs / Future Work / Known issues
@@ -4,6 +4,7 @@ setup.cfg
4
4
  setup.py
5
5
  searcharray/__init__.py
6
6
  searcharray/postings.py
7
+ searcharray/similarity.py
7
8
  searcharray/solr.py
8
9
  searcharray/term_dict.py
9
10
  searcharray.egg-info/PKG-INFO
@@ -36,7 +36,7 @@ setup(
36
36
  # For a discussion on single-sourcing the version across setup.py and the
37
37
  # project code, see
38
38
  # https://packaging.python.org/guides/single-sourcing-package-version/
39
- version="0.0.25", # Required
39
+ version="0.0.27", # Required
40
40
  # This is a one-line description or tagline of what your project does. This
41
41
  # corresponds to the "Summary" metadata field:
42
42
  # https://packaging.python.org/specifications/core-metadata/#summary
@@ -40,10 +40,8 @@ def test_doc_lengths(data):
40
40
  assert data.avg_doc_length == 2.5
41
41
 
42
42
 
43
- def test_bm25_matches_lucene(data):
44
- bm25_idf = data.bm25_idf("bar")
45
- assert bm25_idf > 0.0
46
- bm25 = data.bm25("bar")
43
+ def test_default_score_matches_lucene(data):
44
+ bm25 = data.score("bar")
47
45
  assert bm25.shape == (100,)
48
46
  assert np.isclose(bm25, [0.37066694, 0., 0.34314217, 0.] * 25).all()
49
47
 
@@ -85,11 +85,11 @@ edismax_scenarios = {
85
85
  'title': lambda: PostingsArray.index(["foo bar bar baz", "data2", "data3 bar", "bunny funny wunny"]),
86
86
  'body': lambda: PostingsArray.index(["buzz", "data2", "data3 bar", "bunny funny wunny"])
87
87
  },
88
- "expected": [lambda frame: sum([frame['title'].array.bm25("foo")[0],
89
- frame['title'].array.bm25("bar")[0]]),
88
+ "expected": [lambda frame: sum([frame['title'].array.score("foo")[0],
89
+ frame['title'].array.score("bar")[0]]),
90
90
  0,
91
- lambda frame: max(frame['title'].array.bm25("bar")[2],
92
- frame['body'].array.bm25("bar")[2]),
91
+ lambda frame: max(frame['title'].array.score("bar")[2],
92
+ frame['body'].array.score("bar")[2]),
93
93
  0],
94
94
  "params": {'q': "foo bar", 'qf': ["title", "body"]},
95
95
  },
@@ -99,11 +99,11 @@ edismax_scenarios = {
99
99
  'body': lambda: PostingsArray.index(["foo bar", "data2", "data3 bar", "bunny funny wunny"],
100
100
  tokenizer=just_lowercasing_tokenizer)
101
101
  },
102
- "expected": [lambda frame: max(sum([frame['title'].array.bm25("foo")[0],
103
- frame['title'].array.bm25("bar")[0]]),
104
- frame['body'].array.bm25("foo bar")[0]),
102
+ "expected": [lambda frame: max(sum([frame['title'].array.score("foo")[0],
103
+ frame['title'].array.score("bar")[0]]),
104
+ frame['body'].array.score("foo bar")[0]),
105
105
  0,
106
- lambda frame: frame['title'].array.bm25("bar")[2],
106
+ lambda frame: frame['title'].array.score("bar")[2],
107
107
  0],
108
108
  "params": {'q': "foo bar", 'qf': ["title", "body"]},
109
109
  },
@@ -113,9 +113,9 @@ edismax_scenarios = {
113
113
  'body': lambda: PostingsArray.index(["foo bar", "data2", "data3 bar", "bunny funny wunny"],
114
114
  tokenizer=just_lowercasing_tokenizer)
115
115
  },
116
- "expected": [lambda frame: max(sum([frame['title'].array.bm25("foo")[0],
117
- frame['title'].array.bm25("bar")[0]]),
118
- frame['body'].array.bm25("foo bar")[0]),
116
+ "expected": [lambda frame: max(sum([frame['title'].array.score("foo")[0],
117
+ frame['title'].array.score("bar")[0]]),
118
+ frame['body'].array.score("foo bar")[0]),
119
119
  0,
120
120
  0,
121
121
  0],
@@ -127,9 +127,9 @@ edismax_scenarios = {
127
127
  'body': lambda: PostingsArray.index(["foo bar", "data2", "data3 bar", "bunny funny wunny"],
128
128
  tokenizer=just_lowercasing_tokenizer)
129
129
  },
130
- "expected": [lambda frame: max(sum([frame['title'].array.bm25("foo")[0],
131
- frame['title'].array.bm25("bar")[0]]),
132
- frame['body'].array.bm25("foo bar")[0]),
130
+ "expected": [lambda frame: max(sum([frame['title'].array.score("foo")[0],
131
+ frame['title'].array.score("bar")[0]]),
132
+ frame['body'].array.score("foo bar")[0]),
133
133
  0,
134
134
  0,
135
135
  0],
@@ -140,11 +140,11 @@ edismax_scenarios = {
140
140
  'title': lambda: PostingsArray.index(["foo bar bar baz", "data2", "data3 bar", "bunny funny wunny"]),
141
141
  'body': lambda: PostingsArray.index(["buzz", "data2", "data3 bar", "bunny funny wunny"])
142
142
  },
143
- "expected": [lambda frame: sum([frame['title'].array.bm25("foo")[0] * 10,
144
- frame['title'].array.bm25("bar")[0] * 10]),
143
+ "expected": [lambda frame: sum([frame['title'].array.score("foo")[0] * 10,
144
+ frame['title'].array.score("bar")[0] * 10]),
145
145
  0,
146
- lambda frame: max(frame['title'].array.bm25("bar")[2] * 10,
147
- frame['body'].array.bm25("bar")[2]),
146
+ lambda frame: max(frame['title'].array.score("bar")[2] * 10,
147
+ frame['body'].array.score("bar")[2]),
148
148
  0],
149
149
  "params": {'q': "foo bar", 'qf': ["title^10", "body"]},
150
150
  },
@@ -153,12 +153,12 @@ edismax_scenarios = {
153
153
  'title': lambda: PostingsArray.index(["foo bar bar baz", "data2", "data3 bar", "bunny funny wunny"]),
154
154
  'body': lambda: PostingsArray.index(["buzz", "data2", "data3 bar", "bunny funny wunny"])
155
155
  },
156
- "expected": [lambda frame: sum([frame['title'].array.bm25(["foo", "bar"])[0],
157
- frame['title'].array.bm25("foo")[0],
158
- frame['title'].array.bm25("bar")[0]]),
156
+ "expected": [lambda frame: sum([frame['title'].array.score(["foo", "bar"])[0],
157
+ frame['title'].array.score("foo")[0],
158
+ frame['title'].array.score("bar")[0]]),
159
159
  0,
160
- lambda frame: max(frame['title'].array.bm25("bar")[2],
161
- frame['body'].array.bm25("bar")[2]),
160
+ lambda frame: max(frame['title'].array.score("bar")[2],
161
+ frame['body'].array.score("bar")[2]),
162
162
  0],
163
163
  "params": {'q': "foo bar", 'qf': ["title", "body"],
164
164
  'pf': ["title"]}
@@ -169,15 +169,15 @@ edismax_scenarios = {
169
169
  'body': lambda: PostingsArray.index(["buzz", "data2", "data3 bar", "bunny funny wunny"],
170
170
  tokenizer=everythings_a_b_tokenizer)
171
171
  },
172
- "expected": [lambda frame: max(frame['title'].array.bm25("bar")[0],
173
- frame['body'].array.bm25("b")[0]),
172
+ "expected": [lambda frame: max(frame['title'].array.score("bar")[0],
173
+ frame['body'].array.score("b")[0]),
174
174
 
175
- lambda frame: frame['body'].array.bm25("b")[1],
175
+ lambda frame: frame['body'].array.score("b")[1],
176
176
 
177
- lambda frame: max(frame['title'].array.bm25("bar")[2],
178
- frame['body'].array.bm25("b")[2]),
177
+ lambda frame: max(frame['title'].array.score("bar")[2],
178
+ frame['body'].array.score("b")[2]),
179
179
 
180
- lambda frame: frame['body'].array.bm25("b")[3]],
180
+ lambda frame: frame['body'].array.score("b")[3]],
181
181
  "params": {'q': "bar", 'qf': ["title", "body"]},
182
182
  },
183
183
  }
@@ -206,3 +206,17 @@ def test_edismax(frame, expected, params):
206
206
  expected = list(compute_expected(expected, frame))
207
207
  scores, explain = edismax(frame, **params)
208
208
  assert np.allclose(scores, expected)
209
+
210
+
211
+ def always_one_similarity(*args, **kwargs) -> np.ndarray:
212
+ term_freqs = kwargs['term_freqs']
213
+ return term_freqs > 0
214
+
215
+
216
+ @w_scenarios(edismax_scenarios)
217
+ def test_edismax_custom_similarity(frame, expected, params):
218
+ frame = build_df(frame)
219
+ expected = list(compute_expected(expected, frame))
220
+ params['similarity'] = always_one_similarity
221
+ scores, explain = edismax(frame, **params)
222
+ assert np.all(scores.astype(np.int64) == scores)
@@ -87,10 +87,23 @@ def test_tokenize_tmdb(tmdb_raw_data):
87
87
  def test_slice_then_search(tmdb_data):
88
88
  star_wars_in_title = tmdb_data['title_tokens'].array.match(["Star", "Wars"])
89
89
  star_wars_in_title = tmdb_data[star_wars_in_title]
90
- skywalker_bm25 = star_wars_in_title['overview_tokens'].array.bm25(["Skywalker"])
90
+ skywalker_bm25 = star_wars_in_title['overview_tokens'].array.score(["Skywalker"])
91
91
  assert skywalker_bm25.shape[0] == 3
92
92
 
93
93
 
94
+ tmdb_term_matches = [
95
+ ("Star", ['11', '330459', '76180']),
96
+ ("Black", ['374430']),
97
+ ]
98
+
99
+
100
+ @pytest.mark.parametrize("term,expected_matches", tmdb_term_matches)
101
+ def test_term_freqs(tmdb_data, term, expected_matches):
102
+ sliced = tmdb_data[tmdb_data['doc_id'].isin(expected_matches)]
103
+ term_freqs = sliced['title_tokens'].array.term_freq(term)
104
+ assert np.all(term_freqs == 1)
105
+
106
+
94
107
  tmdb_phrase_matches = [
95
108
  (["Star", "Wars"], ['11', '330459', '76180']),
96
109
  (["Black", "Mirror:"], ['374430']),
@@ -151,8 +164,8 @@ def test_gather_results(benchmark, tmdb_data):
151
164
  N = 10
152
165
  all_results = []
153
166
  for keywords in [['Star', 'Wars'], ['Black', 'Mirror:'], ['rambo']]:
154
- score = tmdb_data['title_tokens'].array.bm25(keywords)
155
- score += tmdb_data['overview_tokens'].array.bm25(keywords)
167
+ score = tmdb_data['title_tokens'].array.score(keywords)
168
+ score += tmdb_data['overview_tokens'].array.score(keywords)
156
169
  tmdb_data['score'] = score
157
170
  top_n = tmdb_data.sort_values('score', ascending=False)[:N].copy()
158
171
  top_n.loc[:, 'doc_id'] = top_n['doc_id'].astype(int)
File without changes