searcharray 0.0.30__tar.gz → 0.0.31__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (34) hide show
  1. {searcharray-0.0.30 → searcharray-0.0.31}/PKG-INFO +27 -24
  2. {searcharray-0.0.30 → searcharray-0.0.31}/README.md +26 -23
  3. searcharray-0.0.31/searcharray/__init__.py +1 -0
  4. {searcharray-0.0.30 → searcharray-0.0.31}/searcharray/postings.py +45 -45
  5. {searcharray-0.0.30 → searcharray-0.0.31}/searcharray/solr.py +3 -3
  6. {searcharray-0.0.30 → searcharray-0.0.31}/searcharray.egg-info/PKG-INFO +27 -24
  7. {searcharray-0.0.30 → searcharray-0.0.31}/setup.py +1 -1
  8. {searcharray-0.0.30 → searcharray-0.0.31}/test/test_extension_array.py +13 -12
  9. {searcharray-0.0.30 → searcharray-0.0.31}/test/test_msmarco.py +5 -5
  10. {searcharray-0.0.30 → searcharray-0.0.31}/test/test_phrase_matches.py +59 -59
  11. {searcharray-0.0.30 → searcharray-0.0.31}/test/test_search.py +11 -11
  12. {searcharray-0.0.30 → searcharray-0.0.31}/test/test_solr.py +19 -19
  13. {searcharray-0.0.30 → searcharray-0.0.31}/test/test_tmdb.py +7 -7
  14. searcharray-0.0.30/searcharray/utils/__init__.py +0 -0
  15. {searcharray-0.0.30 → searcharray-0.0.31}/pyproject.toml +0 -0
  16. {searcharray-0.0.30/searcharray → searcharray-0.0.31/searcharray/phrase}/__init__.py +0 -0
  17. {searcharray-0.0.30 → searcharray-0.0.31}/searcharray/phrase/middle_out.py +0 -0
  18. {searcharray-0.0.30 → searcharray-0.0.31}/searcharray/phrase/posn_diffs.py +0 -0
  19. {searcharray-0.0.30 → searcharray-0.0.31}/searcharray/phrase/scan_merge.py +0 -0
  20. {searcharray-0.0.30 → searcharray-0.0.31}/searcharray/similarity.py +0 -0
  21. {searcharray-0.0.30 → searcharray-0.0.31}/searcharray/term_dict.py +0 -0
  22. {searcharray-0.0.30/searcharray/phrase → searcharray-0.0.31/searcharray/utils}/__init__.py +0 -0
  23. {searcharray-0.0.30 → searcharray-0.0.31}/searcharray/utils/bitcount.py +0 -0
  24. {searcharray-0.0.30 → searcharray-0.0.31}/searcharray/utils/cache.py +0 -0
  25. {searcharray-0.0.30 → searcharray-0.0.31}/searcharray/utils/mat_set.py +0 -0
  26. {searcharray-0.0.30 → searcharray-0.0.31}/searcharray/utils/roaringish.py +0 -0
  27. {searcharray-0.0.30 → searcharray-0.0.31}/searcharray/utils/row_viewable_matrix.py +0 -0
  28. {searcharray-0.0.30 → searcharray-0.0.31}/searcharray.egg-info/SOURCES.txt +0 -0
  29. {searcharray-0.0.30 → searcharray-0.0.31}/searcharray.egg-info/dependency_links.txt +0 -0
  30. {searcharray-0.0.30 → searcharray-0.0.31}/searcharray.egg-info/entry_points.txt +0 -0
  31. {searcharray-0.0.30 → searcharray-0.0.31}/searcharray.egg-info/requires.txt +0 -0
  32. {searcharray-0.0.30 → searcharray-0.0.31}/searcharray.egg-info/top_level.txt +0 -0
  33. {searcharray-0.0.30 → searcharray-0.0.31}/setup.cfg +0 -0
  34. {searcharray-0.0.30 → searcharray-0.0.31}/test/test_utils.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: searcharray
3
- Version: 0.0.30
3
+ Version: 0.0.31
4
4
  Summary: Searchable pandas text extension arrays for prototyping search
5
5
  Home-page: https://github.com/softwaredoug/searcharray
6
6
  Author: Doug Turnbull
@@ -36,8 +36,11 @@ SearchArray turns Pandas string columns into a term index. It alows efficient BM
36
36
  Think Lucene, but as a Pandas column.
37
37
 
38
38
  ```python
39
- In[3]: df['title_indexed'] = PostingsArray.index(df['title'])
39
+ In[2]: from searcharray import SearchArray
40
+
41
+ In[3]: df['title_indexed'] = SearchArray.index(df['title'])
40
42
  np.sort(df['title_indexed'].array.score('Cat'))
43
+
41
44
  Out[3]: array([ 0. , 0. , 0. , ..., 15.84568033,
42
45
  15.84568033, 15.84568033])
43
46
  ```
@@ -79,22 +82,22 @@ Out[1]:
79
82
  Index the text:
80
83
 
81
84
  ```
82
- In[2]: df['title_indexed'] = PostingsArray.index(df['title'])
85
+ In[2]: df['title_indexed'] = SearchArray.index(df['title'])
83
86
  df
84
87
 
85
88
  Out[2]:
86
89
  title overview title_indexed
87
- 374430 Black Mirror: White Christmas This feature-length special consists of three ... PostingsRow({'Black': 1, 'Mirror:': 1, 'White'...
88
- 19404 The Brave-Hearted Will Take the Bride Raj is a rich, carefree, happy-go-lucky second... PostingsRow({'The': 1, 'Brave-Hearted': 1, 'Wi...
89
- 278 The Shawshank Redemption Framed in the 1940s for the double murder of h... PostingsRow({'The': 1, 'Shawshank': 1, 'Redemp...
90
- 372058 Your Name. High schoolers Mitsuha and Taki are complete s... PostingsRow({'Your': 1, 'Name.': 1}, {'Your': ...
91
- 238 The Godfather Spanning the years 1945 to 1955, a chronicle o... PostingsRow({'The': 1, 'Godfather': 1}, {'The'...
90
+ 374430 Black Mirror: White Christmas This feature-length special consists of three ... Terms({'Black': 1, 'Mirror:': 1, 'White'...
91
+ 19404 The Brave-Hearted Will Take the Bride Raj is a rich, carefree, happy-go-lucky second... Terms({'The': 1, 'Brave-Hearted': 1, 'Wi...
92
+ 278 The Shawshank Redemption Framed in the 1940s for the double murder of h... Terms({'The': 1, 'Shawshank': 1, 'Redemp...
93
+ 372058 Your Name. High schoolers Mitsuha and Taki are complete s... Terms({'Your': 1, 'Name.': 1}, {'Your': ...
94
+ 238 The Godfather Spanning the years 1945 to 1955, a chronicle o... Terms({'The': 1, 'Godfather': 1}, {'The'...
92
95
  ... ... ... ...
93
- 65513 They Came Back The lives of the residents of a small French t... PostingsRow({'Back': 1, 'They': 1, 'Came': 1},...
94
- 65515 The Eleventh Hour An ex-Navy SEAL, Michael Adams, (Matthew Reese... PostingsRow({'The': 1, 'Hour': 1, 'Eleventh': ...
95
- 65521 Pyaar Ka Punchnama Outspoken and overly critical Nishant Agarwal ... PostingsRow({'Ka': 1, 'Pyaar': 1, 'Punchnama':...
96
- 32767 Romero Romero is a compelling and deeply moving look ... PostingsRow({'Romero': 1}, {'Romero': [0]})
97
- 65534 Poison Paul Braconnier and his wife Blandine only hav... PostingsRow({'Poison': 1}, {'Poison': [0]})```
96
+ 65513 They Came Back The lives of the residents of a small French t... Terms({'Back': 1, 'They': 1, 'Came': 1},...
97
+ 65515 The Eleventh Hour An ex-Navy SEAL, Michael Adams, (Matthew Reese... Terms({'The': 1, 'Hour': 1, 'Eleventh': ...
98
+ 65521 Pyaar Ka Punchnama Outspoken and overly critical Nishant Agarwal ... Terms({'Ka': 1, 'Pyaar': 1, 'Punchnama':...
99
+ 32767 Romero Romero is a compelling and deeply moving look ... Terms({'Romero': 1}, {'Romero': [0]})
100
+ 65534 Poison Paul Braconnier and his wife Blandine only hav... Terms({'Poison': 1}, {'Poison': [0]})```
98
101
  ```
99
102
 
100
103
  Then search, getting top N with `Cat`
@@ -116,16 +119,16 @@ And since its just pandas, we can, of course just retrieve the top matches
116
119
  In[5]: df.iloc[top_n_cat[-10:]]
117
120
  Out[5]:
118
121
  title overview title_indexed
119
- 24106 The Black Cat American honeymooners in Hungary are trapped i... PostingsRow({'Black': 1, 'The': 1, 'Cat': 1}, ...
120
- 12593 Fritz the Cat A hypocritical swinging college student cat ra... PostingsRow({'Cat': 1, 'the': 1, 'Fritz': 1}, ...
121
- 39853 The Cat Concerto Tom enters from stage left in white tie and ta... PostingsRow({'The': 1, 'Cat': 1, 'Concerto': 1...
122
- 75491 The Rabbi's Cat Based on the best-selling graphic novel by Joa... PostingsRow({'The': 1, 'Cat': 1, "Rabbi's": 1}...
123
- 57353 Cat Run When a sexy, high-end escort holds the key evi... PostingsRow({'Cat': 1, 'Run': 1}, {'Cat': [0],...
124
- 25508 Cat People Sketch artist Irena Dubrovna (Simon) and Ameri... PostingsRow({'Cat': 1, 'People': 1}, {'Cat': [...
125
- 11694 Cat Ballou A woman seeking revenge for her murdered fathe... PostingsRow({'Cat': 1, 'Ballou': 1}, {'Cat': [...
126
- 25078 Cat Soup The surreal black comedy follows Nyatta, an an... PostingsRow({'Cat': 1, 'Soup': 1}, {'Cat': [0]...
127
- 35888 Cat Chaser A Miami hotel owner finds danger when be becom... PostingsRow({'Cat': 1, 'Chaser': 1}, {'Cat': [...
128
- 6217 Cat People After years of separation, Irina (Nastassja Ki... PostingsRow({'Cat': 1, 'People': 1}, {'Cat': [...
122
+ 24106 The Black Cat American honeymooners in Hungary are trapped i... Terms({'Black': 1, 'The': 1, 'Cat': 1}, ...
123
+ 12593 Fritz the Cat A hypocritical swinging college student cat ra... Terms({'Cat': 1, 'the': 1, 'Fritz': 1}, ...
124
+ 39853 The Cat Concerto Tom enters from stage left in white tie and ta... Terms({'The': 1, 'Cat': 1, 'Concerto': 1...
125
+ 75491 The Rabbi's Cat Based on the best-selling graphic novel by Joa... Terms({'The': 1, 'Cat': 1, "Rabbi's": 1}...
126
+ 57353 Cat Run When a sexy, high-end escort holds the key evi... Terms({'Cat': 1, 'Run': 1}, {'Cat': [0],...
127
+ 25508 Cat People Sketch artist Irena Dubrovna (Simon) and Ameri... Terms({'Cat': 1, 'People': 1}, {'Cat': [...
128
+ 11694 Cat Ballou A woman seeking revenge for her murdered fathe... Terms({'Cat': 1, 'Ballou': 1}, {'Cat': [...
129
+ 25078 Cat Soup The surreal black comedy follows Nyatta, an an... Terms({'Cat': 1, 'Soup': 1}, {'Cat': [0]...
130
+ 35888 Cat Chaser A Miami hotel owner finds danger when be becom... Terms({'Cat': 1, 'Chaser': 1}, {'Cat': [...
131
+ 6217 Cat People After years of separation, Irina (Nastassja Ki... Terms({'Cat': 1, 'People': 1}, {'Cat': [...
129
132
  ```
130
133
 
131
134
  More use cases can be seen [in the colab notebook](https://colab.research.google.com/drive/1w_Ajn5rHzcISKhdCuPhhVFav3zrvKWn1)
@@ -170,7 +173,7 @@ And you can pass any tokenizer that matches this signature to index:
170
173
  def ws_lowercase_tokenizer(string):
171
174
  return string.lower().split()
172
175
 
173
- df['title_indexed'] = PostingsArray.index(df['title'], tokenizer=ws_lowercase_tokenizer)
176
+ df['title_indexed'] = SearchArray.index(df['title'], tokenizer=ws_lowercase_tokenizer)
174
177
  ```
175
178
 
176
179
  Create your own using stemming libraries, or whatever Python functionality you want.
@@ -7,8 +7,11 @@ SearchArray turns Pandas string columns into a term index. It alows efficient BM
7
7
  Think Lucene, but as a Pandas column.
8
8
 
9
9
  ```python
10
- In[3]: df['title_indexed'] = PostingsArray.index(df['title'])
10
+ In[2]: from searcharray import SearchArray
11
+
12
+ In[3]: df['title_indexed'] = SearchArray.index(df['title'])
11
13
  np.sort(df['title_indexed'].array.score('Cat'))
14
+
12
15
  Out[3]: array([ 0. , 0. , 0. , ..., 15.84568033,
13
16
  15.84568033, 15.84568033])
14
17
  ```
@@ -50,22 +53,22 @@ Out[1]:
50
53
  Index the text:
51
54
 
52
55
  ```
53
- In[2]: df['title_indexed'] = PostingsArray.index(df['title'])
56
+ In[2]: df['title_indexed'] = SearchArray.index(df['title'])
54
57
  df
55
58
 
56
59
  Out[2]:
57
60
  title overview title_indexed
58
- 374430 Black Mirror: White Christmas This feature-length special consists of three ... PostingsRow({'Black': 1, 'Mirror:': 1, 'White'...
59
- 19404 The Brave-Hearted Will Take the Bride Raj is a rich, carefree, happy-go-lucky second... PostingsRow({'The': 1, 'Brave-Hearted': 1, 'Wi...
60
- 278 The Shawshank Redemption Framed in the 1940s for the double murder of h... PostingsRow({'The': 1, 'Shawshank': 1, 'Redemp...
61
- 372058 Your Name. High schoolers Mitsuha and Taki are complete s... PostingsRow({'Your': 1, 'Name.': 1}, {'Your': ...
62
- 238 The Godfather Spanning the years 1945 to 1955, a chronicle o... PostingsRow({'The': 1, 'Godfather': 1}, {'The'...
61
+ 374430 Black Mirror: White Christmas This feature-length special consists of three ... Terms({'Black': 1, 'Mirror:': 1, 'White'...
62
+ 19404 The Brave-Hearted Will Take the Bride Raj is a rich, carefree, happy-go-lucky second... Terms({'The': 1, 'Brave-Hearted': 1, 'Wi...
63
+ 278 The Shawshank Redemption Framed in the 1940s for the double murder of h... Terms({'The': 1, 'Shawshank': 1, 'Redemp...
64
+ 372058 Your Name. High schoolers Mitsuha and Taki are complete s... Terms({'Your': 1, 'Name.': 1}, {'Your': ...
65
+ 238 The Godfather Spanning the years 1945 to 1955, a chronicle o... Terms({'The': 1, 'Godfather': 1}, {'The'...
63
66
  ... ... ... ...
64
- 65513 They Came Back The lives of the residents of a small French t... PostingsRow({'Back': 1, 'They': 1, 'Came': 1},...
65
- 65515 The Eleventh Hour An ex-Navy SEAL, Michael Adams, (Matthew Reese... PostingsRow({'The': 1, 'Hour': 1, 'Eleventh': ...
66
- 65521 Pyaar Ka Punchnama Outspoken and overly critical Nishant Agarwal ... PostingsRow({'Ka': 1, 'Pyaar': 1, 'Punchnama':...
67
- 32767 Romero Romero is a compelling and deeply moving look ... PostingsRow({'Romero': 1}, {'Romero': [0]})
68
- 65534 Poison Paul Braconnier and his wife Blandine only hav... PostingsRow({'Poison': 1}, {'Poison': [0]})```
67
+ 65513 They Came Back The lives of the residents of a small French t... Terms({'Back': 1, 'They': 1, 'Came': 1},...
68
+ 65515 The Eleventh Hour An ex-Navy SEAL, Michael Adams, (Matthew Reese... Terms({'The': 1, 'Hour': 1, 'Eleventh': ...
69
+ 65521 Pyaar Ka Punchnama Outspoken and overly critical Nishant Agarwal ... Terms({'Ka': 1, 'Pyaar': 1, 'Punchnama':...
70
+ 32767 Romero Romero is a compelling and deeply moving look ... Terms({'Romero': 1}, {'Romero': [0]})
71
+ 65534 Poison Paul Braconnier and his wife Blandine only hav... Terms({'Poison': 1}, {'Poison': [0]})```
69
72
  ```
70
73
 
71
74
  Then search, getting top N with `Cat`
@@ -87,16 +90,16 @@ And since its just pandas, we can, of course just retrieve the top matches
87
90
  In[5]: df.iloc[top_n_cat[-10:]]
88
91
  Out[5]:
89
92
  title overview title_indexed
90
- 24106 The Black Cat American honeymooners in Hungary are trapped i... PostingsRow({'Black': 1, 'The': 1, 'Cat': 1}, ...
91
- 12593 Fritz the Cat A hypocritical swinging college student cat ra... PostingsRow({'Cat': 1, 'the': 1, 'Fritz': 1}, ...
92
- 39853 The Cat Concerto Tom enters from stage left in white tie and ta... PostingsRow({'The': 1, 'Cat': 1, 'Concerto': 1...
93
- 75491 The Rabbi's Cat Based on the best-selling graphic novel by Joa... PostingsRow({'The': 1, 'Cat': 1, "Rabbi's": 1}...
94
- 57353 Cat Run When a sexy, high-end escort holds the key evi... PostingsRow({'Cat': 1, 'Run': 1}, {'Cat': [0],...
95
- 25508 Cat People Sketch artist Irena Dubrovna (Simon) and Ameri... PostingsRow({'Cat': 1, 'People': 1}, {'Cat': [...
96
- 11694 Cat Ballou A woman seeking revenge for her murdered fathe... PostingsRow({'Cat': 1, 'Ballou': 1}, {'Cat': [...
97
- 25078 Cat Soup The surreal black comedy follows Nyatta, an an... PostingsRow({'Cat': 1, 'Soup': 1}, {'Cat': [0]...
98
- 35888 Cat Chaser A Miami hotel owner finds danger when be becom... PostingsRow({'Cat': 1, 'Chaser': 1}, {'Cat': [...
99
- 6217 Cat People After years of separation, Irina (Nastassja Ki... PostingsRow({'Cat': 1, 'People': 1}, {'Cat': [...
93
+ 24106 The Black Cat American honeymooners in Hungary are trapped i... Terms({'Black': 1, 'The': 1, 'Cat': 1}, ...
94
+ 12593 Fritz the Cat A hypocritical swinging college student cat ra... Terms({'Cat': 1, 'the': 1, 'Fritz': 1}, ...
95
+ 39853 The Cat Concerto Tom enters from stage left in white tie and ta... Terms({'The': 1, 'Cat': 1, 'Concerto': 1...
96
+ 75491 The Rabbi's Cat Based on the best-selling graphic novel by Joa... Terms({'The': 1, 'Cat': 1, "Rabbi's": 1}...
97
+ 57353 Cat Run When a sexy, high-end escort holds the key evi... Terms({'Cat': 1, 'Run': 1}, {'Cat': [0],...
98
+ 25508 Cat People Sketch artist Irena Dubrovna (Simon) and Ameri... Terms({'Cat': 1, 'People': 1}, {'Cat': [...
99
+ 11694 Cat Ballou A woman seeking revenge for her murdered fathe... Terms({'Cat': 1, 'Ballou': 1}, {'Cat': [...
100
+ 25078 Cat Soup The surreal black comedy follows Nyatta, an an... Terms({'Cat': 1, 'Soup': 1}, {'Cat': [0]...
101
+ 35888 Cat Chaser A Miami hotel owner finds danger when be becom... Terms({'Cat': 1, 'Chaser': 1}, {'Cat': [...
102
+ 6217 Cat People After years of separation, Irina (Nastassja Ki... Terms({'Cat': 1, 'People': 1}, {'Cat': [...
100
103
  ```
101
104
 
102
105
  More use cases can be seen [in the colab notebook](https://colab.research.google.com/drive/1w_Ajn5rHzcISKhdCuPhhVFav3zrvKWn1)
@@ -141,7 +144,7 @@ And you can pass any tokenizer that matches this signature to index:
141
144
  def ws_lowercase_tokenizer(string):
142
145
  return string.lower().split()
143
146
 
144
- df['title_indexed'] = PostingsArray.index(df['title'], tokenizer=ws_lowercase_tokenizer)
147
+ df['title_indexed'] = SearchArray.index(df['title'], tokenizer=ws_lowercase_tokenizer)
145
148
  ```
146
149
 
147
150
  Create your own using stemming libraries, or whatever Python functionality you want.
@@ -0,0 +1 @@
1
+ from searcharray.postings import SearchArray, Terms, TermsDtype # noqa: E401, F401
@@ -33,7 +33,7 @@ logger.addHandler(handler)
33
33
  logger.setLevel(logging.ERROR)
34
34
 
35
35
 
36
- class PostingsRow:
36
+ class Terms:
37
37
  """An indexed search doc - a row of postings, with bag of tokenized words and positions."""
38
38
 
39
39
  def __init__(self,
@@ -99,25 +99,25 @@ class PostingsRow:
99
99
 
100
100
  def __repr__(self):
101
101
  if self.encoded:
102
- rval = f"PostingsRow({repr(self.postings)}, encoded=True)"
102
+ rval = f"Terms({repr(self.postings)}, encoded=True)"
103
103
  else:
104
- rval = f"PostingsRow({repr(self.postings)})"
104
+ rval = f"Terms({repr(self.postings)})"
105
105
  return rval
106
106
 
107
107
  def __str__(self):
108
108
  return repr(self)
109
109
 
110
110
  def __eq__(self, other):
111
- # Flip to the other implementation if we're comparing to a PostingsArray
111
+ # Flip to the other implementation if we're comparing to a SearchArray
112
112
  # to get a boolean array back
113
- if isinstance(other, PostingsArray):
113
+ if isinstance(other, SearchArray):
114
114
  return other == self
115
- same_postings = isinstance(other, PostingsRow) and self.postings == other.postings
115
+ same_postings = isinstance(other, Terms) and self.postings == other.postings
116
116
  if same_postings and self.doc_len == other.doc_len:
117
117
  return True
118
118
 
119
119
  def __lt__(self, other):
120
- # return isinstance(other, PostingsRow) and hash(self) < hash(other)
120
+ # return isinstance(other, Terms) and hash(self) < hash(other)
121
121
  keys_both = set(self.postings.keys()).union(set(other.postings.keys()))
122
122
  # Sort lexically
123
123
  keys_both = sorted(keys_both)
@@ -154,10 +154,10 @@ class PostingsRow:
154
154
  return hash(json.dumps(self.postings, sort_keys=True))
155
155
 
156
156
 
157
- class PostingsDtype(ExtensionDtype):
157
+ class TermsDtype(ExtensionDtype):
158
158
 
159
159
  name = 'tokenized_text'
160
- type = PostingsRow
160
+ type = Terms
161
161
  kind = 'O'
162
162
 
163
163
  @classmethod
@@ -175,20 +175,20 @@ class PostingsDtype(ExtensionDtype):
175
175
 
176
176
  @classmethod
177
177
  def construct_array_type(cls):
178
- return PostingsArray
178
+ return SearchArray
179
179
 
180
180
  def __repr__(self):
181
- return 'PostingsDtype()'
181
+ return 'TermsDtype()'
182
182
 
183
183
  @property
184
184
  def na_value(self):
185
- return PostingsRow({})
185
+ return Terms({})
186
186
 
187
187
  def valid_value(self, value):
188
- return isinstance(value, dict) or pd.isna(value) or isinstance(value, PostingsRow)
188
+ return isinstance(value, dict) or pd.isna(value) or isinstance(value, Terms)
189
189
 
190
190
 
191
- register_extension_dtype(PostingsDtype)
191
+ register_extension_dtype(TermsDtype)
192
192
 
193
193
 
194
194
  def ws_tokenizer(string):
@@ -226,9 +226,9 @@ def _build_index_from_dict(postings):
226
226
  # https://www.austintripp.ca/blog/2018/09/12/sparse-matrices-tips1
227
227
  for doc_id, tokenized in enumerate(postings):
228
228
  if isinstance(tokenized, dict):
229
- tokenized = PostingsRow(tokenized, doc_len=len(tokenized))
230
- elif not isinstance(tokenized, PostingsRow):
231
- raise TypeError("Expected a PostingsRow or a dict")
229
+ tokenized = Terms(tokenized, doc_len=len(tokenized))
230
+ elif not isinstance(tokenized, Terms):
231
+ raise TypeError("Expected a Terms or a dict")
232
232
 
233
233
  if tokenized.encoded:
234
234
  posns = posns_enc
@@ -288,16 +288,16 @@ def _row_to_postings_row(doc_id, row, doc_len, term_dict, posns: PosnBitArray):
288
288
  enc_term_posns = posns.doc_encoded_posns(term_idx, doc_id=doc_id)
289
289
  labeled_posns[term] = enc_term_posns
290
290
 
291
- result = PostingsRow(tfs, posns=labeled_posns,
292
- doc_len=doc_len, encoded=True)
291
+ result = Terms(tfs, posns=labeled_posns,
292
+ doc_len=doc_len, encoded=True)
293
293
  # TODO add positions
294
294
  return result
295
295
 
296
296
 
297
- class PostingsArray(ExtensionArray):
298
- """An array of tokenized text (PostingsRows)."""
297
+ class SearchArray(ExtensionArray):
298
+ """An array of tokenized text (Termss)."""
299
299
 
300
- dtype = PostingsDtype()
300
+ dtype = TermsDtype()
301
301
 
302
302
  def __init__(self, postings, tokenizer=ws_tokenizer, avoid_copies=True):
303
303
  # Check dtype, raise TypeError
@@ -322,26 +322,26 @@ class PostingsArray(ExtensionArray):
322
322
  def tokenized_docs(docs):
323
323
  for doc_id, doc in enumerate(docs):
324
324
  if pd.isna(doc):
325
- yield PostingsRow({})
325
+ yield Terms({})
326
326
  else:
327
327
  token_stream = tokenizer(doc)
328
328
  term_freqs = Counter(token_stream)
329
329
  positions = defaultdict(list)
330
330
  for posn in range(len(token_stream)):
331
331
  positions[token_stream[posn]].append(posn)
332
- yield PostingsRow(term_freqs,
333
- doc_len=len(token_stream),
334
- posns=positions)
332
+ yield Terms(term_freqs,
333
+ doc_len=len(token_stream),
334
+ posns=positions)
335
335
 
336
336
  return cls(tokenized_docs(array), tokenizer)
337
337
 
338
338
  @classmethod
339
339
  def _from_sequence(cls, scalars, dtype=None, copy=False):
340
- """Construct a new PostingsArray from a sequence of scalars (PostingRow or convertible into)."""
340
+ """Construct a new SearchArray from a sequence of scalars (PostingRow or convertible into)."""
341
341
  if dtype is not None:
342
- if not isinstance(dtype, PostingsDtype):
342
+ if not isinstance(dtype, TermsDtype):
343
343
  return scalars
344
- if isinstance(scalars, np.ndarray) and scalars.dtype == PostingsDtype():
344
+ if isinstance(scalars, np.ndarray) and scalars.dtype == TermsDtype():
345
345
  return cls(scalars)
346
346
  # String types
347
347
  elif isinstance(scalars, np.ndarray) and scalars.dtype.kind in 'US':
@@ -377,7 +377,7 @@ class PostingsArray(ExtensionArray):
377
377
  # Construct a sliced view of this array
378
378
  sliced_tfs = self.term_mat.slice(key)
379
379
  sliced_posns = self.posns
380
- arr = PostingsArray([], tokenizer=self.tokenizer)
380
+ arr = SearchArray([], tokenizer=self.tokenizer)
381
381
  arr.term_mat = sliced_tfs
382
382
  arr.doc_lens = self.doc_lens[key]
383
383
  arr.posns = sliced_posns
@@ -392,13 +392,13 @@ class PostingsArray(ExtensionArray):
392
392
  value = value.values
393
393
  if isinstance(value, pd.DataFrame):
394
394
  value = value.values.flatten()
395
- if isinstance(value, PostingsArray):
395
+ if isinstance(value, SearchArray):
396
396
  value = value.to_numpy()
397
397
  if isinstance(value, list):
398
398
  value = np.asarray(value, dtype=object)
399
399
 
400
400
  if not isinstance(value, np.ndarray) and not self.dtype.valid_value(value):
401
- raise ValueError(f"Cannot set non-object array to PostingsArray -- you passed type:{type(value)} -- {value}")
401
+ raise ValueError(f"Cannot set non-object array to SearchArray -- you passed type:{type(value)} -- {value}")
402
402
 
403
403
  # Cant set a single value to an array
404
404
  if isinstance(key, numbers.Integral) and isinstance(value, np.ndarray):
@@ -412,7 +412,7 @@ class PostingsArray(ExtensionArray):
412
412
  if isinstance(value, float):
413
413
  term_mat = np.asarray([value])
414
414
  doc_lens = np.asarray([0])
415
- elif isinstance(value, PostingsRow):
415
+ elif isinstance(value, Terms):
416
416
  term_mat = np.asarray([value.tf_to_dense(self.term_dict)])
417
417
  doc_lens = np.asarray([value.doc_len])
418
418
  is_encoded = value.encoded
@@ -442,7 +442,7 @@ class PostingsArray(ExtensionArray):
442
442
  warnings.warn(msg)
443
443
 
444
444
  scan_value = value
445
- if isinstance(value, PostingsRow):
445
+ if isinstance(value, Terms):
446
446
  scan_value = np.asarray([value])
447
447
  for row in scan_value:
448
448
  for term in row.terms():
@@ -458,7 +458,7 @@ class PostingsArray(ExtensionArray):
458
458
  ):
459
459
  if dropna:
460
460
  counts = Counter(self[:])
461
- counts.pop(PostingsRow({}), None)
461
+ counts.pop(Terms({}), None)
462
462
  else:
463
463
  counts = Counter(self[:])
464
464
  return pd.Series(counts)
@@ -480,7 +480,7 @@ class PostingsArray(ExtensionArray):
480
480
  return NotImplemented
481
481
 
482
482
  # When other is an ExtensionArray
483
- if isinstance(other, PostingsArray):
483
+ if isinstance(other, SearchArray):
484
484
  if len(self) != len(other):
485
485
  return False
486
486
  elif len(other) == 0:
@@ -495,9 +495,9 @@ class PostingsArray(ExtensionArray):
495
495
  # return np.array(self[:]) == np.array(other[:])
496
496
 
497
497
  # When other is a scalar value
498
- elif isinstance(other, PostingsRow):
499
- other = PostingsArray([other], tokenizer=self.tokenizer)
500
- warnings.warn("Comparing a scalar value to a PostingsArray. This is slow.")
498
+ elif isinstance(other, Terms):
499
+ other = SearchArray([other], tokenizer=self.tokenizer)
500
+ warnings.warn("Comparing a scalar value to a SearchArray. This is slow.")
501
501
  return np.array(self[:]) == np.array(other[:])
502
502
 
503
503
  # When other is a sequence but not an ExtensionArray
@@ -508,7 +508,7 @@ class PostingsArray(ExtensionArray):
508
508
  elif len(other) == 0:
509
509
  return np.array([], dtype=bool)
510
510
  # We actually don't know how it was tokenized
511
- other = PostingsArray(other, tokenizer=self.tokenizer)
511
+ other = SearchArray(other, tokenizer=self.tokenizer)
512
512
  return np.array(self[:]) == np.array(other[:])
513
513
 
514
514
  # Return False where 'other' is neither the same length nor a scalar
@@ -528,14 +528,14 @@ class PostingsArray(ExtensionArray):
528
528
 
529
529
  if allow_fill and -1 in result_indices:
530
530
  if fill_value is None or pd.isna(fill_value):
531
- fill_value = PostingsRow({}, encoded=True)
531
+ fill_value = Terms({}, encoded=True)
532
532
 
533
533
  to_fill_mask = result_indices == -1
534
534
  # This is slow as it rebuilds all the term dictionaries
535
535
  # on the subsequent assignment lines
536
536
  # However, this case tends to be the exception for
537
537
  # most dataframe operations
538
- taken = PostingsArray([fill_value] * len(result_indices))
538
+ taken = SearchArray([fill_value] * len(result_indices))
539
539
  taken[~to_fill_mask] = self[result_indices[~to_fill_mask]].copy()
540
540
 
541
541
  return taken
@@ -544,7 +544,7 @@ class PostingsArray(ExtensionArray):
544
544
  return taken
545
545
 
546
546
  def copy(self):
547
- postings_arr = PostingsArray([], tokenizer=self.tokenizer)
547
+ postings_arr = SearchArray([], tokenizer=self.tokenizer)
548
548
  postings_arr.doc_lens = self.doc_lens.copy()
549
549
  postings_arr.term_mat = self.term_mat.copy()
550
550
  postings_arr.posns = self.posns
@@ -559,7 +559,7 @@ class PostingsArray(ExtensionArray):
559
559
  @classmethod
560
560
  def _concat_same_type(cls, to_concat):
561
561
  concatenated_data = np.concatenate([ea[:] for ea in to_concat])
562
- return PostingsArray(concatenated_data, tokenizer=to_concat[0].tokenizer)
562
+ return SearchArray(concatenated_data, tokenizer=to_concat[0].tokenizer)
563
563
 
564
564
  @classmethod
565
565
  def _from_factorized(cls, values, original):
@@ -568,7 +568,7 @@ class PostingsArray(ExtensionArray):
568
568
  def _values_for_factorize(self):
569
569
  """Return an array and missing value suitable for factorization (ie grouping)."""
570
570
  arr = np.asarray(self[:], dtype=object)
571
- return arr, PostingsRow({})
571
+ return arr, Terms({})
572
572
 
573
573
  def _check_token_arg(self, token):
574
574
  if isinstance(token, str):
@@ -3,7 +3,7 @@ import re
3
3
  import pandas as pd
4
4
  import numpy as np
5
5
  from typing import List, Optional, Dict, Tuple
6
- from searcharray.postings import PostingsArray
6
+ from searcharray.postings import SearchArray
7
7
  from searcharray.similarity import Similarity, default_bm25
8
8
 
9
9
 
@@ -75,10 +75,10 @@ def parse_field_boosts(field_lists: List[str]) -> dict:
75
75
  return out
76
76
 
77
77
 
78
- def get_field(frame, field) -> PostingsArray:
78
+ def get_field(frame, field) -> SearchArray:
79
79
  if field not in frame.columns:
80
80
  raise ValueError(f"Field {field} not in dataframe")
81
- if not isinstance(frame[field].array, PostingsArray):
81
+ if not isinstance(frame[field].array, SearchArray):
82
82
  raise ValueError(f"Field {field} is not a searcharray field")
83
83
  return frame[field].array
84
84
 
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: searcharray
3
- Version: 0.0.30
3
+ Version: 0.0.31
4
4
  Summary: Searchable pandas text extension arrays for prototyping search
5
5
  Home-page: https://github.com/softwaredoug/searcharray
6
6
  Author: Doug Turnbull
@@ -36,8 +36,11 @@ SearchArray turns Pandas string columns into a term index. It alows efficient BM
36
36
  Think Lucene, but as a Pandas column.
37
37
 
38
38
  ```python
39
- In[3]: df['title_indexed'] = PostingsArray.index(df['title'])
39
+ In[2]: from searcharray import SearchArray
40
+
41
+ In[3]: df['title_indexed'] = SearchArray.index(df['title'])
40
42
  np.sort(df['title_indexed'].array.score('Cat'))
43
+
41
44
  Out[3]: array([ 0. , 0. , 0. , ..., 15.84568033,
42
45
  15.84568033, 15.84568033])
43
46
  ```
@@ -79,22 +82,22 @@ Out[1]:
79
82
  Index the text:
80
83
 
81
84
  ```
82
- In[2]: df['title_indexed'] = PostingsArray.index(df['title'])
85
+ In[2]: df['title_indexed'] = SearchArray.index(df['title'])
83
86
  df
84
87
 
85
88
  Out[2]:
86
89
  title overview title_indexed
87
- 374430 Black Mirror: White Christmas This feature-length special consists of three ... PostingsRow({'Black': 1, 'Mirror:': 1, 'White'...
88
- 19404 The Brave-Hearted Will Take the Bride Raj is a rich, carefree, happy-go-lucky second... PostingsRow({'The': 1, 'Brave-Hearted': 1, 'Wi...
89
- 278 The Shawshank Redemption Framed in the 1940s for the double murder of h... PostingsRow({'The': 1, 'Shawshank': 1, 'Redemp...
90
- 372058 Your Name. High schoolers Mitsuha and Taki are complete s... PostingsRow({'Your': 1, 'Name.': 1}, {'Your': ...
91
- 238 The Godfather Spanning the years 1945 to 1955, a chronicle o... PostingsRow({'The': 1, 'Godfather': 1}, {'The'...
90
+ 374430 Black Mirror: White Christmas This feature-length special consists of three ... Terms({'Black': 1, 'Mirror:': 1, 'White'...
91
+ 19404 The Brave-Hearted Will Take the Bride Raj is a rich, carefree, happy-go-lucky second... Terms({'The': 1, 'Brave-Hearted': 1, 'Wi...
92
+ 278 The Shawshank Redemption Framed in the 1940s for the double murder of h... Terms({'The': 1, 'Shawshank': 1, 'Redemp...
93
+ 372058 Your Name. High schoolers Mitsuha and Taki are complete s... Terms({'Your': 1, 'Name.': 1}, {'Your': ...
94
+ 238 The Godfather Spanning the years 1945 to 1955, a chronicle o... Terms({'The': 1, 'Godfather': 1}, {'The'...
92
95
  ... ... ... ...
93
- 65513 They Came Back The lives of the residents of a small French t... PostingsRow({'Back': 1, 'They': 1, 'Came': 1},...
94
- 65515 The Eleventh Hour An ex-Navy SEAL, Michael Adams, (Matthew Reese... PostingsRow({'The': 1, 'Hour': 1, 'Eleventh': ...
95
- 65521 Pyaar Ka Punchnama Outspoken and overly critical Nishant Agarwal ... PostingsRow({'Ka': 1, 'Pyaar': 1, 'Punchnama':...
96
- 32767 Romero Romero is a compelling and deeply moving look ... PostingsRow({'Romero': 1}, {'Romero': [0]})
97
- 65534 Poison Paul Braconnier and his wife Blandine only hav... PostingsRow({'Poison': 1}, {'Poison': [0]})```
96
+ 65513 They Came Back The lives of the residents of a small French t... Terms({'Back': 1, 'They': 1, 'Came': 1},...
97
+ 65515 The Eleventh Hour An ex-Navy SEAL, Michael Adams, (Matthew Reese... Terms({'The': 1, 'Hour': 1, 'Eleventh': ...
98
+ 65521 Pyaar Ka Punchnama Outspoken and overly critical Nishant Agarwal ... Terms({'Ka': 1, 'Pyaar': 1, 'Punchnama':...
99
+ 32767 Romero Romero is a compelling and deeply moving look ... Terms({'Romero': 1}, {'Romero': [0]})
100
+ 65534 Poison Paul Braconnier and his wife Blandine only hav... Terms({'Poison': 1}, {'Poison': [0]})```
98
101
  ```
99
102
 
100
103
  Then search, getting top N with `Cat`
@@ -116,16 +119,16 @@ And since its just pandas, we can, of course just retrieve the top matches
116
119
  In[5]: df.iloc[top_n_cat[-10:]]
117
120
  Out[5]:
118
121
  title overview title_indexed
119
- 24106 The Black Cat American honeymooners in Hungary are trapped i... PostingsRow({'Black': 1, 'The': 1, 'Cat': 1}, ...
120
- 12593 Fritz the Cat A hypocritical swinging college student cat ra... PostingsRow({'Cat': 1, 'the': 1, 'Fritz': 1}, ...
121
- 39853 The Cat Concerto Tom enters from stage left in white tie and ta... PostingsRow({'The': 1, 'Cat': 1, 'Concerto': 1...
122
- 75491 The Rabbi's Cat Based on the best-selling graphic novel by Joa... PostingsRow({'The': 1, 'Cat': 1, "Rabbi's": 1}...
123
- 57353 Cat Run When a sexy, high-end escort holds the key evi... PostingsRow({'Cat': 1, 'Run': 1}, {'Cat': [0],...
124
- 25508 Cat People Sketch artist Irena Dubrovna (Simon) and Ameri... PostingsRow({'Cat': 1, 'People': 1}, {'Cat': [...
125
- 11694 Cat Ballou A woman seeking revenge for her murdered fathe... PostingsRow({'Cat': 1, 'Ballou': 1}, {'Cat': [...
126
- 25078 Cat Soup The surreal black comedy follows Nyatta, an an... PostingsRow({'Cat': 1, 'Soup': 1}, {'Cat': [0]...
127
- 35888 Cat Chaser A Miami hotel owner finds danger when be becom... PostingsRow({'Cat': 1, 'Chaser': 1}, {'Cat': [...
128
- 6217 Cat People After years of separation, Irina (Nastassja Ki... PostingsRow({'Cat': 1, 'People': 1}, {'Cat': [...
122
+ 24106 The Black Cat American honeymooners in Hungary are trapped i... Terms({'Black': 1, 'The': 1, 'Cat': 1}, ...
123
+ 12593 Fritz the Cat A hypocritical swinging college student cat ra... Terms({'Cat': 1, 'the': 1, 'Fritz': 1}, ...
124
+ 39853 The Cat Concerto Tom enters from stage left in white tie and ta... Terms({'The': 1, 'Cat': 1, 'Concerto': 1...
125
+ 75491 The Rabbi's Cat Based on the best-selling graphic novel by Joa... Terms({'The': 1, 'Cat': 1, "Rabbi's": 1}...
126
+ 57353 Cat Run When a sexy, high-end escort holds the key evi... Terms({'Cat': 1, 'Run': 1}, {'Cat': [0],...
127
+ 25508 Cat People Sketch artist Irena Dubrovna (Simon) and Ameri... Terms({'Cat': 1, 'People': 1}, {'Cat': [...
128
+ 11694 Cat Ballou A woman seeking revenge for her murdered fathe... Terms({'Cat': 1, 'Ballou': 1}, {'Cat': [...
129
+ 25078 Cat Soup The surreal black comedy follows Nyatta, an an... Terms({'Cat': 1, 'Soup': 1}, {'Cat': [0]...
130
+ 35888 Cat Chaser A Miami hotel owner finds danger when be becom... Terms({'Cat': 1, 'Chaser': 1}, {'Cat': [...
131
+ 6217 Cat People After years of separation, Irina (Nastassja Ki... Terms({'Cat': 1, 'People': 1}, {'Cat': [...
129
132
  ```
130
133
 
131
134
  More use cases can be seen [in the colab notebook](https://colab.research.google.com/drive/1w_Ajn5rHzcISKhdCuPhhVFav3zrvKWn1)
@@ -170,7 +173,7 @@ And you can pass any tokenizer that matches this signature to index:
170
173
  def ws_lowercase_tokenizer(string):
171
174
  return string.lower().split()
172
175
 
173
- df['title_indexed'] = PostingsArray.index(df['title'], tokenizer=ws_lowercase_tokenizer)
176
+ df['title_indexed'] = SearchArray.index(df['title'], tokenizer=ws_lowercase_tokenizer)
174
177
  ```
175
178
 
176
179
  Create your own using stemming libraries, or whatever Python functionality you want.
@@ -36,7 +36,7 @@ setup(
36
36
  # For a discussion on single-sourcing the version across setup.py and the
37
37
  # project code, see
38
38
  # https://packaging.python.org/guides/single-sourcing-package-version/
39
- version="0.0.30", # Required
39
+ version="0.0.31", # Required
40
40
  # This is a one-line description or tagline of what your project does. This
41
41
  # corresponds to the "Summary" metadata field:
42
42
  # https://packaging.python.org/specifications/core-metadata/#summary