searcharray 0.0.30__tar.gz → 0.0.31__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {searcharray-0.0.30 → searcharray-0.0.31}/PKG-INFO +27 -24
- {searcharray-0.0.30 → searcharray-0.0.31}/README.md +26 -23
- searcharray-0.0.31/searcharray/__init__.py +1 -0
- {searcharray-0.0.30 → searcharray-0.0.31}/searcharray/postings.py +45 -45
- {searcharray-0.0.30 → searcharray-0.0.31}/searcharray/solr.py +3 -3
- {searcharray-0.0.30 → searcharray-0.0.31}/searcharray.egg-info/PKG-INFO +27 -24
- {searcharray-0.0.30 → searcharray-0.0.31}/setup.py +1 -1
- {searcharray-0.0.30 → searcharray-0.0.31}/test/test_extension_array.py +13 -12
- {searcharray-0.0.30 → searcharray-0.0.31}/test/test_msmarco.py +5 -5
- {searcharray-0.0.30 → searcharray-0.0.31}/test/test_phrase_matches.py +59 -59
- {searcharray-0.0.30 → searcharray-0.0.31}/test/test_search.py +11 -11
- {searcharray-0.0.30 → searcharray-0.0.31}/test/test_solr.py +19 -19
- {searcharray-0.0.30 → searcharray-0.0.31}/test/test_tmdb.py +7 -7
- searcharray-0.0.30/searcharray/utils/__init__.py +0 -0
- {searcharray-0.0.30 → searcharray-0.0.31}/pyproject.toml +0 -0
- {searcharray-0.0.30/searcharray → searcharray-0.0.31/searcharray/phrase}/__init__.py +0 -0
- {searcharray-0.0.30 → searcharray-0.0.31}/searcharray/phrase/middle_out.py +0 -0
- {searcharray-0.0.30 → searcharray-0.0.31}/searcharray/phrase/posn_diffs.py +0 -0
- {searcharray-0.0.30 → searcharray-0.0.31}/searcharray/phrase/scan_merge.py +0 -0
- {searcharray-0.0.30 → searcharray-0.0.31}/searcharray/similarity.py +0 -0
- {searcharray-0.0.30 → searcharray-0.0.31}/searcharray/term_dict.py +0 -0
- {searcharray-0.0.30/searcharray/phrase → searcharray-0.0.31/searcharray/utils}/__init__.py +0 -0
- {searcharray-0.0.30 → searcharray-0.0.31}/searcharray/utils/bitcount.py +0 -0
- {searcharray-0.0.30 → searcharray-0.0.31}/searcharray/utils/cache.py +0 -0
- {searcharray-0.0.30 → searcharray-0.0.31}/searcharray/utils/mat_set.py +0 -0
- {searcharray-0.0.30 → searcharray-0.0.31}/searcharray/utils/roaringish.py +0 -0
- {searcharray-0.0.30 → searcharray-0.0.31}/searcharray/utils/row_viewable_matrix.py +0 -0
- {searcharray-0.0.30 → searcharray-0.0.31}/searcharray.egg-info/SOURCES.txt +0 -0
- {searcharray-0.0.30 → searcharray-0.0.31}/searcharray.egg-info/dependency_links.txt +0 -0
- {searcharray-0.0.30 → searcharray-0.0.31}/searcharray.egg-info/entry_points.txt +0 -0
- {searcharray-0.0.30 → searcharray-0.0.31}/searcharray.egg-info/requires.txt +0 -0
- {searcharray-0.0.30 → searcharray-0.0.31}/searcharray.egg-info/top_level.txt +0 -0
- {searcharray-0.0.30 → searcharray-0.0.31}/setup.cfg +0 -0
- {searcharray-0.0.30 → searcharray-0.0.31}/test/test_utils.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: searcharray
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.31
|
|
4
4
|
Summary: Searchable pandas text extension arrays for prototyping search
|
|
5
5
|
Home-page: https://github.com/softwaredoug/searcharray
|
|
6
6
|
Author: Doug Turnbull
|
|
@@ -36,8 +36,11 @@ SearchArray turns Pandas string columns into a term index. It alows efficient BM
|
|
|
36
36
|
Think Lucene, but as a Pandas column.
|
|
37
37
|
|
|
38
38
|
```python
|
|
39
|
-
In[
|
|
39
|
+
In[2]: from searcharray import SearchArray
|
|
40
|
+
|
|
41
|
+
In[3]: df['title_indexed'] = SearchArray.index(df['title'])
|
|
40
42
|
np.sort(df['title_indexed'].array.score('Cat'))
|
|
43
|
+
|
|
41
44
|
Out[3]: array([ 0. , 0. , 0. , ..., 15.84568033,
|
|
42
45
|
15.84568033, 15.84568033])
|
|
43
46
|
```
|
|
@@ -79,22 +82,22 @@ Out[1]:
|
|
|
79
82
|
Index the text:
|
|
80
83
|
|
|
81
84
|
```
|
|
82
|
-
In[2]: df['title_indexed'] =
|
|
85
|
+
In[2]: df['title_indexed'] = SearchArray.index(df['title'])
|
|
83
86
|
df
|
|
84
87
|
|
|
85
88
|
Out[2]:
|
|
86
89
|
title overview title_indexed
|
|
87
|
-
374430 Black Mirror: White Christmas This feature-length special consists of three ...
|
|
88
|
-
19404 The Brave-Hearted Will Take the Bride Raj is a rich, carefree, happy-go-lucky second...
|
|
89
|
-
278 The Shawshank Redemption Framed in the 1940s for the double murder of h...
|
|
90
|
-
372058 Your Name. High schoolers Mitsuha and Taki are complete s...
|
|
91
|
-
238 The Godfather Spanning the years 1945 to 1955, a chronicle o...
|
|
90
|
+
374430 Black Mirror: White Christmas This feature-length special consists of three ... Terms({'Black': 1, 'Mirror:': 1, 'White'...
|
|
91
|
+
19404 The Brave-Hearted Will Take the Bride Raj is a rich, carefree, happy-go-lucky second... Terms({'The': 1, 'Brave-Hearted': 1, 'Wi...
|
|
92
|
+
278 The Shawshank Redemption Framed in the 1940s for the double murder of h... Terms({'The': 1, 'Shawshank': 1, 'Redemp...
|
|
93
|
+
372058 Your Name. High schoolers Mitsuha and Taki are complete s... Terms({'Your': 1, 'Name.': 1}, {'Your': ...
|
|
94
|
+
238 The Godfather Spanning the years 1945 to 1955, a chronicle o... Terms({'The': 1, 'Godfather': 1}, {'The'...
|
|
92
95
|
... ... ... ...
|
|
93
|
-
65513 They Came Back The lives of the residents of a small French t...
|
|
94
|
-
65515 The Eleventh Hour An ex-Navy SEAL, Michael Adams, (Matthew Reese...
|
|
95
|
-
65521 Pyaar Ka Punchnama Outspoken and overly critical Nishant Agarwal ...
|
|
96
|
-
32767 Romero Romero is a compelling and deeply moving look ...
|
|
97
|
-
65534 Poison Paul Braconnier and his wife Blandine only hav...
|
|
96
|
+
65513 They Came Back The lives of the residents of a small French t... Terms({'Back': 1, 'They': 1, 'Came': 1},...
|
|
97
|
+
65515 The Eleventh Hour An ex-Navy SEAL, Michael Adams, (Matthew Reese... Terms({'The': 1, 'Hour': 1, 'Eleventh': ...
|
|
98
|
+
65521 Pyaar Ka Punchnama Outspoken and overly critical Nishant Agarwal ... Terms({'Ka': 1, 'Pyaar': 1, 'Punchnama':...
|
|
99
|
+
32767 Romero Romero is a compelling and deeply moving look ... Terms({'Romero': 1}, {'Romero': [0]})
|
|
100
|
+
65534 Poison Paul Braconnier and his wife Blandine only hav... Terms({'Poison': 1}, {'Poison': [0]})```
|
|
98
101
|
```
|
|
99
102
|
|
|
100
103
|
Then search, getting top N with `Cat`
|
|
@@ -116,16 +119,16 @@ And since its just pandas, we can, of course just retrieve the top matches
|
|
|
116
119
|
In[5]: df.iloc[top_n_cat[-10:]]
|
|
117
120
|
Out[5]:
|
|
118
121
|
title overview title_indexed
|
|
119
|
-
24106 The Black Cat American honeymooners in Hungary are trapped i...
|
|
120
|
-
12593 Fritz the Cat A hypocritical swinging college student cat ra...
|
|
121
|
-
39853 The Cat Concerto Tom enters from stage left in white tie and ta...
|
|
122
|
-
75491 The Rabbi's Cat Based on the best-selling graphic novel by Joa...
|
|
123
|
-
57353 Cat Run When a sexy, high-end escort holds the key evi...
|
|
124
|
-
25508 Cat People Sketch artist Irena Dubrovna (Simon) and Ameri...
|
|
125
|
-
11694 Cat Ballou A woman seeking revenge for her murdered fathe...
|
|
126
|
-
25078 Cat Soup The surreal black comedy follows Nyatta, an an...
|
|
127
|
-
35888 Cat Chaser A Miami hotel owner finds danger when be becom...
|
|
128
|
-
6217 Cat People After years of separation, Irina (Nastassja Ki...
|
|
122
|
+
24106 The Black Cat American honeymooners in Hungary are trapped i... Terms({'Black': 1, 'The': 1, 'Cat': 1}, ...
|
|
123
|
+
12593 Fritz the Cat A hypocritical swinging college student cat ra... Terms({'Cat': 1, 'the': 1, 'Fritz': 1}, ...
|
|
124
|
+
39853 The Cat Concerto Tom enters from stage left in white tie and ta... Terms({'The': 1, 'Cat': 1, 'Concerto': 1...
|
|
125
|
+
75491 The Rabbi's Cat Based on the best-selling graphic novel by Joa... Terms({'The': 1, 'Cat': 1, "Rabbi's": 1}...
|
|
126
|
+
57353 Cat Run When a sexy, high-end escort holds the key evi... Terms({'Cat': 1, 'Run': 1}, {'Cat': [0],...
|
|
127
|
+
25508 Cat People Sketch artist Irena Dubrovna (Simon) and Ameri... Terms({'Cat': 1, 'People': 1}, {'Cat': [...
|
|
128
|
+
11694 Cat Ballou A woman seeking revenge for her murdered fathe... Terms({'Cat': 1, 'Ballou': 1}, {'Cat': [...
|
|
129
|
+
25078 Cat Soup The surreal black comedy follows Nyatta, an an... Terms({'Cat': 1, 'Soup': 1}, {'Cat': [0]...
|
|
130
|
+
35888 Cat Chaser A Miami hotel owner finds danger when be becom... Terms({'Cat': 1, 'Chaser': 1}, {'Cat': [...
|
|
131
|
+
6217 Cat People After years of separation, Irina (Nastassja Ki... Terms({'Cat': 1, 'People': 1}, {'Cat': [...
|
|
129
132
|
```
|
|
130
133
|
|
|
131
134
|
More use cases can be seen [in the colab notebook](https://colab.research.google.com/drive/1w_Ajn5rHzcISKhdCuPhhVFav3zrvKWn1)
|
|
@@ -170,7 +173,7 @@ And you can pass any tokenizer that matches this signature to index:
|
|
|
170
173
|
def ws_lowercase_tokenizer(string):
|
|
171
174
|
return string.lower().split()
|
|
172
175
|
|
|
173
|
-
df['title_indexed'] =
|
|
176
|
+
df['title_indexed'] = SearchArray.index(df['title'], tokenizer=ws_lowercase_tokenizer)
|
|
174
177
|
```
|
|
175
178
|
|
|
176
179
|
Create your own using stemming libraries, or whatever Python functionality you want.
|
|
@@ -7,8 +7,11 @@ SearchArray turns Pandas string columns into a term index. It alows efficient BM
|
|
|
7
7
|
Think Lucene, but as a Pandas column.
|
|
8
8
|
|
|
9
9
|
```python
|
|
10
|
-
In[
|
|
10
|
+
In[2]: from searcharray import SearchArray
|
|
11
|
+
|
|
12
|
+
In[3]: df['title_indexed'] = SearchArray.index(df['title'])
|
|
11
13
|
np.sort(df['title_indexed'].array.score('Cat'))
|
|
14
|
+
|
|
12
15
|
Out[3]: array([ 0. , 0. , 0. , ..., 15.84568033,
|
|
13
16
|
15.84568033, 15.84568033])
|
|
14
17
|
```
|
|
@@ -50,22 +53,22 @@ Out[1]:
|
|
|
50
53
|
Index the text:
|
|
51
54
|
|
|
52
55
|
```
|
|
53
|
-
In[2]: df['title_indexed'] =
|
|
56
|
+
In[2]: df['title_indexed'] = SearchArray.index(df['title'])
|
|
54
57
|
df
|
|
55
58
|
|
|
56
59
|
Out[2]:
|
|
57
60
|
title overview title_indexed
|
|
58
|
-
374430 Black Mirror: White Christmas This feature-length special consists of three ...
|
|
59
|
-
19404 The Brave-Hearted Will Take the Bride Raj is a rich, carefree, happy-go-lucky second...
|
|
60
|
-
278 The Shawshank Redemption Framed in the 1940s for the double murder of h...
|
|
61
|
-
372058 Your Name. High schoolers Mitsuha and Taki are complete s...
|
|
62
|
-
238 The Godfather Spanning the years 1945 to 1955, a chronicle o...
|
|
61
|
+
374430 Black Mirror: White Christmas This feature-length special consists of three ... Terms({'Black': 1, 'Mirror:': 1, 'White'...
|
|
62
|
+
19404 The Brave-Hearted Will Take the Bride Raj is a rich, carefree, happy-go-lucky second... Terms({'The': 1, 'Brave-Hearted': 1, 'Wi...
|
|
63
|
+
278 The Shawshank Redemption Framed in the 1940s for the double murder of h... Terms({'The': 1, 'Shawshank': 1, 'Redemp...
|
|
64
|
+
372058 Your Name. High schoolers Mitsuha and Taki are complete s... Terms({'Your': 1, 'Name.': 1}, {'Your': ...
|
|
65
|
+
238 The Godfather Spanning the years 1945 to 1955, a chronicle o... Terms({'The': 1, 'Godfather': 1}, {'The'...
|
|
63
66
|
... ... ... ...
|
|
64
|
-
65513 They Came Back The lives of the residents of a small French t...
|
|
65
|
-
65515 The Eleventh Hour An ex-Navy SEAL, Michael Adams, (Matthew Reese...
|
|
66
|
-
65521 Pyaar Ka Punchnama Outspoken and overly critical Nishant Agarwal ...
|
|
67
|
-
32767 Romero Romero is a compelling and deeply moving look ...
|
|
68
|
-
65534 Poison Paul Braconnier and his wife Blandine only hav...
|
|
67
|
+
65513 They Came Back The lives of the residents of a small French t... Terms({'Back': 1, 'They': 1, 'Came': 1},...
|
|
68
|
+
65515 The Eleventh Hour An ex-Navy SEAL, Michael Adams, (Matthew Reese... Terms({'The': 1, 'Hour': 1, 'Eleventh': ...
|
|
69
|
+
65521 Pyaar Ka Punchnama Outspoken and overly critical Nishant Agarwal ... Terms({'Ka': 1, 'Pyaar': 1, 'Punchnama':...
|
|
70
|
+
32767 Romero Romero is a compelling and deeply moving look ... Terms({'Romero': 1}, {'Romero': [0]})
|
|
71
|
+
65534 Poison Paul Braconnier and his wife Blandine only hav... Terms({'Poison': 1}, {'Poison': [0]})```
|
|
69
72
|
```
|
|
70
73
|
|
|
71
74
|
Then search, getting top N with `Cat`
|
|
@@ -87,16 +90,16 @@ And since its just pandas, we can, of course just retrieve the top matches
|
|
|
87
90
|
In[5]: df.iloc[top_n_cat[-10:]]
|
|
88
91
|
Out[5]:
|
|
89
92
|
title overview title_indexed
|
|
90
|
-
24106 The Black Cat American honeymooners in Hungary are trapped i...
|
|
91
|
-
12593 Fritz the Cat A hypocritical swinging college student cat ra...
|
|
92
|
-
39853 The Cat Concerto Tom enters from stage left in white tie and ta...
|
|
93
|
-
75491 The Rabbi's Cat Based on the best-selling graphic novel by Joa...
|
|
94
|
-
57353 Cat Run When a sexy, high-end escort holds the key evi...
|
|
95
|
-
25508 Cat People Sketch artist Irena Dubrovna (Simon) and Ameri...
|
|
96
|
-
11694 Cat Ballou A woman seeking revenge for her murdered fathe...
|
|
97
|
-
25078 Cat Soup The surreal black comedy follows Nyatta, an an...
|
|
98
|
-
35888 Cat Chaser A Miami hotel owner finds danger when be becom...
|
|
99
|
-
6217 Cat People After years of separation, Irina (Nastassja Ki...
|
|
93
|
+
24106 The Black Cat American honeymooners in Hungary are trapped i... Terms({'Black': 1, 'The': 1, 'Cat': 1}, ...
|
|
94
|
+
12593 Fritz the Cat A hypocritical swinging college student cat ra... Terms({'Cat': 1, 'the': 1, 'Fritz': 1}, ...
|
|
95
|
+
39853 The Cat Concerto Tom enters from stage left in white tie and ta... Terms({'The': 1, 'Cat': 1, 'Concerto': 1...
|
|
96
|
+
75491 The Rabbi's Cat Based on the best-selling graphic novel by Joa... Terms({'The': 1, 'Cat': 1, "Rabbi's": 1}...
|
|
97
|
+
57353 Cat Run When a sexy, high-end escort holds the key evi... Terms({'Cat': 1, 'Run': 1}, {'Cat': [0],...
|
|
98
|
+
25508 Cat People Sketch artist Irena Dubrovna (Simon) and Ameri... Terms({'Cat': 1, 'People': 1}, {'Cat': [...
|
|
99
|
+
11694 Cat Ballou A woman seeking revenge for her murdered fathe... Terms({'Cat': 1, 'Ballou': 1}, {'Cat': [...
|
|
100
|
+
25078 Cat Soup The surreal black comedy follows Nyatta, an an... Terms({'Cat': 1, 'Soup': 1}, {'Cat': [0]...
|
|
101
|
+
35888 Cat Chaser A Miami hotel owner finds danger when be becom... Terms({'Cat': 1, 'Chaser': 1}, {'Cat': [...
|
|
102
|
+
6217 Cat People After years of separation, Irina (Nastassja Ki... Terms({'Cat': 1, 'People': 1}, {'Cat': [...
|
|
100
103
|
```
|
|
101
104
|
|
|
102
105
|
More use cases can be seen [in the colab notebook](https://colab.research.google.com/drive/1w_Ajn5rHzcISKhdCuPhhVFav3zrvKWn1)
|
|
@@ -141,7 +144,7 @@ And you can pass any tokenizer that matches this signature to index:
|
|
|
141
144
|
def ws_lowercase_tokenizer(string):
|
|
142
145
|
return string.lower().split()
|
|
143
146
|
|
|
144
|
-
df['title_indexed'] =
|
|
147
|
+
df['title_indexed'] = SearchArray.index(df['title'], tokenizer=ws_lowercase_tokenizer)
|
|
145
148
|
```
|
|
146
149
|
|
|
147
150
|
Create your own using stemming libraries, or whatever Python functionality you want.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
from searcharray.postings import SearchArray, Terms, TermsDtype # noqa: E401, F401
|
|
@@ -33,7 +33,7 @@ logger.addHandler(handler)
|
|
|
33
33
|
logger.setLevel(logging.ERROR)
|
|
34
34
|
|
|
35
35
|
|
|
36
|
-
class
|
|
36
|
+
class Terms:
|
|
37
37
|
"""An indexed search doc - a row of postings, with bag of tokenized words and positions."""
|
|
38
38
|
|
|
39
39
|
def __init__(self,
|
|
@@ -99,25 +99,25 @@ class PostingsRow:
|
|
|
99
99
|
|
|
100
100
|
def __repr__(self):
|
|
101
101
|
if self.encoded:
|
|
102
|
-
rval = f"
|
|
102
|
+
rval = f"Terms({repr(self.postings)}, encoded=True)"
|
|
103
103
|
else:
|
|
104
|
-
rval = f"
|
|
104
|
+
rval = f"Terms({repr(self.postings)})"
|
|
105
105
|
return rval
|
|
106
106
|
|
|
107
107
|
def __str__(self):
|
|
108
108
|
return repr(self)
|
|
109
109
|
|
|
110
110
|
def __eq__(self, other):
|
|
111
|
-
# Flip to the other implementation if we're comparing to a
|
|
111
|
+
# Flip to the other implementation if we're comparing to a SearchArray
|
|
112
112
|
# to get a boolean array back
|
|
113
|
-
if isinstance(other,
|
|
113
|
+
if isinstance(other, SearchArray):
|
|
114
114
|
return other == self
|
|
115
|
-
same_postings = isinstance(other,
|
|
115
|
+
same_postings = isinstance(other, Terms) and self.postings == other.postings
|
|
116
116
|
if same_postings and self.doc_len == other.doc_len:
|
|
117
117
|
return True
|
|
118
118
|
|
|
119
119
|
def __lt__(self, other):
|
|
120
|
-
# return isinstance(other,
|
|
120
|
+
# return isinstance(other, Terms) and hash(self) < hash(other)
|
|
121
121
|
keys_both = set(self.postings.keys()).union(set(other.postings.keys()))
|
|
122
122
|
# Sort lexically
|
|
123
123
|
keys_both = sorted(keys_both)
|
|
@@ -154,10 +154,10 @@ class PostingsRow:
|
|
|
154
154
|
return hash(json.dumps(self.postings, sort_keys=True))
|
|
155
155
|
|
|
156
156
|
|
|
157
|
-
class
|
|
157
|
+
class TermsDtype(ExtensionDtype):
|
|
158
158
|
|
|
159
159
|
name = 'tokenized_text'
|
|
160
|
-
type =
|
|
160
|
+
type = Terms
|
|
161
161
|
kind = 'O'
|
|
162
162
|
|
|
163
163
|
@classmethod
|
|
@@ -175,20 +175,20 @@ class PostingsDtype(ExtensionDtype):
|
|
|
175
175
|
|
|
176
176
|
@classmethod
|
|
177
177
|
def construct_array_type(cls):
|
|
178
|
-
return
|
|
178
|
+
return SearchArray
|
|
179
179
|
|
|
180
180
|
def __repr__(self):
|
|
181
|
-
return '
|
|
181
|
+
return 'TermsDtype()'
|
|
182
182
|
|
|
183
183
|
@property
|
|
184
184
|
def na_value(self):
|
|
185
|
-
return
|
|
185
|
+
return Terms({})
|
|
186
186
|
|
|
187
187
|
def valid_value(self, value):
|
|
188
|
-
return isinstance(value, dict) or pd.isna(value) or isinstance(value,
|
|
188
|
+
return isinstance(value, dict) or pd.isna(value) or isinstance(value, Terms)
|
|
189
189
|
|
|
190
190
|
|
|
191
|
-
register_extension_dtype(
|
|
191
|
+
register_extension_dtype(TermsDtype)
|
|
192
192
|
|
|
193
193
|
|
|
194
194
|
def ws_tokenizer(string):
|
|
@@ -226,9 +226,9 @@ def _build_index_from_dict(postings):
|
|
|
226
226
|
# https://www.austintripp.ca/blog/2018/09/12/sparse-matrices-tips1
|
|
227
227
|
for doc_id, tokenized in enumerate(postings):
|
|
228
228
|
if isinstance(tokenized, dict):
|
|
229
|
-
tokenized =
|
|
230
|
-
elif not isinstance(tokenized,
|
|
231
|
-
raise TypeError("Expected a
|
|
229
|
+
tokenized = Terms(tokenized, doc_len=len(tokenized))
|
|
230
|
+
elif not isinstance(tokenized, Terms):
|
|
231
|
+
raise TypeError("Expected a Terms or a dict")
|
|
232
232
|
|
|
233
233
|
if tokenized.encoded:
|
|
234
234
|
posns = posns_enc
|
|
@@ -288,16 +288,16 @@ def _row_to_postings_row(doc_id, row, doc_len, term_dict, posns: PosnBitArray):
|
|
|
288
288
|
enc_term_posns = posns.doc_encoded_posns(term_idx, doc_id=doc_id)
|
|
289
289
|
labeled_posns[term] = enc_term_posns
|
|
290
290
|
|
|
291
|
-
result =
|
|
292
|
-
|
|
291
|
+
result = Terms(tfs, posns=labeled_posns,
|
|
292
|
+
doc_len=doc_len, encoded=True)
|
|
293
293
|
# TODO add positions
|
|
294
294
|
return result
|
|
295
295
|
|
|
296
296
|
|
|
297
|
-
class
|
|
298
|
-
"""An array of tokenized text (
|
|
297
|
+
class SearchArray(ExtensionArray):
|
|
298
|
+
"""An array of tokenized text (Termss)."""
|
|
299
299
|
|
|
300
|
-
dtype =
|
|
300
|
+
dtype = TermsDtype()
|
|
301
301
|
|
|
302
302
|
def __init__(self, postings, tokenizer=ws_tokenizer, avoid_copies=True):
|
|
303
303
|
# Check dtype, raise TypeError
|
|
@@ -322,26 +322,26 @@ class PostingsArray(ExtensionArray):
|
|
|
322
322
|
def tokenized_docs(docs):
|
|
323
323
|
for doc_id, doc in enumerate(docs):
|
|
324
324
|
if pd.isna(doc):
|
|
325
|
-
yield
|
|
325
|
+
yield Terms({})
|
|
326
326
|
else:
|
|
327
327
|
token_stream = tokenizer(doc)
|
|
328
328
|
term_freqs = Counter(token_stream)
|
|
329
329
|
positions = defaultdict(list)
|
|
330
330
|
for posn in range(len(token_stream)):
|
|
331
331
|
positions[token_stream[posn]].append(posn)
|
|
332
|
-
yield
|
|
333
|
-
|
|
334
|
-
|
|
332
|
+
yield Terms(term_freqs,
|
|
333
|
+
doc_len=len(token_stream),
|
|
334
|
+
posns=positions)
|
|
335
335
|
|
|
336
336
|
return cls(tokenized_docs(array), tokenizer)
|
|
337
337
|
|
|
338
338
|
@classmethod
|
|
339
339
|
def _from_sequence(cls, scalars, dtype=None, copy=False):
|
|
340
|
-
"""Construct a new
|
|
340
|
+
"""Construct a new SearchArray from a sequence of scalars (PostingRow or convertible into)."""
|
|
341
341
|
if dtype is not None:
|
|
342
|
-
if not isinstance(dtype,
|
|
342
|
+
if not isinstance(dtype, TermsDtype):
|
|
343
343
|
return scalars
|
|
344
|
-
if isinstance(scalars, np.ndarray) and scalars.dtype ==
|
|
344
|
+
if isinstance(scalars, np.ndarray) and scalars.dtype == TermsDtype():
|
|
345
345
|
return cls(scalars)
|
|
346
346
|
# String types
|
|
347
347
|
elif isinstance(scalars, np.ndarray) and scalars.dtype.kind in 'US':
|
|
@@ -377,7 +377,7 @@ class PostingsArray(ExtensionArray):
|
|
|
377
377
|
# Construct a sliced view of this array
|
|
378
378
|
sliced_tfs = self.term_mat.slice(key)
|
|
379
379
|
sliced_posns = self.posns
|
|
380
|
-
arr =
|
|
380
|
+
arr = SearchArray([], tokenizer=self.tokenizer)
|
|
381
381
|
arr.term_mat = sliced_tfs
|
|
382
382
|
arr.doc_lens = self.doc_lens[key]
|
|
383
383
|
arr.posns = sliced_posns
|
|
@@ -392,13 +392,13 @@ class PostingsArray(ExtensionArray):
|
|
|
392
392
|
value = value.values
|
|
393
393
|
if isinstance(value, pd.DataFrame):
|
|
394
394
|
value = value.values.flatten()
|
|
395
|
-
if isinstance(value,
|
|
395
|
+
if isinstance(value, SearchArray):
|
|
396
396
|
value = value.to_numpy()
|
|
397
397
|
if isinstance(value, list):
|
|
398
398
|
value = np.asarray(value, dtype=object)
|
|
399
399
|
|
|
400
400
|
if not isinstance(value, np.ndarray) and not self.dtype.valid_value(value):
|
|
401
|
-
raise ValueError(f"Cannot set non-object array to
|
|
401
|
+
raise ValueError(f"Cannot set non-object array to SearchArray -- you passed type:{type(value)} -- {value}")
|
|
402
402
|
|
|
403
403
|
# Cant set a single value to an array
|
|
404
404
|
if isinstance(key, numbers.Integral) and isinstance(value, np.ndarray):
|
|
@@ -412,7 +412,7 @@ class PostingsArray(ExtensionArray):
|
|
|
412
412
|
if isinstance(value, float):
|
|
413
413
|
term_mat = np.asarray([value])
|
|
414
414
|
doc_lens = np.asarray([0])
|
|
415
|
-
elif isinstance(value,
|
|
415
|
+
elif isinstance(value, Terms):
|
|
416
416
|
term_mat = np.asarray([value.tf_to_dense(self.term_dict)])
|
|
417
417
|
doc_lens = np.asarray([value.doc_len])
|
|
418
418
|
is_encoded = value.encoded
|
|
@@ -442,7 +442,7 @@ class PostingsArray(ExtensionArray):
|
|
|
442
442
|
warnings.warn(msg)
|
|
443
443
|
|
|
444
444
|
scan_value = value
|
|
445
|
-
if isinstance(value,
|
|
445
|
+
if isinstance(value, Terms):
|
|
446
446
|
scan_value = np.asarray([value])
|
|
447
447
|
for row in scan_value:
|
|
448
448
|
for term in row.terms():
|
|
@@ -458,7 +458,7 @@ class PostingsArray(ExtensionArray):
|
|
|
458
458
|
):
|
|
459
459
|
if dropna:
|
|
460
460
|
counts = Counter(self[:])
|
|
461
|
-
counts.pop(
|
|
461
|
+
counts.pop(Terms({}), None)
|
|
462
462
|
else:
|
|
463
463
|
counts = Counter(self[:])
|
|
464
464
|
return pd.Series(counts)
|
|
@@ -480,7 +480,7 @@ class PostingsArray(ExtensionArray):
|
|
|
480
480
|
return NotImplemented
|
|
481
481
|
|
|
482
482
|
# When other is an ExtensionArray
|
|
483
|
-
if isinstance(other,
|
|
483
|
+
if isinstance(other, SearchArray):
|
|
484
484
|
if len(self) != len(other):
|
|
485
485
|
return False
|
|
486
486
|
elif len(other) == 0:
|
|
@@ -495,9 +495,9 @@ class PostingsArray(ExtensionArray):
|
|
|
495
495
|
# return np.array(self[:]) == np.array(other[:])
|
|
496
496
|
|
|
497
497
|
# When other is a scalar value
|
|
498
|
-
elif isinstance(other,
|
|
499
|
-
other =
|
|
500
|
-
warnings.warn("Comparing a scalar value to a
|
|
498
|
+
elif isinstance(other, Terms):
|
|
499
|
+
other = SearchArray([other], tokenizer=self.tokenizer)
|
|
500
|
+
warnings.warn("Comparing a scalar value to a SearchArray. This is slow.")
|
|
501
501
|
return np.array(self[:]) == np.array(other[:])
|
|
502
502
|
|
|
503
503
|
# When other is a sequence but not an ExtensionArray
|
|
@@ -508,7 +508,7 @@ class PostingsArray(ExtensionArray):
|
|
|
508
508
|
elif len(other) == 0:
|
|
509
509
|
return np.array([], dtype=bool)
|
|
510
510
|
# We actually don't know how it was tokenized
|
|
511
|
-
other =
|
|
511
|
+
other = SearchArray(other, tokenizer=self.tokenizer)
|
|
512
512
|
return np.array(self[:]) == np.array(other[:])
|
|
513
513
|
|
|
514
514
|
# Return False where 'other' is neither the same length nor a scalar
|
|
@@ -528,14 +528,14 @@ class PostingsArray(ExtensionArray):
|
|
|
528
528
|
|
|
529
529
|
if allow_fill and -1 in result_indices:
|
|
530
530
|
if fill_value is None or pd.isna(fill_value):
|
|
531
|
-
fill_value =
|
|
531
|
+
fill_value = Terms({}, encoded=True)
|
|
532
532
|
|
|
533
533
|
to_fill_mask = result_indices == -1
|
|
534
534
|
# This is slow as it rebuilds all the term dictionaries
|
|
535
535
|
# on the subsequent assignment lines
|
|
536
536
|
# However, this case tends to be the exception for
|
|
537
537
|
# most dataframe operations
|
|
538
|
-
taken =
|
|
538
|
+
taken = SearchArray([fill_value] * len(result_indices))
|
|
539
539
|
taken[~to_fill_mask] = self[result_indices[~to_fill_mask]].copy()
|
|
540
540
|
|
|
541
541
|
return taken
|
|
@@ -544,7 +544,7 @@ class PostingsArray(ExtensionArray):
|
|
|
544
544
|
return taken
|
|
545
545
|
|
|
546
546
|
def copy(self):
|
|
547
|
-
postings_arr =
|
|
547
|
+
postings_arr = SearchArray([], tokenizer=self.tokenizer)
|
|
548
548
|
postings_arr.doc_lens = self.doc_lens.copy()
|
|
549
549
|
postings_arr.term_mat = self.term_mat.copy()
|
|
550
550
|
postings_arr.posns = self.posns
|
|
@@ -559,7 +559,7 @@ class PostingsArray(ExtensionArray):
|
|
|
559
559
|
@classmethod
|
|
560
560
|
def _concat_same_type(cls, to_concat):
|
|
561
561
|
concatenated_data = np.concatenate([ea[:] for ea in to_concat])
|
|
562
|
-
return
|
|
562
|
+
return SearchArray(concatenated_data, tokenizer=to_concat[0].tokenizer)
|
|
563
563
|
|
|
564
564
|
@classmethod
|
|
565
565
|
def _from_factorized(cls, values, original):
|
|
@@ -568,7 +568,7 @@ class PostingsArray(ExtensionArray):
|
|
|
568
568
|
def _values_for_factorize(self):
|
|
569
569
|
"""Return an array and missing value suitable for factorization (ie grouping)."""
|
|
570
570
|
arr = np.asarray(self[:], dtype=object)
|
|
571
|
-
return arr,
|
|
571
|
+
return arr, Terms({})
|
|
572
572
|
|
|
573
573
|
def _check_token_arg(self, token):
|
|
574
574
|
if isinstance(token, str):
|
|
@@ -3,7 +3,7 @@ import re
|
|
|
3
3
|
import pandas as pd
|
|
4
4
|
import numpy as np
|
|
5
5
|
from typing import List, Optional, Dict, Tuple
|
|
6
|
-
from searcharray.postings import
|
|
6
|
+
from searcharray.postings import SearchArray
|
|
7
7
|
from searcharray.similarity import Similarity, default_bm25
|
|
8
8
|
|
|
9
9
|
|
|
@@ -75,10 +75,10 @@ def parse_field_boosts(field_lists: List[str]) -> dict:
|
|
|
75
75
|
return out
|
|
76
76
|
|
|
77
77
|
|
|
78
|
-
def get_field(frame, field) ->
|
|
78
|
+
def get_field(frame, field) -> SearchArray:
|
|
79
79
|
if field not in frame.columns:
|
|
80
80
|
raise ValueError(f"Field {field} not in dataframe")
|
|
81
|
-
if not isinstance(frame[field].array,
|
|
81
|
+
if not isinstance(frame[field].array, SearchArray):
|
|
82
82
|
raise ValueError(f"Field {field} is not a searcharray field")
|
|
83
83
|
return frame[field].array
|
|
84
84
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: searcharray
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.31
|
|
4
4
|
Summary: Searchable pandas text extension arrays for prototyping search
|
|
5
5
|
Home-page: https://github.com/softwaredoug/searcharray
|
|
6
6
|
Author: Doug Turnbull
|
|
@@ -36,8 +36,11 @@ SearchArray turns Pandas string columns into a term index. It alows efficient BM
|
|
|
36
36
|
Think Lucene, but as a Pandas column.
|
|
37
37
|
|
|
38
38
|
```python
|
|
39
|
-
In[
|
|
39
|
+
In[2]: from searcharray import SearchArray
|
|
40
|
+
|
|
41
|
+
In[3]: df['title_indexed'] = SearchArray.index(df['title'])
|
|
40
42
|
np.sort(df['title_indexed'].array.score('Cat'))
|
|
43
|
+
|
|
41
44
|
Out[3]: array([ 0. , 0. , 0. , ..., 15.84568033,
|
|
42
45
|
15.84568033, 15.84568033])
|
|
43
46
|
```
|
|
@@ -79,22 +82,22 @@ Out[1]:
|
|
|
79
82
|
Index the text:
|
|
80
83
|
|
|
81
84
|
```
|
|
82
|
-
In[2]: df['title_indexed'] =
|
|
85
|
+
In[2]: df['title_indexed'] = SearchArray.index(df['title'])
|
|
83
86
|
df
|
|
84
87
|
|
|
85
88
|
Out[2]:
|
|
86
89
|
title overview title_indexed
|
|
87
|
-
374430 Black Mirror: White Christmas This feature-length special consists of three ...
|
|
88
|
-
19404 The Brave-Hearted Will Take the Bride Raj is a rich, carefree, happy-go-lucky second...
|
|
89
|
-
278 The Shawshank Redemption Framed in the 1940s for the double murder of h...
|
|
90
|
-
372058 Your Name. High schoolers Mitsuha and Taki are complete s...
|
|
91
|
-
238 The Godfather Spanning the years 1945 to 1955, a chronicle o...
|
|
90
|
+
374430 Black Mirror: White Christmas This feature-length special consists of three ... Terms({'Black': 1, 'Mirror:': 1, 'White'...
|
|
91
|
+
19404 The Brave-Hearted Will Take the Bride Raj is a rich, carefree, happy-go-lucky second... Terms({'The': 1, 'Brave-Hearted': 1, 'Wi...
|
|
92
|
+
278 The Shawshank Redemption Framed in the 1940s for the double murder of h... Terms({'The': 1, 'Shawshank': 1, 'Redemp...
|
|
93
|
+
372058 Your Name. High schoolers Mitsuha and Taki are complete s... Terms({'Your': 1, 'Name.': 1}, {'Your': ...
|
|
94
|
+
238 The Godfather Spanning the years 1945 to 1955, a chronicle o... Terms({'The': 1, 'Godfather': 1}, {'The'...
|
|
92
95
|
... ... ... ...
|
|
93
|
-
65513 They Came Back The lives of the residents of a small French t...
|
|
94
|
-
65515 The Eleventh Hour An ex-Navy SEAL, Michael Adams, (Matthew Reese...
|
|
95
|
-
65521 Pyaar Ka Punchnama Outspoken and overly critical Nishant Agarwal ...
|
|
96
|
-
32767 Romero Romero is a compelling and deeply moving look ...
|
|
97
|
-
65534 Poison Paul Braconnier and his wife Blandine only hav...
|
|
96
|
+
65513 They Came Back The lives of the residents of a small French t... Terms({'Back': 1, 'They': 1, 'Came': 1},...
|
|
97
|
+
65515 The Eleventh Hour An ex-Navy SEAL, Michael Adams, (Matthew Reese... Terms({'The': 1, 'Hour': 1, 'Eleventh': ...
|
|
98
|
+
65521 Pyaar Ka Punchnama Outspoken and overly critical Nishant Agarwal ... Terms({'Ka': 1, 'Pyaar': 1, 'Punchnama':...
|
|
99
|
+
32767 Romero Romero is a compelling and deeply moving look ... Terms({'Romero': 1}, {'Romero': [0]})
|
|
100
|
+
65534 Poison Paul Braconnier and his wife Blandine only hav... Terms({'Poison': 1}, {'Poison': [0]})```
|
|
98
101
|
```
|
|
99
102
|
|
|
100
103
|
Then search, getting top N with `Cat`
|
|
@@ -116,16 +119,16 @@ And since its just pandas, we can, of course just retrieve the top matches
|
|
|
116
119
|
In[5]: df.iloc[top_n_cat[-10:]]
|
|
117
120
|
Out[5]:
|
|
118
121
|
title overview title_indexed
|
|
119
|
-
24106 The Black Cat American honeymooners in Hungary are trapped i...
|
|
120
|
-
12593 Fritz the Cat A hypocritical swinging college student cat ra...
|
|
121
|
-
39853 The Cat Concerto Tom enters from stage left in white tie and ta...
|
|
122
|
-
75491 The Rabbi's Cat Based on the best-selling graphic novel by Joa...
|
|
123
|
-
57353 Cat Run When a sexy, high-end escort holds the key evi...
|
|
124
|
-
25508 Cat People Sketch artist Irena Dubrovna (Simon) and Ameri...
|
|
125
|
-
11694 Cat Ballou A woman seeking revenge for her murdered fathe...
|
|
126
|
-
25078 Cat Soup The surreal black comedy follows Nyatta, an an...
|
|
127
|
-
35888 Cat Chaser A Miami hotel owner finds danger when be becom...
|
|
128
|
-
6217 Cat People After years of separation, Irina (Nastassja Ki...
|
|
122
|
+
24106 The Black Cat American honeymooners in Hungary are trapped i... Terms({'Black': 1, 'The': 1, 'Cat': 1}, ...
|
|
123
|
+
12593 Fritz the Cat A hypocritical swinging college student cat ra... Terms({'Cat': 1, 'the': 1, 'Fritz': 1}, ...
|
|
124
|
+
39853 The Cat Concerto Tom enters from stage left in white tie and ta... Terms({'The': 1, 'Cat': 1, 'Concerto': 1...
|
|
125
|
+
75491 The Rabbi's Cat Based on the best-selling graphic novel by Joa... Terms({'The': 1, 'Cat': 1, "Rabbi's": 1}...
|
|
126
|
+
57353 Cat Run When a sexy, high-end escort holds the key evi... Terms({'Cat': 1, 'Run': 1}, {'Cat': [0],...
|
|
127
|
+
25508 Cat People Sketch artist Irena Dubrovna (Simon) and Ameri... Terms({'Cat': 1, 'People': 1}, {'Cat': [...
|
|
128
|
+
11694 Cat Ballou A woman seeking revenge for her murdered fathe... Terms({'Cat': 1, 'Ballou': 1}, {'Cat': [...
|
|
129
|
+
25078 Cat Soup The surreal black comedy follows Nyatta, an an... Terms({'Cat': 1, 'Soup': 1}, {'Cat': [0]...
|
|
130
|
+
35888 Cat Chaser A Miami hotel owner finds danger when be becom... Terms({'Cat': 1, 'Chaser': 1}, {'Cat': [...
|
|
131
|
+
6217 Cat People After years of separation, Irina (Nastassja Ki... Terms({'Cat': 1, 'People': 1}, {'Cat': [...
|
|
129
132
|
```
|
|
130
133
|
|
|
131
134
|
More use cases can be seen [in the colab notebook](https://colab.research.google.com/drive/1w_Ajn5rHzcISKhdCuPhhVFav3zrvKWn1)
|
|
@@ -170,7 +173,7 @@ And you can pass any tokenizer that matches this signature to index:
|
|
|
170
173
|
def ws_lowercase_tokenizer(string):
|
|
171
174
|
return string.lower().split()
|
|
172
175
|
|
|
173
|
-
df['title_indexed'] =
|
|
176
|
+
df['title_indexed'] = SearchArray.index(df['title'], tokenizer=ws_lowercase_tokenizer)
|
|
174
177
|
```
|
|
175
178
|
|
|
176
179
|
Create your own using stemming libraries, or whatever Python functionality you want.
|
|
@@ -36,7 +36,7 @@ setup(
|
|
|
36
36
|
# For a discussion on single-sourcing the version across setup.py and the
|
|
37
37
|
# project code, see
|
|
38
38
|
# https://packaging.python.org/guides/single-sourcing-package-version/
|
|
39
|
-
version="0.0.
|
|
39
|
+
version="0.0.31", # Required
|
|
40
40
|
# This is a one-line description or tagline of what your project does. This
|
|
41
41
|
# corresponds to the "Summary" metadata field:
|
|
42
42
|
# https://packaging.python.org/specifications/core-metadata/#summary
|