searcharray 0.0.50__tar.gz → 0.0.52__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {searcharray-0.0.50 → searcharray-0.0.52}/PKG-INFO +8 -4
- {searcharray-0.0.50 → searcharray-0.0.52}/README.md +7 -3
- {searcharray-0.0.50 → searcharray-0.0.52}/searcharray/__init__.py +1 -0
- {searcharray-0.0.50 → searcharray-0.0.52}/searcharray/phrase/middle_out.py +10 -11
- {searcharray-0.0.50 → searcharray-0.0.52}/searcharray/roaringish/roaringish_ops.c +117 -117
- {searcharray-0.0.50 → searcharray-0.0.52}/searcharray/roaringish/snp_ops.c +2120 -1750
- {searcharray-0.0.50 → searcharray-0.0.52}/searcharray/roaringish/snp_ops.pyx +94 -85
- {searcharray-0.0.50 → searcharray-0.0.52}/searcharray/roaringish/spans.c +117 -117
- searcharray-0.0.52/searcharray/utils/sort.py +45 -0
- {searcharray-0.0.50 → searcharray-0.0.52}/searcharray.egg-info/PKG-INFO +8 -4
- {searcharray-0.0.50 → searcharray-0.0.52}/searcharray.egg-info/SOURCES.txt +1 -0
- {searcharray-0.0.50 → searcharray-0.0.52}/setup.py +1 -1
- {searcharray-0.0.50 → searcharray-0.0.52}/test/test_msmarco.py +71 -0
- {searcharray-0.0.50 → searcharray-0.0.52}/test/test_snp_ops.py +92 -0
- {searcharray-0.0.50 → searcharray-0.0.52}/pyproject.toml +0 -0
- {searcharray-0.0.50 → searcharray-0.0.52}/searcharray/indexing.py +0 -0
- {searcharray-0.0.50 → searcharray-0.0.52}/searcharray/phrase/__init__.py +0 -0
- {searcharray-0.0.50 → searcharray-0.0.52}/searcharray/phrase/bigram_freqs.py +0 -0
- {searcharray-0.0.50 → searcharray-0.0.52}/searcharray/phrase/posn_diffs.py +0 -0
- {searcharray-0.0.50 → searcharray-0.0.52}/searcharray/phrase/scan_merge.py +0 -0
- {searcharray-0.0.50 → searcharray-0.0.52}/searcharray/phrase/spans.py +0 -0
- {searcharray-0.0.50 → searcharray-0.0.52}/searcharray/postings.py +0 -0
- {searcharray-0.0.50 → searcharray-0.0.52}/searcharray/roaringish/__init__.py +0 -0
- {searcharray-0.0.50 → searcharray-0.0.52}/searcharray/roaringish/roaringish.py +0 -0
- {searcharray-0.0.50 → searcharray-0.0.52}/searcharray/roaringish/roaringish_ops.pyx +0 -0
- {searcharray-0.0.50 → searcharray-0.0.52}/searcharray/roaringish/snp_ops.pxd +0 -0
- {searcharray-0.0.50 → searcharray-0.0.52}/searcharray/roaringish/spans.pyx +0 -0
- {searcharray-0.0.50 → searcharray-0.0.52}/searcharray/similarity.py +0 -0
- {searcharray-0.0.50 → searcharray-0.0.52}/searcharray/solr.py +0 -0
- {searcharray-0.0.50 → searcharray-0.0.52}/searcharray/term_dict.py +0 -0
- {searcharray-0.0.50 → searcharray-0.0.52}/searcharray/utils/__init__.py +0 -0
- {searcharray-0.0.50 → searcharray-0.0.52}/searcharray/utils/bitcount.py +0 -0
- {searcharray-0.0.50 → searcharray-0.0.52}/searcharray/utils/mat_set.py +0 -0
- {searcharray-0.0.50 → searcharray-0.0.52}/searcharray/utils/row_viewable_matrix.py +0 -0
- {searcharray-0.0.50 → searcharray-0.0.52}/searcharray.egg-info/dependency_links.txt +0 -0
- {searcharray-0.0.50 → searcharray-0.0.52}/searcharray.egg-info/entry_points.txt +0 -0
- {searcharray-0.0.50 → searcharray-0.0.52}/searcharray.egg-info/requires.txt +0 -0
- {searcharray-0.0.50 → searcharray-0.0.52}/searcharray.egg-info/top_level.txt +0 -0
- {searcharray-0.0.50 → searcharray-0.0.52}/setup.cfg +0 -0
- {searcharray-0.0.50 → searcharray-0.0.52}/test/test_bitcount64.py +0 -0
- {searcharray-0.0.50 → searcharray-0.0.52}/test/test_extension_array.py +0 -0
- {searcharray-0.0.50 → searcharray-0.0.52}/test/test_minmax_posns.py +0 -0
- {searcharray-0.0.50 → searcharray-0.0.52}/test/test_phrase_matches.py +0 -0
- {searcharray-0.0.50 → searcharray-0.0.52}/test/test_search.py +0 -0
- {searcharray-0.0.50 → searcharray-0.0.52}/test/test_similarity.py +0 -0
- {searcharray-0.0.50 → searcharray-0.0.52}/test/test_slop_matches.py +0 -0
- {searcharray-0.0.50 → searcharray-0.0.52}/test/test_solr.py +0 -0
- {searcharray-0.0.50 → searcharray-0.0.52}/test/test_tmdb.py +0 -0
- {searcharray-0.0.50 → searcharray-0.0.52}/test/test_utils.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: searcharray
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.52
|
|
4
4
|
Summary: Searchable pandas text extension arrays for prototyping search
|
|
5
5
|
Home-page: https://github.com/softwaredoug/searcharray
|
|
6
6
|
Author: Doug Turnbull
|
|
@@ -29,7 +29,7 @@ Requires-Dist: coverage; extra == "test"
|
|
|
29
29
|
|
|
30
30
|
# SearchArray
|
|
31
31
|
|
|
32
|
-
[](https://github.com/softwaredoug/searcharray/actions/workflows/test.yml)
|
|
32
|
+
[](https://github.com/softwaredoug/searcharray/actions/workflows/test.yml) | Discuss at [Relevance Slack](https://join.slack.com/t/relevancy/shared_invite/zt-2ccxvzn1w-2_50zf9xBOSv3n06Cu15jg)
|
|
33
33
|
|
|
34
34
|
SearchArray turns Pandas string columns into a term index. It alows efficient BM25 / TFIDF scoring of phrases and individual tokens.
|
|
35
35
|
|
|
@@ -153,13 +153,13 @@ We want you to be able to work with a reasonable dataset (100X-1M docs) relative
|
|
|
153
153
|
|
|
154
154
|
### Experimentation, reranking, functionality over scalability
|
|
155
155
|
|
|
156
|
-
Instead of building for 'big data' our goal is to build for
|
|
156
|
+
Instead of building for 'big data' our goal is to build for *small-data*. That is, focus on capabilities and expressiveness of Pandas, over limiting functionality in favor of scalability.
|
|
157
157
|
|
|
158
158
|
To this end, the applications of searcharray will tend to be focused on experimentation and top N candidate reranking. For experimentation, we want any ideas expressed in Pandas to have a somewhat clear path / "contract" in how they'd be implemented in a classical lexical search engine. For reranking, we want to load some top N results from a base system and be able to modify them.
|
|
159
159
|
|
|
160
160
|
### Make lexical search compatible with the data stack
|
|
161
161
|
|
|
162
|
-
We know in search, RAG, and other retrieval problems
|
|
162
|
+
We know in search, RAG, and other retrieval problems [hybrid search](https://www.pinecone.io/learn/hybrid-search-intro/) techniques dominate. Yet often its cast in terms of a giant, weird, big data lexical search engine that looks odd to most data scientists being joined with a vector database. We want lexical search to be more approachable to data scientists and ML engineers building these systems.
|
|
163
163
|
|
|
164
164
|
## Non-goals
|
|
165
165
|
|
|
@@ -206,6 +206,10 @@ df['score'] = df['title_indexed'].score('Cat') * df['hrs_into_past']
|
|
|
206
206
|
|
|
207
207
|
We focus on the lexical, ie "BM25-ish" and adjacent problems. There are other great tools for vector search out there.
|
|
208
208
|
|
|
209
|
+
## Need help?
|
|
210
|
+
|
|
211
|
+
Visit the [#searcharray channel on Relevance Slack](https://join.slack.com/t/relevancy/shared_invite/zt-2ccxvzn1w-2_50zf9xBOSv3n06Cu15jg)
|
|
212
|
+
|
|
209
213
|
## TODOs / Future Work / Known issues
|
|
210
214
|
|
|
211
215
|
* Always more efficient
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# SearchArray
|
|
2
2
|
|
|
3
|
-
[](https://github.com/softwaredoug/searcharray/actions/workflows/test.yml)
|
|
3
|
+
[](https://github.com/softwaredoug/searcharray/actions/workflows/test.yml) | Discuss at [Relevance Slack](https://join.slack.com/t/relevancy/shared_invite/zt-2ccxvzn1w-2_50zf9xBOSv3n06Cu15jg)
|
|
4
4
|
|
|
5
5
|
SearchArray turns Pandas string columns into a term index. It alows efficient BM25 / TFIDF scoring of phrases and individual tokens.
|
|
6
6
|
|
|
@@ -124,13 +124,13 @@ We want you to be able to work with a reasonable dataset (100X-1M docs) relative
|
|
|
124
124
|
|
|
125
125
|
### Experimentation, reranking, functionality over scalability
|
|
126
126
|
|
|
127
|
-
Instead of building for 'big data' our goal is to build for
|
|
127
|
+
Instead of building for 'big data' our goal is to build for *small-data*. That is, focus on capabilities and expressiveness of Pandas, over limiting functionality in favor of scalability.
|
|
128
128
|
|
|
129
129
|
To this end, the applications of searcharray will tend to be focused on experimentation and top N candidate reranking. For experimentation, we want any ideas expressed in Pandas to have a somewhat clear path / "contract" in how they'd be implemented in a classical lexical search engine. For reranking, we want to load some top N results from a base system and be able to modify them.
|
|
130
130
|
|
|
131
131
|
### Make lexical search compatible with the data stack
|
|
132
132
|
|
|
133
|
-
We know in search, RAG, and other retrieval problems
|
|
133
|
+
We know in search, RAG, and other retrieval problems [hybrid search](https://www.pinecone.io/learn/hybrid-search-intro/) techniques dominate. Yet often its cast in terms of a giant, weird, big data lexical search engine that looks odd to most data scientists being joined with a vector database. We want lexical search to be more approachable to data scientists and ML engineers building these systems.
|
|
134
134
|
|
|
135
135
|
## Non-goals
|
|
136
136
|
|
|
@@ -177,6 +177,10 @@ df['score'] = df['title_indexed'].score('Cat') * df['hrs_into_past']
|
|
|
177
177
|
|
|
178
178
|
We focus on the lexical, ie "BM25-ish" and adjacent problems. There are other great tools for vector search out there.
|
|
179
179
|
|
|
180
|
+
## Need help?
|
|
181
|
+
|
|
182
|
+
Visit the [#searcharray channel on Relevance Slack](https://join.slack.com/t/relevancy/shared_invite/zt-2ccxvzn1w-2_50zf9xBOSv3n06Cu15jg)
|
|
183
|
+
|
|
180
184
|
## TODOs / Future Work / Known issues
|
|
181
185
|
|
|
182
186
|
* Always more efficient
|
|
@@ -70,9 +70,9 @@ def trim_phrase_search(encoded_posns: List[np.ndarray],
|
|
|
70
70
|
return encoded_posns
|
|
71
71
|
|
|
72
72
|
|
|
73
|
-
def
|
|
74
|
-
|
|
75
|
-
|
|
73
|
+
def _compute_phrase_freqs_left_to_right(encoded_posns: List[np.ndarray],
|
|
74
|
+
phrase_freqs: np.ndarray,
|
|
75
|
+
trim: bool = True) -> np.ndarray:
|
|
76
76
|
"""Compute phrase freqs from a set of encoded positions."""
|
|
77
77
|
if len(encoded_posns) < 2:
|
|
78
78
|
raise ValueError("phrase must have at least two terms")
|
|
@@ -97,9 +97,9 @@ def _compute_phrase_freqs_rhs(encoded_posns: List[np.ndarray],
|
|
|
97
97
|
return phrase_freqs
|
|
98
98
|
|
|
99
99
|
|
|
100
|
-
def
|
|
101
|
-
|
|
102
|
-
|
|
100
|
+
def _compute_phrase_freqs_right_to_left(encoded_posns: List[np.ndarray],
|
|
101
|
+
phrase_freqs: np.ndarray,
|
|
102
|
+
trim: bool = True) -> np.ndarray:
|
|
103
103
|
"""Compute phrase freqs from a set of encoded positions."""
|
|
104
104
|
if len(encoded_posns) < 2:
|
|
105
105
|
raise ValueError("phrase must have at least two terms")
|
|
@@ -129,16 +129,15 @@ def compute_phrase_freqs(encoded_posns: List[np.ndarray],
|
|
|
129
129
|
trim: bool = False) -> np.ndarray:
|
|
130
130
|
"""Compute phrase freqs from a set of encoded positions."""
|
|
131
131
|
shortest_len_index = min(enumerate(encoded_posns), key=lambda x: len(x[1]))[0]
|
|
132
|
-
# Slice LHS / RHS at shortest_len_index
|
|
133
132
|
if shortest_len_index <= 1:
|
|
134
|
-
return
|
|
133
|
+
return _compute_phrase_freqs_left_to_right(encoded_posns, phrase_freqs, trim=trim)
|
|
135
134
|
elif shortest_len_index >= len(encoded_posns) - 2:
|
|
136
|
-
return
|
|
135
|
+
return _compute_phrase_freqs_right_to_left(encoded_posns, phrase_freqs, trim=trim)
|
|
137
136
|
else:
|
|
138
137
|
# We optimize this case by going middle-out
|
|
139
138
|
# We can take the min of both directions phrase freqs
|
|
140
|
-
lhs =
|
|
141
|
-
rhs =
|
|
139
|
+
lhs = _compute_phrase_freqs_left_to_right(encoded_posns[:shortest_len_index], phrase_freqs, trim=trim)
|
|
140
|
+
rhs = _compute_phrase_freqs_right_to_left(encoded_posns[shortest_len_index:], phrase_freqs, trim=trim)
|
|
142
141
|
phrase_freqs = np.minimum(lhs, rhs)
|
|
143
142
|
return phrase_freqs
|
|
144
143
|
|