searcharray 0.0.51__tar.gz → 0.0.53__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {searcharray-0.0.51 → searcharray-0.0.53}/PKG-INFO +3 -3
- {searcharray-0.0.51 → searcharray-0.0.53}/README.md +2 -2
- {searcharray-0.0.51 → searcharray-0.0.53}/searcharray/phrase/middle_out.py +10 -11
- {searcharray-0.0.51 → searcharray-0.0.53}/searcharray/roaringish/roaringish_ops.c +117 -117
- {searcharray-0.0.51 → searcharray-0.0.53}/searcharray/roaringish/snp_ops.c +4122 -3701
- {searcharray-0.0.51 → searcharray-0.0.53}/searcharray/roaringish/snp_ops.pyx +205 -192
- {searcharray-0.0.51 → searcharray-0.0.53}/searcharray/roaringish/spans.c +117 -117
- {searcharray-0.0.51 → searcharray-0.0.53}/searcharray.egg-info/PKG-INFO +3 -3
- {searcharray-0.0.51 → searcharray-0.0.53}/setup.py +1 -1
- {searcharray-0.0.51 → searcharray-0.0.53}/test/test_snp_ops.py +92 -0
- {searcharray-0.0.51 → searcharray-0.0.53}/pyproject.toml +0 -0
- {searcharray-0.0.51 → searcharray-0.0.53}/searcharray/__init__.py +0 -0
- {searcharray-0.0.51 → searcharray-0.0.53}/searcharray/indexing.py +0 -0
- {searcharray-0.0.51 → searcharray-0.0.53}/searcharray/phrase/__init__.py +0 -0
- {searcharray-0.0.51 → searcharray-0.0.53}/searcharray/phrase/bigram_freqs.py +0 -0
- {searcharray-0.0.51 → searcharray-0.0.53}/searcharray/phrase/posn_diffs.py +0 -0
- {searcharray-0.0.51 → searcharray-0.0.53}/searcharray/phrase/scan_merge.py +0 -0
- {searcharray-0.0.51 → searcharray-0.0.53}/searcharray/phrase/spans.py +0 -0
- {searcharray-0.0.51 → searcharray-0.0.53}/searcharray/postings.py +0 -0
- {searcharray-0.0.51 → searcharray-0.0.53}/searcharray/roaringish/__init__.py +0 -0
- {searcharray-0.0.51 → searcharray-0.0.53}/searcharray/roaringish/roaringish.py +0 -0
- {searcharray-0.0.51 → searcharray-0.0.53}/searcharray/roaringish/roaringish_ops.pyx +0 -0
- {searcharray-0.0.51 → searcharray-0.0.53}/searcharray/roaringish/snp_ops.pxd +0 -0
- {searcharray-0.0.51 → searcharray-0.0.53}/searcharray/roaringish/spans.pyx +0 -0
- {searcharray-0.0.51 → searcharray-0.0.53}/searcharray/similarity.py +0 -0
- {searcharray-0.0.51 → searcharray-0.0.53}/searcharray/solr.py +0 -0
- {searcharray-0.0.51 → searcharray-0.0.53}/searcharray/term_dict.py +0 -0
- {searcharray-0.0.51 → searcharray-0.0.53}/searcharray/utils/__init__.py +0 -0
- {searcharray-0.0.51 → searcharray-0.0.53}/searcharray/utils/bitcount.py +0 -0
- {searcharray-0.0.51 → searcharray-0.0.53}/searcharray/utils/mat_set.py +0 -0
- {searcharray-0.0.51 → searcharray-0.0.53}/searcharray/utils/row_viewable_matrix.py +0 -0
- {searcharray-0.0.51 → searcharray-0.0.53}/searcharray/utils/sort.py +0 -0
- {searcharray-0.0.51 → searcharray-0.0.53}/searcharray.egg-info/SOURCES.txt +0 -0
- {searcharray-0.0.51 → searcharray-0.0.53}/searcharray.egg-info/dependency_links.txt +0 -0
- {searcharray-0.0.51 → searcharray-0.0.53}/searcharray.egg-info/entry_points.txt +0 -0
- {searcharray-0.0.51 → searcharray-0.0.53}/searcharray.egg-info/requires.txt +0 -0
- {searcharray-0.0.51 → searcharray-0.0.53}/searcharray.egg-info/top_level.txt +0 -0
- {searcharray-0.0.51 → searcharray-0.0.53}/setup.cfg +0 -0
- {searcharray-0.0.51 → searcharray-0.0.53}/test/test_bitcount64.py +0 -0
- {searcharray-0.0.51 → searcharray-0.0.53}/test/test_extension_array.py +0 -0
- {searcharray-0.0.51 → searcharray-0.0.53}/test/test_minmax_posns.py +0 -0
- {searcharray-0.0.51 → searcharray-0.0.53}/test/test_msmarco.py +0 -0
- {searcharray-0.0.51 → searcharray-0.0.53}/test/test_phrase_matches.py +0 -0
- {searcharray-0.0.51 → searcharray-0.0.53}/test/test_search.py +0 -0
- {searcharray-0.0.51 → searcharray-0.0.53}/test/test_similarity.py +0 -0
- {searcharray-0.0.51 → searcharray-0.0.53}/test/test_slop_matches.py +0 -0
- {searcharray-0.0.51 → searcharray-0.0.53}/test/test_solr.py +0 -0
- {searcharray-0.0.51 → searcharray-0.0.53}/test/test_tmdb.py +0 -0
- {searcharray-0.0.51 → searcharray-0.0.53}/test/test_utils.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: searcharray
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.53
|
|
4
4
|
Summary: Searchable pandas text extension arrays for prototyping search
|
|
5
5
|
Home-page: https://github.com/softwaredoug/searcharray
|
|
6
6
|
Author: Doug Turnbull
|
|
@@ -153,13 +153,13 @@ We want you to be able to work with a reasonable dataset (100X-1M docs) relative
|
|
|
153
153
|
|
|
154
154
|
### Experimentation, reranking, functionality over scalability
|
|
155
155
|
|
|
156
|
-
Instead of building for 'big data' our goal is to build for
|
|
156
|
+
Instead of building for 'big data' our goal is to build for *small-data*. That is, focus on capabilities and expressiveness of Pandas, over limiting functionality in favor of scalability.
|
|
157
157
|
|
|
158
158
|
To this end, the applications of searcharray will tend to be focused on experimentation and top N candidate reranking. For experimentation, we want any ideas expressed in Pandas to have a somewhat clear path / "contract" in how they'd be implemented in a classical lexical search engine. For reranking, we want to load some top N results from a base system and be able to modify them.
|
|
159
159
|
|
|
160
160
|
### Make lexical search compatible with the data stack
|
|
161
161
|
|
|
162
|
-
We know in search, RAG, and other retrieval problems
|
|
162
|
+
We know in search, RAG, and other retrieval problems [hybrid search](https://www.pinecone.io/learn/hybrid-search-intro/) techniques dominate. Yet often its cast in terms of a giant, weird, big data lexical search engine that looks odd to most data scientists being joined with a vector database. We want lexical search to be more approachable to data scientists and ML engineers building these systems.
|
|
163
163
|
|
|
164
164
|
## Non-goals
|
|
165
165
|
|
|
@@ -124,13 +124,13 @@ We want you to be able to work with a reasonable dataset (100X-1M docs) relative
|
|
|
124
124
|
|
|
125
125
|
### Experimentation, reranking, functionality over scalability
|
|
126
126
|
|
|
127
|
-
Instead of building for 'big data' our goal is to build for
|
|
127
|
+
Instead of building for 'big data' our goal is to build for *small-data*. That is, focus on capabilities and expressiveness of Pandas, over limiting functionality in favor of scalability.
|
|
128
128
|
|
|
129
129
|
To this end, the applications of searcharray will tend to be focused on experimentation and top N candidate reranking. For experimentation, we want any ideas expressed in Pandas to have a somewhat clear path / "contract" in how they'd be implemented in a classical lexical search engine. For reranking, we want to load some top N results from a base system and be able to modify them.
|
|
130
130
|
|
|
131
131
|
### Make lexical search compatible with the data stack
|
|
132
132
|
|
|
133
|
-
We know in search, RAG, and other retrieval problems
|
|
133
|
+
We know in search, RAG, and other retrieval problems [hybrid search](https://www.pinecone.io/learn/hybrid-search-intro/) techniques dominate. Yet often its cast in terms of a giant, weird, big data lexical search engine that looks odd to most data scientists being joined with a vector database. We want lexical search to be more approachable to data scientists and ML engineers building these systems.
|
|
134
134
|
|
|
135
135
|
## Non-goals
|
|
136
136
|
|
|
@@ -70,9 +70,9 @@ def trim_phrase_search(encoded_posns: List[np.ndarray],
|
|
|
70
70
|
return encoded_posns
|
|
71
71
|
|
|
72
72
|
|
|
73
|
-
def
|
|
74
|
-
|
|
75
|
-
|
|
73
|
+
def _compute_phrase_freqs_left_to_right(encoded_posns: List[np.ndarray],
|
|
74
|
+
phrase_freqs: np.ndarray,
|
|
75
|
+
trim: bool = True) -> np.ndarray:
|
|
76
76
|
"""Compute phrase freqs from a set of encoded positions."""
|
|
77
77
|
if len(encoded_posns) < 2:
|
|
78
78
|
raise ValueError("phrase must have at least two terms")
|
|
@@ -97,9 +97,9 @@ def _compute_phrase_freqs_rhs(encoded_posns: List[np.ndarray],
|
|
|
97
97
|
return phrase_freqs
|
|
98
98
|
|
|
99
99
|
|
|
100
|
-
def
|
|
101
|
-
|
|
102
|
-
|
|
100
|
+
def _compute_phrase_freqs_right_to_left(encoded_posns: List[np.ndarray],
|
|
101
|
+
phrase_freqs: np.ndarray,
|
|
102
|
+
trim: bool = True) -> np.ndarray:
|
|
103
103
|
"""Compute phrase freqs from a set of encoded positions."""
|
|
104
104
|
if len(encoded_posns) < 2:
|
|
105
105
|
raise ValueError("phrase must have at least two terms")
|
|
@@ -129,16 +129,15 @@ def compute_phrase_freqs(encoded_posns: List[np.ndarray],
|
|
|
129
129
|
trim: bool = False) -> np.ndarray:
|
|
130
130
|
"""Compute phrase freqs from a set of encoded positions."""
|
|
131
131
|
shortest_len_index = min(enumerate(encoded_posns), key=lambda x: len(x[1]))[0]
|
|
132
|
-
# Slice LHS / RHS at shortest_len_index
|
|
133
132
|
if shortest_len_index <= 1:
|
|
134
|
-
return
|
|
133
|
+
return _compute_phrase_freqs_left_to_right(encoded_posns, phrase_freqs, trim=trim)
|
|
135
134
|
elif shortest_len_index >= len(encoded_posns) - 2:
|
|
136
|
-
return
|
|
135
|
+
return _compute_phrase_freqs_right_to_left(encoded_posns, phrase_freqs, trim=trim)
|
|
137
136
|
else:
|
|
138
137
|
# We optimize this case by going middle-out
|
|
139
138
|
# We can take the min of both directions phrase freqs
|
|
140
|
-
lhs =
|
|
141
|
-
rhs =
|
|
139
|
+
lhs = _compute_phrase_freqs_left_to_right(encoded_posns[:shortest_len_index], phrase_freqs, trim=trim)
|
|
140
|
+
rhs = _compute_phrase_freqs_right_to_left(encoded_posns[shortest_len_index:], phrase_freqs, trim=trim)
|
|
142
141
|
phrase_freqs = np.minimum(lhs, rhs)
|
|
143
142
|
return phrase_freqs
|
|
144
143
|
|