searcharray 0.0.51__tar.gz → 0.0.52__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. {searcharray-0.0.51 → searcharray-0.0.52}/PKG-INFO +3 -3
  2. {searcharray-0.0.51 → searcharray-0.0.52}/README.md +2 -2
  3. {searcharray-0.0.51 → searcharray-0.0.52}/searcharray/phrase/middle_out.py +10 -11
  4. {searcharray-0.0.51 → searcharray-0.0.52}/searcharray/roaringish/roaringish_ops.c +117 -117
  5. {searcharray-0.0.51 → searcharray-0.0.52}/searcharray/roaringish/snp_ops.c +2120 -1750
  6. {searcharray-0.0.51 → searcharray-0.0.52}/searcharray/roaringish/snp_ops.pyx +94 -85
  7. {searcharray-0.0.51 → searcharray-0.0.52}/searcharray/roaringish/spans.c +117 -117
  8. {searcharray-0.0.51 → searcharray-0.0.52}/searcharray.egg-info/PKG-INFO +3 -3
  9. {searcharray-0.0.51 → searcharray-0.0.52}/setup.py +1 -1
  10. {searcharray-0.0.51 → searcharray-0.0.52}/test/test_snp_ops.py +92 -0
  11. {searcharray-0.0.51 → searcharray-0.0.52}/pyproject.toml +0 -0
  12. {searcharray-0.0.51 → searcharray-0.0.52}/searcharray/__init__.py +0 -0
  13. {searcharray-0.0.51 → searcharray-0.0.52}/searcharray/indexing.py +0 -0
  14. {searcharray-0.0.51 → searcharray-0.0.52}/searcharray/phrase/__init__.py +0 -0
  15. {searcharray-0.0.51 → searcharray-0.0.52}/searcharray/phrase/bigram_freqs.py +0 -0
  16. {searcharray-0.0.51 → searcharray-0.0.52}/searcharray/phrase/posn_diffs.py +0 -0
  17. {searcharray-0.0.51 → searcharray-0.0.52}/searcharray/phrase/scan_merge.py +0 -0
  18. {searcharray-0.0.51 → searcharray-0.0.52}/searcharray/phrase/spans.py +0 -0
  19. {searcharray-0.0.51 → searcharray-0.0.52}/searcharray/postings.py +0 -0
  20. {searcharray-0.0.51 → searcharray-0.0.52}/searcharray/roaringish/__init__.py +0 -0
  21. {searcharray-0.0.51 → searcharray-0.0.52}/searcharray/roaringish/roaringish.py +0 -0
  22. {searcharray-0.0.51 → searcharray-0.0.52}/searcharray/roaringish/roaringish_ops.pyx +0 -0
  23. {searcharray-0.0.51 → searcharray-0.0.52}/searcharray/roaringish/snp_ops.pxd +0 -0
  24. {searcharray-0.0.51 → searcharray-0.0.52}/searcharray/roaringish/spans.pyx +0 -0
  25. {searcharray-0.0.51 → searcharray-0.0.52}/searcharray/similarity.py +0 -0
  26. {searcharray-0.0.51 → searcharray-0.0.52}/searcharray/solr.py +0 -0
  27. {searcharray-0.0.51 → searcharray-0.0.52}/searcharray/term_dict.py +0 -0
  28. {searcharray-0.0.51 → searcharray-0.0.52}/searcharray/utils/__init__.py +0 -0
  29. {searcharray-0.0.51 → searcharray-0.0.52}/searcharray/utils/bitcount.py +0 -0
  30. {searcharray-0.0.51 → searcharray-0.0.52}/searcharray/utils/mat_set.py +0 -0
  31. {searcharray-0.0.51 → searcharray-0.0.52}/searcharray/utils/row_viewable_matrix.py +0 -0
  32. {searcharray-0.0.51 → searcharray-0.0.52}/searcharray/utils/sort.py +0 -0
  33. {searcharray-0.0.51 → searcharray-0.0.52}/searcharray.egg-info/SOURCES.txt +0 -0
  34. {searcharray-0.0.51 → searcharray-0.0.52}/searcharray.egg-info/dependency_links.txt +0 -0
  35. {searcharray-0.0.51 → searcharray-0.0.52}/searcharray.egg-info/entry_points.txt +0 -0
  36. {searcharray-0.0.51 → searcharray-0.0.52}/searcharray.egg-info/requires.txt +0 -0
  37. {searcharray-0.0.51 → searcharray-0.0.52}/searcharray.egg-info/top_level.txt +0 -0
  38. {searcharray-0.0.51 → searcharray-0.0.52}/setup.cfg +0 -0
  39. {searcharray-0.0.51 → searcharray-0.0.52}/test/test_bitcount64.py +0 -0
  40. {searcharray-0.0.51 → searcharray-0.0.52}/test/test_extension_array.py +0 -0
  41. {searcharray-0.0.51 → searcharray-0.0.52}/test/test_minmax_posns.py +0 -0
  42. {searcharray-0.0.51 → searcharray-0.0.52}/test/test_msmarco.py +0 -0
  43. {searcharray-0.0.51 → searcharray-0.0.52}/test/test_phrase_matches.py +0 -0
  44. {searcharray-0.0.51 → searcharray-0.0.52}/test/test_search.py +0 -0
  45. {searcharray-0.0.51 → searcharray-0.0.52}/test/test_similarity.py +0 -0
  46. {searcharray-0.0.51 → searcharray-0.0.52}/test/test_slop_matches.py +0 -0
  47. {searcharray-0.0.51 → searcharray-0.0.52}/test/test_solr.py +0 -0
  48. {searcharray-0.0.51 → searcharray-0.0.52}/test/test_tmdb.py +0 -0
  49. {searcharray-0.0.51 → searcharray-0.0.52}/test/test_utils.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: searcharray
3
- Version: 0.0.51
3
+ Version: 0.0.52
4
4
  Summary: Searchable pandas text extension arrays for prototyping search
5
5
  Home-page: https://github.com/softwaredoug/searcharray
6
6
  Author: Doug Turnbull
@@ -153,13 +153,13 @@ We want you to be able to work with a reasonable dataset (100X-1M docs) relative
153
153
 
154
154
  ### Experimentation, reranking, functionality over scalability
155
155
 
156
- Instead of building for 'big data' our goal is to build for for *small-data*. That is, focus on capabilities and expressiveness of Pandas, over limiting functionality in favor of scalability.
156
+ Instead of building for 'big data' our goal is to build for *small-data*. That is, focus on capabilities and expressiveness of Pandas, over limiting functionality in favor of scalability.
157
157
 
158
158
  To this end, the applications of searcharray will tend to be focused on experimentation and top N candidate reranking. For experimentation, we want any ideas expressed in Pandas to have a somewhat clear path / "contract" in how they'd be implemented in a classical lexical search engine. For reranking, we want to load some top N results from a base system and be able to modify them.
159
159
 
160
160
  ### Make lexical search compatible with the data stack
161
161
 
162
- We know in search, RAG, and other retrieval problems,s [hybrid search](https://www.pinecone.io/learn/hybrid-search-intro/) techniques dominate. Yet often its cast in terms of a giant, weird, big data lexical search engine that looks odd to most data scientists being joined with a vector database. We want lexical search to be more approachable to data scientists and ML engineers building these systems.
162
+ We know in search, RAG, and other retrieval problems [hybrid search](https://www.pinecone.io/learn/hybrid-search-intro/) techniques dominate. Yet often its cast in terms of a giant, weird, big data lexical search engine that looks odd to most data scientists being joined with a vector database. We want lexical search to be more approachable to data scientists and ML engineers building these systems.
163
163
 
164
164
  ## Non-goals
165
165
 
@@ -124,13 +124,13 @@ We want you to be able to work with a reasonable dataset (100X-1M docs) relative
124
124
 
125
125
  ### Experimentation, reranking, functionality over scalability
126
126
 
127
- Instead of building for 'big data' our goal is to build for for *small-data*. That is, focus on capabilities and expressiveness of Pandas, over limiting functionality in favor of scalability.
127
+ Instead of building for 'big data' our goal is to build for *small-data*. That is, focus on capabilities and expressiveness of Pandas, over limiting functionality in favor of scalability.
128
128
 
129
129
  To this end, the applications of searcharray will tend to be focused on experimentation and top N candidate reranking. For experimentation, we want any ideas expressed in Pandas to have a somewhat clear path / "contract" in how they'd be implemented in a classical lexical search engine. For reranking, we want to load some top N results from a base system and be able to modify them.
130
130
 
131
131
  ### Make lexical search compatible with the data stack
132
132
 
133
- We know in search, RAG, and other retrieval problems,s [hybrid search](https://www.pinecone.io/learn/hybrid-search-intro/) techniques dominate. Yet often its cast in terms of a giant, weird, big data lexical search engine that looks odd to most data scientists being joined with a vector database. We want lexical search to be more approachable to data scientists and ML engineers building these systems.
133
+ We know in search, RAG, and other retrieval problems [hybrid search](https://www.pinecone.io/learn/hybrid-search-intro/) techniques dominate. Yet often its cast in terms of a giant, weird, big data lexical search engine that looks odd to most data scientists being joined with a vector database. We want lexical search to be more approachable to data scientists and ML engineers building these systems.
134
134
 
135
135
  ## Non-goals
136
136
 
@@ -70,9 +70,9 @@ def trim_phrase_search(encoded_posns: List[np.ndarray],
70
70
  return encoded_posns
71
71
 
72
72
 
73
- def _compute_phrase_freqs_rhs(encoded_posns: List[np.ndarray],
74
- phrase_freqs: np.ndarray,
75
- trim: bool = True) -> np.ndarray:
73
+ def _compute_phrase_freqs_left_to_right(encoded_posns: List[np.ndarray],
74
+ phrase_freqs: np.ndarray,
75
+ trim: bool = True) -> np.ndarray:
76
76
  """Compute phrase freqs from a set of encoded positions."""
77
77
  if len(encoded_posns) < 2:
78
78
  raise ValueError("phrase must have at least two terms")
@@ -97,9 +97,9 @@ def _compute_phrase_freqs_rhs(encoded_posns: List[np.ndarray],
97
97
  return phrase_freqs
98
98
 
99
99
 
100
- def _compute_phrase_freqs_lhs(encoded_posns: List[np.ndarray],
101
- phrase_freqs: np.ndarray,
102
- trim: bool = True) -> np.ndarray:
100
+ def _compute_phrase_freqs_right_to_left(encoded_posns: List[np.ndarray],
101
+ phrase_freqs: np.ndarray,
102
+ trim: bool = True) -> np.ndarray:
103
103
  """Compute phrase freqs from a set of encoded positions."""
104
104
  if len(encoded_posns) < 2:
105
105
  raise ValueError("phrase must have at least two terms")
@@ -129,16 +129,15 @@ def compute_phrase_freqs(encoded_posns: List[np.ndarray],
129
129
  trim: bool = False) -> np.ndarray:
130
130
  """Compute phrase freqs from a set of encoded positions."""
131
131
  shortest_len_index = min(enumerate(encoded_posns), key=lambda x: len(x[1]))[0]
132
- # Slice LHS / RHS at shortest_len_index
133
132
  if shortest_len_index <= 1:
134
- return _compute_phrase_freqs_lhs(encoded_posns, phrase_freqs, trim=trim)
133
+ return _compute_phrase_freqs_left_to_right(encoded_posns, phrase_freqs, trim=trim)
135
134
  elif shortest_len_index >= len(encoded_posns) - 2:
136
- return _compute_phrase_freqs_rhs(encoded_posns, phrase_freqs, trim=trim)
135
+ return _compute_phrase_freqs_right_to_left(encoded_posns, phrase_freqs, trim=trim)
137
136
  else:
138
137
  # We optimize this case by going middle-out
139
138
  # We can take the min of both directions phrase freqs
140
- lhs = _compute_phrase_freqs_lhs(encoded_posns[:shortest_len_index], phrase_freqs, trim=trim)
141
- rhs = _compute_phrase_freqs_rhs(encoded_posns[shortest_len_index:], phrase_freqs, trim=trim)
139
+ lhs = _compute_phrase_freqs_left_to_right(encoded_posns[:shortest_len_index], phrase_freqs, trim=trim)
140
+ rhs = _compute_phrase_freqs_right_to_left(encoded_posns[shortest_len_index:], phrase_freqs, trim=trim)
142
141
  phrase_freqs = np.minimum(lhs, rhs)
143
142
  return phrase_freqs
144
143