searcharray 0.0.61__tar.gz → 0.0.63__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {searcharray-0.0.61 → searcharray-0.0.63}/PKG-INFO +2 -1
- {searcharray-0.0.61 → searcharray-0.0.63}/README.md +1 -0
- {searcharray-0.0.61 → searcharray-0.0.63}/searcharray/indexing.py +1 -1
- {searcharray-0.0.61 → searcharray-0.0.63}/searcharray/phrase/memmap_arrays.py +32 -14
- {searcharray-0.0.61 → searcharray-0.0.63}/searcharray/phrase/middle_out.py +2 -0
- {searcharray-0.0.61 → searcharray-0.0.63}/searcharray/roaringish/intersect.c +149 -149
- {searcharray-0.0.61 → searcharray-0.0.63}/searcharray/roaringish/merge.c +149 -149
- {searcharray-0.0.61 → searcharray-0.0.63}/searcharray/roaringish/popcount.c +149 -149
- {searcharray-0.0.61 → searcharray-0.0.63}/searcharray/roaringish/roaringish_ops.c +149 -149
- {searcharray-0.0.61 → searcharray-0.0.63}/searcharray/roaringish/search.c +149 -149
- {searcharray-0.0.61 → searcharray-0.0.63}/searcharray/roaringish/spans.c +243 -241
- {searcharray-0.0.61 → searcharray-0.0.63}/searcharray/roaringish/unique.c +149 -149
- {searcharray-0.0.61 → searcharray-0.0.63}/searcharray/similarity.py +4 -1
- {searcharray-0.0.61 → searcharray-0.0.63}/searcharray.egg-info/PKG-INFO +2 -1
- {searcharray-0.0.61 → searcharray-0.0.63}/setup.py +1 -1
- {searcharray-0.0.61 → searcharray-0.0.63}/test/test_search.py +11 -0
- {searcharray-0.0.61 → searcharray-0.0.63}/test/test_tmdb.py +53 -5
- {searcharray-0.0.61 → searcharray-0.0.63}/pyproject.toml +0 -0
- {searcharray-0.0.61 → searcharray-0.0.63}/searcharray/__init__.py +0 -0
- {searcharray-0.0.61 → searcharray-0.0.63}/searcharray/phrase/__init__.py +0 -0
- {searcharray-0.0.61 → searcharray-0.0.63}/searcharray/phrase/bigram_freqs.py +0 -0
- {searcharray-0.0.61 → searcharray-0.0.63}/searcharray/phrase/spans.py +0 -0
- {searcharray-0.0.61 → searcharray-0.0.63}/searcharray/postings.py +0 -0
- {searcharray-0.0.61 → searcharray-0.0.63}/searcharray/roaringish/__init__.py +0 -0
- {searcharray-0.0.61 → searcharray-0.0.63}/searcharray/roaringish/intersect.pyi +0 -0
- {searcharray-0.0.61 → searcharray-0.0.63}/searcharray/roaringish/intersect.pyx +0 -0
- {searcharray-0.0.61 → searcharray-0.0.63}/searcharray/roaringish/merge.pyi +0 -0
- {searcharray-0.0.61 → searcharray-0.0.63}/searcharray/roaringish/merge.pyx +0 -0
- {searcharray-0.0.61 → searcharray-0.0.63}/searcharray/roaringish/popcount.pyi +0 -0
- {searcharray-0.0.61 → searcharray-0.0.63}/searcharray/roaringish/popcount.pyx +0 -0
- {searcharray-0.0.61 → searcharray-0.0.63}/searcharray/roaringish/roaringish.py +0 -0
- {searcharray-0.0.61 → searcharray-0.0.63}/searcharray/roaringish/roaringish_ops.pyx +0 -0
- {searcharray-0.0.61 → searcharray-0.0.63}/searcharray/roaringish/search.pyi +0 -0
- {searcharray-0.0.61 → searcharray-0.0.63}/searcharray/roaringish/search.pyx +0 -0
- {searcharray-0.0.61 → searcharray-0.0.63}/searcharray/roaringish/snp_ops.pxd +0 -0
- {searcharray-0.0.61 → searcharray-0.0.63}/searcharray/roaringish/spans.pyx +0 -0
- {searcharray-0.0.61 → searcharray-0.0.63}/searcharray/roaringish/unique.pyi +0 -0
- {searcharray-0.0.61 → searcharray-0.0.63}/searcharray/roaringish/unique.pyx +0 -0
- {searcharray-0.0.61 → searcharray-0.0.63}/searcharray/solr.py +0 -0
- {searcharray-0.0.61 → searcharray-0.0.63}/searcharray/term_dict.py +0 -0
- {searcharray-0.0.61 → searcharray-0.0.63}/searcharray/utils/__init__.py +0 -0
- {searcharray-0.0.61 → searcharray-0.0.63}/searcharray/utils/bitcount.py +0 -0
- {searcharray-0.0.61 → searcharray-0.0.63}/searcharray/utils/mat_set.py +0 -0
- {searcharray-0.0.61 → searcharray-0.0.63}/searcharray/utils/row_viewable_matrix.py +0 -0
- {searcharray-0.0.61 → searcharray-0.0.63}/searcharray/utils/sort.py +0 -0
- {searcharray-0.0.61 → searcharray-0.0.63}/searcharray.egg-info/SOURCES.txt +0 -0
- {searcharray-0.0.61 → searcharray-0.0.63}/searcharray.egg-info/dependency_links.txt +0 -0
- {searcharray-0.0.61 → searcharray-0.0.63}/searcharray.egg-info/entry_points.txt +0 -0
- {searcharray-0.0.61 → searcharray-0.0.63}/searcharray.egg-info/requires.txt +0 -0
- {searcharray-0.0.61 → searcharray-0.0.63}/searcharray.egg-info/top_level.txt +0 -0
- {searcharray-0.0.61 → searcharray-0.0.63}/setup.cfg +0 -0
- {searcharray-0.0.61 → searcharray-0.0.63}/test/test_bitcount64.py +0 -0
- {searcharray-0.0.61 → searcharray-0.0.63}/test/test_extension_array.py +0 -0
- {searcharray-0.0.61 → searcharray-0.0.63}/test/test_minmax_posns.py +0 -0
- {searcharray-0.0.61 → searcharray-0.0.63}/test/test_msmarco.py +0 -0
- {searcharray-0.0.61 → searcharray-0.0.63}/test/test_phrase_matches.py +0 -0
- {searcharray-0.0.61 → searcharray-0.0.63}/test/test_similarity.py +0 -0
- {searcharray-0.0.61 → searcharray-0.0.63}/test/test_slop_matches.py +0 -0
- {searcharray-0.0.61 → searcharray-0.0.63}/test/test_snp_ops.py +0 -0
- {searcharray-0.0.61 → searcharray-0.0.63}/test/test_solr.py +0 -0
- {searcharray-0.0.61 → searcharray-0.0.63}/test/test_utils.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: searcharray
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.63
|
|
4
4
|
Summary: Searchable pandas text extension arrays for prototyping search
|
|
5
5
|
Home-page: https://github.com/softwaredoug/searcharray
|
|
6
6
|
Author: Doug Turnbull
|
|
@@ -65,6 +65,7 @@ pip install searcharray
|
|
|
65
65
|
* Search w/ a [phrase w/ edit-distance](https://lucene.apache.org/core/9_6_0/core/org/apache/lucene/search/PhraseQuery.html) by passing slop=N.
|
|
66
66
|
* Access raw stats arrays in termfreqs / docfreqs methods on the array
|
|
67
67
|
* Bring your own tokenizer. Pass any (`def tokenize(value: str) -> List[str]`) when indexing.
|
|
68
|
+
* Memory map by passing `data_dir` to index for memory mapped index
|
|
68
69
|
* Accepts any python function to compute similarity. Here's [one similarity](https://github.com/softwaredoug/searcharray/blob/main/searcharray/similarity.py#L103)
|
|
69
70
|
* Scores the entire dataframe, allowing combination w/ other ranking attributes (recency, popularity, etc) or scores from other fields (ie boolean queries)
|
|
70
71
|
* Implement's Solr's [edismax query parser](https://github.com/softwaredoug/searcharray/blob/main/searcharray/solr.py) for efficient prototyping
|
|
@@ -36,6 +36,7 @@ pip install searcharray
|
|
|
36
36
|
* Search w/ a [phrase w/ edit-distance](https://lucene.apache.org/core/9_6_0/core/org/apache/lucene/search/PhraseQuery.html) by passing slop=N.
|
|
37
37
|
* Access raw stats arrays in termfreqs / docfreqs methods on the array
|
|
38
38
|
* Bring your own tokenizer. Pass any (`def tokenize(value: str) -> List[str]`) when indexing.
|
|
39
|
+
* Memory map by passing `data_dir` to index for memory mapped index
|
|
39
40
|
* Accepts any python function to compute similarity. Here's [one similarity](https://github.com/softwaredoug/searcharray/blob/main/searcharray/similarity.py#L103)
|
|
40
41
|
* Scores the entire dataframe, allowing combination w/ other ranking attributes (recency, popularity, etc) or scores from other fields (ie boolean queries)
|
|
41
42
|
* Implement's Solr's [edismax query parser](https://github.com/softwaredoug/searcharray/blob/main/searcharray/solr.py) for efficient prototyping
|
|
@@ -46,7 +46,7 @@ def _compute_doc_lens(posns: np.ndarray, doc_ids: np.ndarray, num_docs: int) ->
|
|
|
46
46
|
non_empty_doc_ids = doc_ids[non_empty_idxs]
|
|
47
47
|
non_empty_doc_lens = non_empty_doc_lens[non_empty_idxs]
|
|
48
48
|
doc_lens[non_empty_doc_ids] = non_empty_doc_lens
|
|
49
|
-
if doc_ids[-1] not in non_empty_doc_ids:
|
|
49
|
+
if len(doc_ids) > 0 and doc_ids[-1] not in non_empty_doc_ids:
|
|
50
50
|
doc_lens[doc_ids[-1]] = posns[-1] + 1
|
|
51
51
|
return doc_lens
|
|
52
52
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import numpy as np
|
|
2
2
|
import os
|
|
3
|
-
from typing import Optional, List
|
|
3
|
+
from typing import Optional, List, Dict
|
|
4
4
|
|
|
5
5
|
|
|
6
6
|
def create_filename(data_dir: str):
|
|
@@ -49,21 +49,39 @@ class ArrayDict:
|
|
|
49
49
|
|
|
50
50
|
@staticmethod
|
|
51
51
|
def concat(lhs: 'ArrayDict', rhs: 'ArrayDict', sort=True):
|
|
52
|
-
metadata = {}
|
|
53
|
-
offset = 0
|
|
54
|
-
for key, value in lhs.metadata.items():
|
|
55
|
-
metadata[key] = {'offset': offset, 'length': value['length']}
|
|
56
|
-
offset += value['length']
|
|
57
|
-
for key, value in rhs.metadata.items():
|
|
58
|
-
metadata[key] = {'offset': offset, 'length': value['length']}
|
|
59
|
-
offset += value['length']
|
|
60
|
-
data = np.concatenate((lhs.data, rhs.data))
|
|
61
|
-
if sort:
|
|
62
|
-
keys = sorted(metadata.keys())
|
|
63
|
-
metadata = {key: metadata[key] for key in keys}
|
|
64
52
|
arr = ArrayDict()
|
|
65
|
-
|
|
53
|
+
metadata: Dict[int, Dict[str, int]] = {}
|
|
54
|
+
lst_of_arrays: List[np.ndarray] = []
|
|
55
|
+
|
|
56
|
+
curr_offset = 0
|
|
57
|
+
last_offset = 0
|
|
58
|
+
|
|
59
|
+
fetched_keys = set()
|
|
60
|
+
for key, value in lhs.items():
|
|
61
|
+
fetched_keys.add(key)
|
|
62
|
+
all_for_key = [value]
|
|
63
|
+
curr_offset += value.size
|
|
64
|
+
if key in rhs.metadata:
|
|
65
|
+
rhs_value = rhs[key]
|
|
66
|
+
all_for_key.append(rhs_value)
|
|
67
|
+
curr_offset += rhs_value.size
|
|
68
|
+
metadata[key] = {'offset': last_offset, 'length': curr_offset - last_offset}
|
|
69
|
+
last_offset = curr_offset
|
|
70
|
+
lst_of_arrays.append(np.concatenate(all_for_key))
|
|
71
|
+
|
|
72
|
+
for key, value in rhs.items():
|
|
73
|
+
if key not in fetched_keys:
|
|
74
|
+
curr_offset += value.size
|
|
75
|
+
lst_of_arrays.append(value)
|
|
76
|
+
metadata[key] = {'offset': last_offset, 'length': curr_offset - last_offset}
|
|
77
|
+
last_offset = curr_offset
|
|
78
|
+
|
|
66
79
|
arr.metadata = metadata
|
|
80
|
+
for curr_arr in lst_of_arrays:
|
|
81
|
+
if sort:
|
|
82
|
+
curr_arr.sort()
|
|
83
|
+
|
|
84
|
+
arr.data = np.concatenate(lst_of_arrays)
|
|
67
85
|
return arr
|
|
68
86
|
|
|
69
87
|
def __getitem__(self, key):
|
|
@@ -170,6 +170,8 @@ class PosnBitArrayFromFlatBuilder:
|
|
|
170
170
|
encoded, enc_term_boundaries = encoder.encode(keys=self.flat_array[1].view(np.uint64),
|
|
171
171
|
boundaries=term_boundaries[:-1],
|
|
172
172
|
payload=self.flat_array[2].view(np.uint64))
|
|
173
|
+
if len(encoded) == 0:
|
|
174
|
+
return PosnBitArray({}, self.max_doc_id)
|
|
173
175
|
term_ids = self.flat_array[0][term_boundaries[:-1]]
|
|
174
176
|
|
|
175
177
|
encoded_term_posns = ArrayDict.from_array_with_boundaries(encoded,
|