searcharray 0.0.60__tar.gz → 0.0.61__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {searcharray-0.0.60 → searcharray-0.0.61}/PKG-INFO +14 -3
- {searcharray-0.0.60 → searcharray-0.0.61}/README.md +13 -2
- {searcharray-0.0.60 → searcharray-0.0.61}/searcharray/indexing.py +27 -3
- searcharray-0.0.61/searcharray/phrase/memmap_arrays.py +184 -0
- {searcharray-0.0.60 → searcharray-0.0.61}/searcharray/phrase/middle_out.py +10 -16
- {searcharray-0.0.60 → searcharray-0.0.61}/searcharray/postings.py +43 -18
- {searcharray-0.0.60 → searcharray-0.0.61}/searcharray/roaringish/intersect.c +149 -149
- {searcharray-0.0.60 → searcharray-0.0.61}/searcharray/roaringish/merge.c +149 -149
- {searcharray-0.0.60 → searcharray-0.0.61}/searcharray/roaringish/popcount.c +149 -149
- {searcharray-0.0.60 → searcharray-0.0.61}/searcharray/roaringish/roaringish_ops.c +149 -149
- {searcharray-0.0.60 → searcharray-0.0.61}/searcharray/roaringish/search.c +149 -149
- {searcharray-0.0.60 → searcharray-0.0.61}/searcharray/roaringish/spans.c +861 -515
- {searcharray-0.0.60 → searcharray-0.0.61}/searcharray/roaringish/spans.pyx +51 -27
- {searcharray-0.0.60 → searcharray-0.0.61}/searcharray/roaringish/unique.c +149 -149
- {searcharray-0.0.60 → searcharray-0.0.61}/searcharray.egg-info/PKG-INFO +14 -3
- {searcharray-0.0.60 → searcharray-0.0.61}/searcharray.egg-info/SOURCES.txt +1 -0
- {searcharray-0.0.60 → searcharray-0.0.61}/setup.py +1 -1
- {searcharray-0.0.60 → searcharray-0.0.61}/test/test_msmarco.py +109 -2
- {searcharray-0.0.60 → searcharray-0.0.61}/test/test_phrase_matches.py +9 -1
- {searcharray-0.0.60 → searcharray-0.0.61}/test/test_tmdb.py +2 -1
- {searcharray-0.0.60 → searcharray-0.0.61}/test/test_utils.py +5 -1
- {searcharray-0.0.60 → searcharray-0.0.61}/pyproject.toml +0 -0
- {searcharray-0.0.60 → searcharray-0.0.61}/searcharray/__init__.py +0 -0
- {searcharray-0.0.60 → searcharray-0.0.61}/searcharray/phrase/__init__.py +0 -0
- {searcharray-0.0.60 → searcharray-0.0.61}/searcharray/phrase/bigram_freqs.py +0 -0
- {searcharray-0.0.60 → searcharray-0.0.61}/searcharray/phrase/spans.py +0 -0
- {searcharray-0.0.60 → searcharray-0.0.61}/searcharray/roaringish/__init__.py +0 -0
- {searcharray-0.0.60 → searcharray-0.0.61}/searcharray/roaringish/intersect.pyi +0 -0
- {searcharray-0.0.60 → searcharray-0.0.61}/searcharray/roaringish/intersect.pyx +0 -0
- {searcharray-0.0.60 → searcharray-0.0.61}/searcharray/roaringish/merge.pyi +0 -0
- {searcharray-0.0.60 → searcharray-0.0.61}/searcharray/roaringish/merge.pyx +0 -0
- {searcharray-0.0.60 → searcharray-0.0.61}/searcharray/roaringish/popcount.pyi +0 -0
- {searcharray-0.0.60 → searcharray-0.0.61}/searcharray/roaringish/popcount.pyx +0 -0
- {searcharray-0.0.60 → searcharray-0.0.61}/searcharray/roaringish/roaringish.py +0 -0
- {searcharray-0.0.60 → searcharray-0.0.61}/searcharray/roaringish/roaringish_ops.pyx +0 -0
- {searcharray-0.0.60 → searcharray-0.0.61}/searcharray/roaringish/search.pyi +0 -0
- {searcharray-0.0.60 → searcharray-0.0.61}/searcharray/roaringish/search.pyx +0 -0
- {searcharray-0.0.60 → searcharray-0.0.61}/searcharray/roaringish/snp_ops.pxd +0 -0
- {searcharray-0.0.60 → searcharray-0.0.61}/searcharray/roaringish/unique.pyi +0 -0
- {searcharray-0.0.60 → searcharray-0.0.61}/searcharray/roaringish/unique.pyx +0 -0
- {searcharray-0.0.60 → searcharray-0.0.61}/searcharray/similarity.py +0 -0
- {searcharray-0.0.60 → searcharray-0.0.61}/searcharray/solr.py +0 -0
- {searcharray-0.0.60 → searcharray-0.0.61}/searcharray/term_dict.py +0 -0
- {searcharray-0.0.60 → searcharray-0.0.61}/searcharray/utils/__init__.py +0 -0
- {searcharray-0.0.60 → searcharray-0.0.61}/searcharray/utils/bitcount.py +0 -0
- {searcharray-0.0.60 → searcharray-0.0.61}/searcharray/utils/mat_set.py +0 -0
- {searcharray-0.0.60 → searcharray-0.0.61}/searcharray/utils/row_viewable_matrix.py +0 -0
- {searcharray-0.0.60 → searcharray-0.0.61}/searcharray/utils/sort.py +0 -0
- {searcharray-0.0.60 → searcharray-0.0.61}/searcharray.egg-info/dependency_links.txt +0 -0
- {searcharray-0.0.60 → searcharray-0.0.61}/searcharray.egg-info/entry_points.txt +0 -0
- {searcharray-0.0.60 → searcharray-0.0.61}/searcharray.egg-info/requires.txt +0 -0
- {searcharray-0.0.60 → searcharray-0.0.61}/searcharray.egg-info/top_level.txt +0 -0
- {searcharray-0.0.60 → searcharray-0.0.61}/setup.cfg +0 -0
- {searcharray-0.0.60 → searcharray-0.0.61}/test/test_bitcount64.py +0 -0
- {searcharray-0.0.60 → searcharray-0.0.61}/test/test_extension_array.py +0 -0
- {searcharray-0.0.60 → searcharray-0.0.61}/test/test_minmax_posns.py +0 -0
- {searcharray-0.0.60 → searcharray-0.0.61}/test/test_search.py +0 -0
- {searcharray-0.0.60 → searcharray-0.0.61}/test/test_similarity.py +0 -0
- {searcharray-0.0.60 → searcharray-0.0.61}/test/test_slop_matches.py +0 -0
- {searcharray-0.0.60 → searcharray-0.0.61}/test/test_snp_ops.py +0 -0
- {searcharray-0.0.60 → searcharray-0.0.61}/test/test_solr.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: searcharray
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.61
|
|
4
4
|
Summary: Searchable pandas text extension arrays for prototyping search
|
|
5
5
|
Home-page: https://github.com/softwaredoug/searcharray
|
|
6
6
|
Author: Doug Turnbull
|
|
@@ -40,7 +40,7 @@ from searcharray import SearchArray
|
|
|
40
40
|
import pandas as pd
|
|
41
41
|
|
|
42
42
|
df['title_indexed'] = SearchArray.index(df['title'])
|
|
43
|
-
np.sort(df['title_indexed'].array.score('
|
|
43
|
+
np.sort(df['title_indexed'].array.score(['cat', 'in', 'the', 'hat'])) # Search w/ phrase
|
|
44
44
|
|
|
45
45
|
> BM25 scores:
|
|
46
46
|
> array([ 0. , 0. , 0. , ..., 15.84568033, 15.84568033, 15.84568033])
|
|
@@ -50,7 +50,7 @@ np.sort(df['title_indexed'].array.score('Cat'))
|
|
|
50
50
|
|
|
51
51
|
SearchArray is documented in these notebooks:
|
|
52
52
|
|
|
53
|
-
[SearchArray Guide](https://colab.research.google.com/drive/1gmgVz53fDPTJakUHb6Mttevqry7gKHLM) | [SearchArray Offline Experiment](https://colab.research.google.com/drive/1w_Ajn5rHzcISKhdCuPhhVFav3zrvKWn1)
|
|
53
|
+
[SearchArray Guide](https://colab.research.google.com/drive/1gmgVz53fDPTJakUHb6Mttevqry7gKHLM) | [SearchArray Offline Experiment](https://colab.research.google.com/drive/1w_Ajn5rHzcISKhdCuPhhVFav3zrvKWn1) | [About internals](https://softwaredoug.com/blog/2024/01/21/search-array-phrase-algorithm)
|
|
54
54
|
|
|
55
55
|
## Installation
|
|
56
56
|
|
|
@@ -58,6 +58,17 @@ SearchArray is documented in these notebooks:
|
|
|
58
58
|
pip install searcharray
|
|
59
59
|
```
|
|
60
60
|
|
|
61
|
+
## Features
|
|
62
|
+
|
|
63
|
+
* Search w/ terms by passing a string
|
|
64
|
+
* Search w/ a phrase by passing a list[str]
|
|
65
|
+
* Search w/ a [phrase w/ edit-distance](https://lucene.apache.org/core/9_6_0/core/org/apache/lucene/search/PhraseQuery.html) by passing slop=N.
|
|
66
|
+
* Access raw stats arrays in termfreqs / docfreqs methods on the array
|
|
67
|
+
* Bring your own tokenizer. Pass any (`def tokenize(value: str) -> List[str]`) when indexing.
|
|
68
|
+
* Accepts any python function to compute similarity. Here's [one similarity](https://github.com/softwaredoug/searcharray/blob/main/searcharray/similarity.py#L103)
|
|
69
|
+
* Scores the entire dataframe, allowing combination w/ other ranking attributes (recency, popularity, etc) or scores from other fields (ie boolean queries)
|
|
70
|
+
* Implement's Solr's [edismax query parser](https://github.com/softwaredoug/searcharray/blob/main/searcharray/solr.py) for efficient prototyping
|
|
71
|
+
|
|
61
72
|
## Motivation
|
|
62
73
|
|
|
63
74
|
To simplify lexical search in the Python data stack.
|
|
@@ -11,7 +11,7 @@ from searcharray import SearchArray
|
|
|
11
11
|
import pandas as pd
|
|
12
12
|
|
|
13
13
|
df['title_indexed'] = SearchArray.index(df['title'])
|
|
14
|
-
np.sort(df['title_indexed'].array.score('
|
|
14
|
+
np.sort(df['title_indexed'].array.score(['cat', 'in', 'the', 'hat'])) # Search w/ phrase
|
|
15
15
|
|
|
16
16
|
> BM25 scores:
|
|
17
17
|
> array([ 0. , 0. , 0. , ..., 15.84568033, 15.84568033, 15.84568033])
|
|
@@ -21,7 +21,7 @@ np.sort(df['title_indexed'].array.score('Cat'))
|
|
|
21
21
|
|
|
22
22
|
SearchArray is documented in these notebooks:
|
|
23
23
|
|
|
24
|
-
[SearchArray Guide](https://colab.research.google.com/drive/1gmgVz53fDPTJakUHb6Mttevqry7gKHLM) | [SearchArray Offline Experiment](https://colab.research.google.com/drive/1w_Ajn5rHzcISKhdCuPhhVFav3zrvKWn1)
|
|
24
|
+
[SearchArray Guide](https://colab.research.google.com/drive/1gmgVz53fDPTJakUHb6Mttevqry7gKHLM) | [SearchArray Offline Experiment](https://colab.research.google.com/drive/1w_Ajn5rHzcISKhdCuPhhVFav3zrvKWn1) | [About internals](https://softwaredoug.com/blog/2024/01/21/search-array-phrase-algorithm)
|
|
25
25
|
|
|
26
26
|
## Installation
|
|
27
27
|
|
|
@@ -29,6 +29,17 @@ SearchArray is documented in these notebooks:
|
|
|
29
29
|
pip install searcharray
|
|
30
30
|
```
|
|
31
31
|
|
|
32
|
+
## Features
|
|
33
|
+
|
|
34
|
+
* Search w/ terms by passing a string
|
|
35
|
+
* Search w/ a phrase by passing a list[str]
|
|
36
|
+
* Search w/ a [phrase w/ edit-distance](https://lucene.apache.org/core/9_6_0/core/org/apache/lucene/search/PhraseQuery.html) by passing slop=N.
|
|
37
|
+
* Access raw stats arrays in termfreqs / docfreqs methods on the array
|
|
38
|
+
* Bring your own tokenizer. Pass any (`def tokenize(value: str) -> List[str]`) when indexing.
|
|
39
|
+
* Accepts any python function to compute similarity. Here's [one similarity](https://github.com/softwaredoug/searcharray/blob/main/searcharray/similarity.py#L103)
|
|
40
|
+
* Scores the entire dataframe, allowing combination w/ other ranking attributes (recency, popularity, etc) or scores from other fields (ie boolean queries)
|
|
41
|
+
* Implement's Solr's [edismax query parser](https://github.com/softwaredoug/searcharray/blob/main/searcharray/solr.py) for efficient prototyping
|
|
42
|
+
|
|
32
43
|
## Motivation
|
|
33
44
|
|
|
34
45
|
To simplify lexical search in the Python data stack.
|
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
import numpy as np
|
|
2
2
|
import math
|
|
3
3
|
import gc
|
|
4
|
+
import os
|
|
4
5
|
import sys
|
|
5
|
-
from typing import Iterable, List
|
|
6
|
+
from typing import Iterable, List, Optional
|
|
6
7
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
7
8
|
from itertools import islice
|
|
8
9
|
from searcharray.phrase.middle_out import MAX_POSN, PosnBitArrayFromFlatBuilder, PosnBitArrayBuilder, PosnBitArrayAlreadyEncBuilder
|
|
@@ -23,6 +24,17 @@ logger.addHandler(handler)
|
|
|
23
24
|
# logging.basicConfig(level=logging.DEBUG)
|
|
24
25
|
|
|
25
26
|
|
|
27
|
+
def home_directory():
|
|
28
|
+
return os.path.expanduser("~")
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def searcharray_home():
|
|
32
|
+
searcharray_dir = os.path.join(home_directory(), ".searcharray")
|
|
33
|
+
# Ensure exists
|
|
34
|
+
os.makedirs(searcharray_dir, exist_ok=True)
|
|
35
|
+
return searcharray_dir
|
|
36
|
+
|
|
37
|
+
|
|
26
38
|
def _compute_doc_lens(posns: np.ndarray, doc_ids: np.ndarray, num_docs: int) -> np.ndarray:
|
|
27
39
|
"""Given an array of positions, compute the length of each document."""
|
|
28
40
|
doc_lens = np.zeros(num_docs, dtype=np.uint32)
|
|
@@ -49,7 +61,8 @@ def convert_size(size_bytes):
|
|
|
49
61
|
return "%s %s" % (s, size_name[i])
|
|
50
62
|
|
|
51
63
|
|
|
52
|
-
def _gather_tokens(array, tokenizer,
|
|
64
|
+
def _gather_tokens(array, tokenizer,
|
|
65
|
+
term_dict, term_doc, start_doc_id=0, trunc_posn=None):
|
|
53
66
|
all_terms = []
|
|
54
67
|
all_docs = []
|
|
55
68
|
all_posns = []
|
|
@@ -127,6 +140,7 @@ def _tokenize_batch(array, tokenizer, term_dict, batch_size, batch_beg, truncate
|
|
|
127
140
|
if np.any(doc_lens > MAX_POSN):
|
|
128
141
|
raise ValueError(f"Document length exceeds maximum of {MAX_POSN}")
|
|
129
142
|
|
|
143
|
+
logger.info("Batch tokenization complete")
|
|
130
144
|
return batch_beg, term_doc, bit_posns, doc_lens
|
|
131
145
|
|
|
132
146
|
|
|
@@ -166,6 +180,7 @@ def _process_batches(term_doc, batch_size,
|
|
|
166
180
|
logger.error(f"Batch {batch_beg} failed to tokenize")
|
|
167
181
|
raise e
|
|
168
182
|
|
|
183
|
+
logger.info(f"(main thread) Processing {len(batch_results)} batch results")
|
|
169
184
|
for result in batch_results:
|
|
170
185
|
assert result is not None
|
|
171
186
|
batch_beg, batch_term_doc, batch_bit_posns, batch_doc_lens = result
|
|
@@ -173,13 +188,16 @@ def _process_batches(term_doc, batch_size,
|
|
|
173
188
|
if bit_posns is None:
|
|
174
189
|
bit_posns = batch_bit_posns
|
|
175
190
|
else:
|
|
191
|
+
logger.info("(main thread) Concatenating bit positions")
|
|
176
192
|
bit_posns.concat(batch_bit_posns)
|
|
193
|
+
logger.info("(main thread) Concatenated bit positions... Done")
|
|
177
194
|
|
|
178
195
|
doc_lens.append(batch_doc_lens)
|
|
179
196
|
return bit_posns
|
|
180
197
|
|
|
181
198
|
|
|
182
199
|
def build_index_from_tokenizer(array: Iterable, tokenizer, batch_size=10000,
|
|
200
|
+
data_dir: Optional[str] = None,
|
|
183
201
|
truncate=False, workers=4):
|
|
184
202
|
"""Build index directly from tokenizing docs (array of string)."""
|
|
185
203
|
term_dict = TermDict()
|
|
@@ -187,7 +205,7 @@ def build_index_from_tokenizer(array: Iterable, tokenizer, batch_size=10000,
|
|
|
187
205
|
doc_lens: List[np.ndarray] = []
|
|
188
206
|
bit_posns = None
|
|
189
207
|
|
|
190
|
-
logger.info("Indexing begins")
|
|
208
|
+
logger.info(f"Indexing begins w/ {workers} workers")
|
|
191
209
|
futures = []
|
|
192
210
|
last_batch_beg_processed = 0
|
|
193
211
|
|
|
@@ -226,6 +244,12 @@ def build_index_from_tokenizer(array: Iterable, tokenizer, batch_size=10000,
|
|
|
226
244
|
|
|
227
245
|
term_doc_built = RowViewableMatrix(term_doc.build())
|
|
228
246
|
logger.info("Indexing from tokenization complete")
|
|
247
|
+
assert bit_posns is not None
|
|
248
|
+
# if data_dir is None:
|
|
249
|
+
# data_dir = searcharray_home()
|
|
250
|
+
if data_dir is not None:
|
|
251
|
+
logger.info(f"Memmapping bit positions to {data_dir}")
|
|
252
|
+
bit_posns.memmap(data_dir)
|
|
229
253
|
return term_doc_built, bit_posns, term_dict, avg_doc_length, np.array(doc_lens)
|
|
230
254
|
|
|
231
255
|
|
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
import numpy as np
|
|
2
|
+
import os
|
|
3
|
+
from typing import Optional, List
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def create_filename(data_dir: str):
|
|
7
|
+
"""Create a filename for the memory-mapped array"""
|
|
8
|
+
# Ls path and get the number of files
|
|
9
|
+
files = os.listdir(data_dir)
|
|
10
|
+
num_files = len(files)
|
|
11
|
+
return os.path.join(data_dir, f'{num_files}')
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class ArrayDict:
|
|
15
|
+
"""An array of ints to np.array as contiguous memory"""
|
|
16
|
+
|
|
17
|
+
def __init__(self):
|
|
18
|
+
self.dtype = np.uint64
|
|
19
|
+
self.data = np.ndarray(0, dtype=self.dtype)
|
|
20
|
+
self.metadata = {}
|
|
21
|
+
|
|
22
|
+
@staticmethod
|
|
23
|
+
def from_arrays(arrays: List[np.ndarray]):
|
|
24
|
+
arr = ArrayDict()
|
|
25
|
+
metadata = {}
|
|
26
|
+
total_len = 0
|
|
27
|
+
for i, array in enumerate(arrays):
|
|
28
|
+
metadata[i] = {'offset': total_len, 'length': array.size}
|
|
29
|
+
total_len += array.size
|
|
30
|
+
data = np.concatenate(arrays)
|
|
31
|
+
arr.data = data
|
|
32
|
+
arr.metadata = metadata
|
|
33
|
+
return arr
|
|
34
|
+
|
|
35
|
+
@staticmethod
|
|
36
|
+
def from_array_with_boundaries(data: np.ndarray,
|
|
37
|
+
ids: np.ndarray,
|
|
38
|
+
boundaries: np.ndarray):
|
|
39
|
+
metadata = {}
|
|
40
|
+
for idx, (beg, end) in enumerate(zip(boundaries[:-1], boundaries[1:])):
|
|
41
|
+
offset = int(beg)
|
|
42
|
+
length = int(end - beg)
|
|
43
|
+
actual_id = ids[idx]
|
|
44
|
+
metadata[int(actual_id)] = {'offset': offset, 'length': length}
|
|
45
|
+
arr = ArrayDict()
|
|
46
|
+
arr.data = data
|
|
47
|
+
arr.metadata = metadata
|
|
48
|
+
return arr
|
|
49
|
+
|
|
50
|
+
@staticmethod
|
|
51
|
+
def concat(lhs: 'ArrayDict', rhs: 'ArrayDict', sort=True):
|
|
52
|
+
metadata = {}
|
|
53
|
+
offset = 0
|
|
54
|
+
for key, value in lhs.metadata.items():
|
|
55
|
+
metadata[key] = {'offset': offset, 'length': value['length']}
|
|
56
|
+
offset += value['length']
|
|
57
|
+
for key, value in rhs.metadata.items():
|
|
58
|
+
metadata[key] = {'offset': offset, 'length': value['length']}
|
|
59
|
+
offset += value['length']
|
|
60
|
+
data = np.concatenate((lhs.data, rhs.data))
|
|
61
|
+
if sort:
|
|
62
|
+
keys = sorted(metadata.keys())
|
|
63
|
+
metadata = {key: metadata[key] for key in keys}
|
|
64
|
+
arr = ArrayDict()
|
|
65
|
+
arr.data = data
|
|
66
|
+
arr.metadata = metadata
|
|
67
|
+
return arr
|
|
68
|
+
|
|
69
|
+
def __getitem__(self, key):
|
|
70
|
+
key = int(key)
|
|
71
|
+
if key in self.metadata:
|
|
72
|
+
offset = self.metadata[key]['offset']
|
|
73
|
+
length = self.metadata[key]['length']
|
|
74
|
+
return self.data[offset:offset + length]
|
|
75
|
+
else:
|
|
76
|
+
raise KeyError(f'Key {key} not found.')
|
|
77
|
+
|
|
78
|
+
def __setitem__(self, key, value):
|
|
79
|
+
key = int(key)
|
|
80
|
+
if value.dtype != self.dtype:
|
|
81
|
+
raise ValueError(f'Value must be of type {self.dtype}')
|
|
82
|
+
offset = self.data.size
|
|
83
|
+
length = value.size
|
|
84
|
+
self.metadata[key] = {'offset': offset, 'length': length}
|
|
85
|
+
self.data = np.append(self.data, value)
|
|
86
|
+
|
|
87
|
+
def __delitem__(self, key):
|
|
88
|
+
key = int(key)
|
|
89
|
+
if key in self.metadata:
|
|
90
|
+
del self.metadata[key]
|
|
91
|
+
else:
|
|
92
|
+
raise KeyError(f'Key {key} not found.')
|
|
93
|
+
|
|
94
|
+
def __len__(self):
|
|
95
|
+
return len(self.metadata)
|
|
96
|
+
|
|
97
|
+
def __iter__(self):
|
|
98
|
+
yield from self.metadata
|
|
99
|
+
|
|
100
|
+
def __contains__(self, key):
|
|
101
|
+
return key in self.metadata
|
|
102
|
+
|
|
103
|
+
def items(self):
|
|
104
|
+
for key, value in self.metadata.items():
|
|
105
|
+
yield key, self.data[value['offset']:value['offset'] + value['length']]
|
|
106
|
+
|
|
107
|
+
def keys(self):
|
|
108
|
+
return self.metadata.keys()
|
|
109
|
+
|
|
110
|
+
def compact(self):
|
|
111
|
+
"""Compact the data array"""
|
|
112
|
+
new_data = np.ndarray(sum([value['length'] for value in self.metadata.values()]), dtype=self.dtype)
|
|
113
|
+
offset = 0
|
|
114
|
+
for value in self.metadata.values():
|
|
115
|
+
new_data[offset:offset + value['length']] = self.data[value['offset']:value['offset'] + value['length']]
|
|
116
|
+
value['offset'] = offset
|
|
117
|
+
offset += value['length']
|
|
118
|
+
self.data = new_data
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
class MemoryMappedArrays:
|
|
122
|
+
def __init__(self,
|
|
123
|
+
data_dir: str,
|
|
124
|
+
arrays: Optional[ArrayDict] = None):
|
|
125
|
+
root_filename = create_filename(data_dir)
|
|
126
|
+
self.filename = root_filename + '.dat'
|
|
127
|
+
self.fp = None
|
|
128
|
+
if arrays is not None:
|
|
129
|
+
self.arrays = arrays
|
|
130
|
+
self._initialize_file()
|
|
131
|
+
else:
|
|
132
|
+
self._load_arrays()
|
|
133
|
+
|
|
134
|
+
def _initialize_file(self):
|
|
135
|
+
"""Memmap underlying array in ArrayDict."""
|
|
136
|
+
with open(self.filename, 'wb') as f:
|
|
137
|
+
self.arrays.data.tofile(f)
|
|
138
|
+
|
|
139
|
+
def _load_arrays(self):
|
|
140
|
+
"""Memmap underlying array in ArrayDict from self.filename."""
|
|
141
|
+
self.arrays.data = np.memmap(self.filename, dtype=self.arrays.dtype, mode='r+')
|
|
142
|
+
|
|
143
|
+
def __getitem__(self, key):
|
|
144
|
+
# Use metadatat to get the offset and length of the array
|
|
145
|
+
return self.arrays[key]
|
|
146
|
+
|
|
147
|
+
def __setitem__(self, key, value):
|
|
148
|
+
"""Append to file and update metadata. Not fast."""
|
|
149
|
+
self.arrays[key] = value
|
|
150
|
+
self._save_metadata()
|
|
151
|
+
self.fp = np.memmap(self.filename, dtype=self.dtype, mode='r+', shape=(self.fp.size + value.size,))
|
|
152
|
+
|
|
153
|
+
def __delitem__(self, key):
|
|
154
|
+
length = self.arrays[key].size
|
|
155
|
+
del self.arrays[key]
|
|
156
|
+
self.fp = np.memmap(self.filename, dtype=self.dtype, mode='r+', shape=(self.fp.size - length,))
|
|
157
|
+
|
|
158
|
+
def __len__(self):
|
|
159
|
+
return len(self.arrays)
|
|
160
|
+
|
|
161
|
+
def __iter__(self):
|
|
162
|
+
yield from self.arrays
|
|
163
|
+
|
|
164
|
+
def __contains__(self, key):
|
|
165
|
+
return key in self.arrays
|
|
166
|
+
|
|
167
|
+
def items(self):
|
|
168
|
+
yield from self.arrays.items()
|
|
169
|
+
|
|
170
|
+
def keys(self):
|
|
171
|
+
return self.arrays.keys()
|
|
172
|
+
|
|
173
|
+
def __getstate__(self):
|
|
174
|
+
"""Return arrays without the numpy array."""
|
|
175
|
+
# Create arrays without data
|
|
176
|
+
arrays = ArrayDict()
|
|
177
|
+
arrays.metadata = self.arrays.metadata
|
|
178
|
+
return {'arrays': arrays,
|
|
179
|
+
'filename': self.filename}
|
|
180
|
+
|
|
181
|
+
def __setstate__(self, state):
|
|
182
|
+
self.filename = state['filename']
|
|
183
|
+
self.arrays = state['arrays']
|
|
184
|
+
self._load_arrays()
|
|
@@ -9,6 +9,7 @@ import numpy as np
|
|
|
9
9
|
from copy import deepcopy
|
|
10
10
|
from typing import List, Tuple, Dict, Union, cast, Optional
|
|
11
11
|
from searcharray.roaringish import RoaringishEncoder, convert_keys, merge
|
|
12
|
+
from searcharray.phrase.memmap_arrays import MemoryMappedArrays, ArrayDict
|
|
12
13
|
from searcharray.phrase.bigram_freqs import bigram_freqs, Continuation
|
|
13
14
|
from searcharray.phrase.spans import span_search
|
|
14
15
|
import numbers
|
|
@@ -171,11 +172,9 @@ class PosnBitArrayFromFlatBuilder:
|
|
|
171
172
|
payload=self.flat_array[2].view(np.uint64))
|
|
172
173
|
term_ids = self.flat_array[0][term_boundaries[:-1]]
|
|
173
174
|
|
|
174
|
-
encoded_term_posns =
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
encoded_term_posns[term_ids[into_terms]] = sliced
|
|
178
|
-
|
|
175
|
+
encoded_term_posns = ArrayDict.from_array_with_boundaries(encoded,
|
|
176
|
+
boundaries=enc_term_boundaries,
|
|
177
|
+
ids=term_ids)
|
|
179
178
|
largest_doc_id_with_term = np.max(self.flat_array[1])
|
|
180
179
|
assert self.max_doc_id >= largest_doc_id_with_term
|
|
181
180
|
return PosnBitArray(encoded_term_posns, self.max_doc_id)
|
|
@@ -197,7 +196,7 @@ class PosnBitArrayBuilder:
|
|
|
197
196
|
self.max_doc_id = max(self.max_doc_id, doc_id)
|
|
198
197
|
|
|
199
198
|
def build(self, check=False):
|
|
200
|
-
encoded_term_posns =
|
|
199
|
+
encoded_term_posns = ArrayDict()
|
|
201
200
|
for term_id, posns in self.term_posns.items():
|
|
202
201
|
if len(posns) == 0:
|
|
203
202
|
posns = np.asarray([], dtype=np.uint32).flatten()
|
|
@@ -294,12 +293,15 @@ class FilteredPosns(abc.Mapping):
|
|
|
294
293
|
|
|
295
294
|
class PosnBitArray:
|
|
296
295
|
|
|
297
|
-
def __init__(self, encoded_term_posns, max_doc_id: int):
|
|
296
|
+
def __init__(self, encoded_term_posns: Union[ArrayDict, FilteredPosns], max_doc_id: int):
|
|
298
297
|
self.encoded_term_posns = encoded_term_posns
|
|
299
298
|
self.max_doc_id = max_doc_id
|
|
300
299
|
self.docfreq_cache : Dict[int, np.uint64] = {}
|
|
301
300
|
self.termfreq_cache : Dict[int, Tuple[np.ndarray, np.ndarray]] = {}
|
|
302
301
|
|
|
302
|
+
def memmap(self, data_dir):
|
|
303
|
+
self.encoded_term_posns = MemoryMappedArrays(data_dir, self.encoded_term_posns)
|
|
304
|
+
|
|
303
305
|
def warm(self):
|
|
304
306
|
"""Warm tf / df cache of most common terms."""
|
|
305
307
|
for term_id, encoded in self.encoded_term_posns.items():
|
|
@@ -333,15 +335,7 @@ class PosnBitArray:
|
|
|
333
335
|
Assumes other's doc ids are not overlapping with self's doc ids.
|
|
334
336
|
"""
|
|
335
337
|
# Shared terms
|
|
336
|
-
|
|
337
|
-
for term_id in shared_terms:
|
|
338
|
-
# Append then sort
|
|
339
|
-
self.encoded_term_posns[term_id] = np.concatenate([self.encoded_term_posns[term_id], other.encoded_term_posns[term_id]])
|
|
340
|
-
self.encoded_term_posns[term_id].sort()
|
|
341
|
-
|
|
342
|
-
only_other_terms = set(other.encoded_term_posns.keys()).difference(set(self.encoded_term_posns.keys()))
|
|
343
|
-
for term_id in only_other_terms:
|
|
344
|
-
self.encoded_term_posns[term_id] = other.encoded_term_posns[term_id]
|
|
338
|
+
self.encoded_term_posns = ArrayDict.concat(self.encoded_term_posns, other.encoded_term_posns)
|
|
345
339
|
self.max_doc_id = max(self.max_doc_id, other.max_doc_id)
|
|
346
340
|
self.clear_cache()
|
|
347
341
|
|
|
@@ -30,6 +30,29 @@ logger.addHandler(handler)
|
|
|
30
30
|
logger.setLevel(logging.ERROR)
|
|
31
31
|
|
|
32
32
|
|
|
33
|
+
def bytes_to_human_readable(num_bytes):
|
|
34
|
+
"""
|
|
35
|
+
Convert a number of bytes into a human-readable format (KB, MB, GB, etc.)
|
|
36
|
+
|
|
37
|
+
:param num_bytes: Number of bytes to be converted
|
|
38
|
+
:type num_bytes: int or float
|
|
39
|
+
:return: Human-readable string
|
|
40
|
+
:rtype: str
|
|
41
|
+
"""
|
|
42
|
+
# Define the suffixes for byte conversion
|
|
43
|
+
suffixes = ['B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB']
|
|
44
|
+
|
|
45
|
+
if num_bytes == 0:
|
|
46
|
+
return "0B"
|
|
47
|
+
|
|
48
|
+
i = 0
|
|
49
|
+
while num_bytes >= 1024 and i < len(suffixes) - 1:
|
|
50
|
+
num_bytes /= 1024.
|
|
51
|
+
i += 1
|
|
52
|
+
|
|
53
|
+
return f"{num_bytes:.2f} {suffixes[i]}"
|
|
54
|
+
|
|
55
|
+
|
|
33
56
|
class Terms:
|
|
34
57
|
"""An indexed search doc - a single bag of tokenized words and positions."""
|
|
35
58
|
|
|
@@ -229,6 +252,7 @@ class SearchArray(ExtensionArray):
|
|
|
229
252
|
batch_size=100000,
|
|
230
253
|
avoid_copies=True,
|
|
231
254
|
workers=4,
|
|
255
|
+
data_dir: Optional[str] = None,
|
|
232
256
|
autowarm=True) -> 'SearchArray':
|
|
233
257
|
"""Index an array of strings into SearchArray using tokenizer.
|
|
234
258
|
|
|
@@ -257,6 +281,7 @@ class SearchArray(ExtensionArray):
|
|
|
257
281
|
term_mat, posns, term_dict, avg_doc_length, doc_lens =\
|
|
258
282
|
build_index_from_tokenizer(array, tokenizer, batch_size=batch_size,
|
|
259
283
|
truncate=truncate,
|
|
284
|
+
data_dir=data_dir,
|
|
260
285
|
workers=workers)
|
|
261
286
|
|
|
262
287
|
if autowarm:
|
|
@@ -539,7 +564,7 @@ class SearchArray(ExtensionArray):
|
|
|
539
564
|
else:
|
|
540
565
|
raise TypeError("Expected a string or list of strings for phrases")
|
|
541
566
|
|
|
542
|
-
def memory_report(self):
|
|
567
|
+
def memory_report(self, N=1000):
|
|
543
568
|
"""Return a string with memory usage information."""
|
|
544
569
|
term_mat_bytes = self.term_mat.nbytes
|
|
545
570
|
posns_bytes = self.posns.nbytes
|
|
@@ -547,32 +572,32 @@ class SearchArray(ExtensionArray):
|
|
|
547
572
|
num_terms = len(self.term_dict)
|
|
548
573
|
top_10_terms_size = []
|
|
549
574
|
|
|
550
|
-
for i in range(
|
|
575
|
+
for i in range(N):
|
|
551
576
|
term_len = len(self.posns.encoded_term_posns[i])
|
|
552
577
|
term = self.term_dict.get_term(i)
|
|
553
|
-
|
|
578
|
+
term_bytes = term_len * 8
|
|
579
|
+
top_10_terms_size.append((term, term_bytes))
|
|
580
|
+
|
|
581
|
+
top_10_terms_size = sorted(top_10_terms_size, key=lambda x: x[1], reverse=True)
|
|
554
582
|
|
|
555
|
-
|
|
583
|
+
report = f"""
|
|
556
584
|
SearchArray Memory Report
|
|
557
585
|
-------------------------
|
|
558
586
|
Number of Terms: {num_terms}
|
|
559
587
|
-------------------------
|
|
560
|
-
Term Matrix: {term_mat_bytes}
|
|
561
|
-
Positions: {posns_bytes}
|
|
562
|
-
Term Dictionary: {term_dict_bytes}
|
|
563
|
-
|
|
564
|
-
Term 0: {top_10_terms_size[0]}
|
|
565
|
-
Term 1: {top_10_terms_size[1]}
|
|
566
|
-
Term 2: {top_10_terms_size[2]}
|
|
567
|
-
Term 3: {top_10_terms_size[3]}
|
|
568
|
-
Term 4: {top_10_terms_size[4]}
|
|
569
|
-
Term 5: {top_10_terms_size[5]}
|
|
570
|
-
Term 6: {top_10_terms_size[6]}
|
|
571
|
-
Term 7: {top_10_terms_size[7]}
|
|
572
|
-
Term 8: {top_10_terms_size[8]}
|
|
573
|
-
Term 9: {top_10_terms_size[9]}
|
|
588
|
+
Term Matrix: {bytes_to_human_readable(term_mat_bytes)}
|
|
589
|
+
Positions: {bytes_to_human_readable(posns_bytes)}
|
|
590
|
+
Term Dictionary: {bytes_to_human_readable(term_dict_bytes)}
|
|
591
|
+
|
|
574
592
|
"""
|
|
575
593
|
|
|
594
|
+
cum_sum = 0
|
|
595
|
+
for i in range(N):
|
|
596
|
+
term, term_bytes = top_10_terms_size[i]
|
|
597
|
+
cum_sum += term_bytes
|
|
598
|
+
report += f" Term {i}: {term} - {bytes_to_human_readable(term_bytes)} - Cumulative: {bytes_to_human_readable(cum_sum)}\n"
|
|
599
|
+
return report
|
|
600
|
+
|
|
576
601
|
# ***********************************************************
|
|
577
602
|
# Search API
|
|
578
603
|
# ***********************************************************
|