searcharray 0.0.60__tar.gz → 0.0.61__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. {searcharray-0.0.60 → searcharray-0.0.61}/PKG-INFO +14 -3
  2. {searcharray-0.0.60 → searcharray-0.0.61}/README.md +13 -2
  3. {searcharray-0.0.60 → searcharray-0.0.61}/searcharray/indexing.py +27 -3
  4. searcharray-0.0.61/searcharray/phrase/memmap_arrays.py +184 -0
  5. {searcharray-0.0.60 → searcharray-0.0.61}/searcharray/phrase/middle_out.py +10 -16
  6. {searcharray-0.0.60 → searcharray-0.0.61}/searcharray/postings.py +43 -18
  7. {searcharray-0.0.60 → searcharray-0.0.61}/searcharray/roaringish/intersect.c +149 -149
  8. {searcharray-0.0.60 → searcharray-0.0.61}/searcharray/roaringish/merge.c +149 -149
  9. {searcharray-0.0.60 → searcharray-0.0.61}/searcharray/roaringish/popcount.c +149 -149
  10. {searcharray-0.0.60 → searcharray-0.0.61}/searcharray/roaringish/roaringish_ops.c +149 -149
  11. {searcharray-0.0.60 → searcharray-0.0.61}/searcharray/roaringish/search.c +149 -149
  12. {searcharray-0.0.60 → searcharray-0.0.61}/searcharray/roaringish/spans.c +861 -515
  13. {searcharray-0.0.60 → searcharray-0.0.61}/searcharray/roaringish/spans.pyx +51 -27
  14. {searcharray-0.0.60 → searcharray-0.0.61}/searcharray/roaringish/unique.c +149 -149
  15. {searcharray-0.0.60 → searcharray-0.0.61}/searcharray.egg-info/PKG-INFO +14 -3
  16. {searcharray-0.0.60 → searcharray-0.0.61}/searcharray.egg-info/SOURCES.txt +1 -0
  17. {searcharray-0.0.60 → searcharray-0.0.61}/setup.py +1 -1
  18. {searcharray-0.0.60 → searcharray-0.0.61}/test/test_msmarco.py +109 -2
  19. {searcharray-0.0.60 → searcharray-0.0.61}/test/test_phrase_matches.py +9 -1
  20. {searcharray-0.0.60 → searcharray-0.0.61}/test/test_tmdb.py +2 -1
  21. {searcharray-0.0.60 → searcharray-0.0.61}/test/test_utils.py +5 -1
  22. {searcharray-0.0.60 → searcharray-0.0.61}/pyproject.toml +0 -0
  23. {searcharray-0.0.60 → searcharray-0.0.61}/searcharray/__init__.py +0 -0
  24. {searcharray-0.0.60 → searcharray-0.0.61}/searcharray/phrase/__init__.py +0 -0
  25. {searcharray-0.0.60 → searcharray-0.0.61}/searcharray/phrase/bigram_freqs.py +0 -0
  26. {searcharray-0.0.60 → searcharray-0.0.61}/searcharray/phrase/spans.py +0 -0
  27. {searcharray-0.0.60 → searcharray-0.0.61}/searcharray/roaringish/__init__.py +0 -0
  28. {searcharray-0.0.60 → searcharray-0.0.61}/searcharray/roaringish/intersect.pyi +0 -0
  29. {searcharray-0.0.60 → searcharray-0.0.61}/searcharray/roaringish/intersect.pyx +0 -0
  30. {searcharray-0.0.60 → searcharray-0.0.61}/searcharray/roaringish/merge.pyi +0 -0
  31. {searcharray-0.0.60 → searcharray-0.0.61}/searcharray/roaringish/merge.pyx +0 -0
  32. {searcharray-0.0.60 → searcharray-0.0.61}/searcharray/roaringish/popcount.pyi +0 -0
  33. {searcharray-0.0.60 → searcharray-0.0.61}/searcharray/roaringish/popcount.pyx +0 -0
  34. {searcharray-0.0.60 → searcharray-0.0.61}/searcharray/roaringish/roaringish.py +0 -0
  35. {searcharray-0.0.60 → searcharray-0.0.61}/searcharray/roaringish/roaringish_ops.pyx +0 -0
  36. {searcharray-0.0.60 → searcharray-0.0.61}/searcharray/roaringish/search.pyi +0 -0
  37. {searcharray-0.0.60 → searcharray-0.0.61}/searcharray/roaringish/search.pyx +0 -0
  38. {searcharray-0.0.60 → searcharray-0.0.61}/searcharray/roaringish/snp_ops.pxd +0 -0
  39. {searcharray-0.0.60 → searcharray-0.0.61}/searcharray/roaringish/unique.pyi +0 -0
  40. {searcharray-0.0.60 → searcharray-0.0.61}/searcharray/roaringish/unique.pyx +0 -0
  41. {searcharray-0.0.60 → searcharray-0.0.61}/searcharray/similarity.py +0 -0
  42. {searcharray-0.0.60 → searcharray-0.0.61}/searcharray/solr.py +0 -0
  43. {searcharray-0.0.60 → searcharray-0.0.61}/searcharray/term_dict.py +0 -0
  44. {searcharray-0.0.60 → searcharray-0.0.61}/searcharray/utils/__init__.py +0 -0
  45. {searcharray-0.0.60 → searcharray-0.0.61}/searcharray/utils/bitcount.py +0 -0
  46. {searcharray-0.0.60 → searcharray-0.0.61}/searcharray/utils/mat_set.py +0 -0
  47. {searcharray-0.0.60 → searcharray-0.0.61}/searcharray/utils/row_viewable_matrix.py +0 -0
  48. {searcharray-0.0.60 → searcharray-0.0.61}/searcharray/utils/sort.py +0 -0
  49. {searcharray-0.0.60 → searcharray-0.0.61}/searcharray.egg-info/dependency_links.txt +0 -0
  50. {searcharray-0.0.60 → searcharray-0.0.61}/searcharray.egg-info/entry_points.txt +0 -0
  51. {searcharray-0.0.60 → searcharray-0.0.61}/searcharray.egg-info/requires.txt +0 -0
  52. {searcharray-0.0.60 → searcharray-0.0.61}/searcharray.egg-info/top_level.txt +0 -0
  53. {searcharray-0.0.60 → searcharray-0.0.61}/setup.cfg +0 -0
  54. {searcharray-0.0.60 → searcharray-0.0.61}/test/test_bitcount64.py +0 -0
  55. {searcharray-0.0.60 → searcharray-0.0.61}/test/test_extension_array.py +0 -0
  56. {searcharray-0.0.60 → searcharray-0.0.61}/test/test_minmax_posns.py +0 -0
  57. {searcharray-0.0.60 → searcharray-0.0.61}/test/test_search.py +0 -0
  58. {searcharray-0.0.60 → searcharray-0.0.61}/test/test_similarity.py +0 -0
  59. {searcharray-0.0.60 → searcharray-0.0.61}/test/test_slop_matches.py +0 -0
  60. {searcharray-0.0.60 → searcharray-0.0.61}/test/test_snp_ops.py +0 -0
  61. {searcharray-0.0.60 → searcharray-0.0.61}/test/test_solr.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: searcharray
3
- Version: 0.0.60
3
+ Version: 0.0.61
4
4
  Summary: Searchable pandas text extension arrays for prototyping search
5
5
  Home-page: https://github.com/softwaredoug/searcharray
6
6
  Author: Doug Turnbull
@@ -40,7 +40,7 @@ from searcharray import SearchArray
40
40
  import pandas as pd
41
41
 
42
42
  df['title_indexed'] = SearchArray.index(df['title'])
43
- np.sort(df['title_indexed'].array.score('Cat'))
43
+ np.sort(df['title_indexed'].array.score(['cat', 'in', 'the', 'hat'])) # Search w/ phrase
44
44
 
45
45
  > BM25 scores:
46
46
  > array([ 0. , 0. , 0. , ..., 15.84568033, 15.84568033, 15.84568033])
@@ -50,7 +50,7 @@ np.sort(df['title_indexed'].array.score('Cat'))
50
50
 
51
51
  SearchArray is documented in these notebooks:
52
52
 
53
- [SearchArray Guide](https://colab.research.google.com/drive/1gmgVz53fDPTJakUHb6Mttevqry7gKHLM) | [SearchArray Offline Experiment](https://colab.research.google.com/drive/1w_Ajn5rHzcISKhdCuPhhVFav3zrvKWn1)
53
+ [SearchArray Guide](https://colab.research.google.com/drive/1gmgVz53fDPTJakUHb6Mttevqry7gKHLM) | [SearchArray Offline Experiment](https://colab.research.google.com/drive/1w_Ajn5rHzcISKhdCuPhhVFav3zrvKWn1) | [About internals](https://softwaredoug.com/blog/2024/01/21/search-array-phrase-algorithm)
54
54
 
55
55
  ## Installation
56
56
 
@@ -58,6 +58,17 @@ SearchArray is documented in these notebooks:
58
58
  pip install searcharray
59
59
  ```
60
60
 
61
+ ## Features
62
+
63
+ * Search w/ terms by passing a string
64
+ * Search w/ a phrase by passing a list[str]
65
+ * Search w/ a [phrase w/ edit-distance](https://lucene.apache.org/core/9_6_0/core/org/apache/lucene/search/PhraseQuery.html) by passing slop=N.
66
+ * Access raw stats arrays in termfreqs / docfreqs methods on the array
67
+ * Bring your own tokenizer. Pass any (`def tokenize(value: str) -> List[str]`) when indexing.
68
+ * Accepts any python function to compute similarity. Here's [one similarity](https://github.com/softwaredoug/searcharray/blob/main/searcharray/similarity.py#L103)
69
+ * Scores the entire dataframe, allowing combination w/ other ranking attributes (recency, popularity, etc) or scores from other fields (ie boolean queries)
70
+ * Implement's Solr's [edismax query parser](https://github.com/softwaredoug/searcharray/blob/main/searcharray/solr.py) for efficient prototyping
71
+
61
72
  ## Motivation
62
73
 
63
74
  To simplify lexical search in the Python data stack.
@@ -11,7 +11,7 @@ from searcharray import SearchArray
11
11
  import pandas as pd
12
12
 
13
13
  df['title_indexed'] = SearchArray.index(df['title'])
14
- np.sort(df['title_indexed'].array.score('Cat'))
14
+ np.sort(df['title_indexed'].array.score(['cat', 'in', 'the', 'hat'])) # Search w/ phrase
15
15
 
16
16
  > BM25 scores:
17
17
  > array([ 0. , 0. , 0. , ..., 15.84568033, 15.84568033, 15.84568033])
@@ -21,7 +21,7 @@ np.sort(df['title_indexed'].array.score('Cat'))
21
21
 
22
22
  SearchArray is documented in these notebooks:
23
23
 
24
- [SearchArray Guide](https://colab.research.google.com/drive/1gmgVz53fDPTJakUHb6Mttevqry7gKHLM) | [SearchArray Offline Experiment](https://colab.research.google.com/drive/1w_Ajn5rHzcISKhdCuPhhVFav3zrvKWn1)
24
+ [SearchArray Guide](https://colab.research.google.com/drive/1gmgVz53fDPTJakUHb6Mttevqry7gKHLM) | [SearchArray Offline Experiment](https://colab.research.google.com/drive/1w_Ajn5rHzcISKhdCuPhhVFav3zrvKWn1) | [About internals](https://softwaredoug.com/blog/2024/01/21/search-array-phrase-algorithm)
25
25
 
26
26
  ## Installation
27
27
 
@@ -29,6 +29,17 @@ SearchArray is documented in these notebooks:
29
29
  pip install searcharray
30
30
  ```
31
31
 
32
+ ## Features
33
+
34
+ * Search w/ terms by passing a string
35
+ * Search w/ a phrase by passing a list[str]
36
+ * Search w/ a [phrase w/ edit-distance](https://lucene.apache.org/core/9_6_0/core/org/apache/lucene/search/PhraseQuery.html) by passing slop=N.
37
+ * Access raw stats arrays in termfreqs / docfreqs methods on the array
38
+ * Bring your own tokenizer. Pass any (`def tokenize(value: str) -> List[str]`) when indexing.
39
+ * Accepts any python function to compute similarity. Here's [one similarity](https://github.com/softwaredoug/searcharray/blob/main/searcharray/similarity.py#L103)
40
+ * Scores the entire dataframe, allowing combination w/ other ranking attributes (recency, popularity, etc) or scores from other fields (ie boolean queries)
41
+ * Implement's Solr's [edismax query parser](https://github.com/softwaredoug/searcharray/blob/main/searcharray/solr.py) for efficient prototyping
42
+
32
43
  ## Motivation
33
44
 
34
45
  To simplify lexical search in the Python data stack.
@@ -1,8 +1,9 @@
1
1
  import numpy as np
2
2
  import math
3
3
  import gc
4
+ import os
4
5
  import sys
5
- from typing import Iterable, List
6
+ from typing import Iterable, List, Optional
6
7
  from concurrent.futures import ThreadPoolExecutor, as_completed
7
8
  from itertools import islice
8
9
  from searcharray.phrase.middle_out import MAX_POSN, PosnBitArrayFromFlatBuilder, PosnBitArrayBuilder, PosnBitArrayAlreadyEncBuilder
@@ -23,6 +24,17 @@ logger.addHandler(handler)
23
24
  # logging.basicConfig(level=logging.DEBUG)
24
25
 
25
26
 
27
+ def home_directory():
28
+ return os.path.expanduser("~")
29
+
30
+
31
+ def searcharray_home():
32
+ searcharray_dir = os.path.join(home_directory(), ".searcharray")
33
+ # Ensure exists
34
+ os.makedirs(searcharray_dir, exist_ok=True)
35
+ return searcharray_dir
36
+
37
+
26
38
  def _compute_doc_lens(posns: np.ndarray, doc_ids: np.ndarray, num_docs: int) -> np.ndarray:
27
39
  """Given an array of positions, compute the length of each document."""
28
40
  doc_lens = np.zeros(num_docs, dtype=np.uint32)
@@ -49,7 +61,8 @@ def convert_size(size_bytes):
49
61
  return "%s %s" % (s, size_name[i])
50
62
 
51
63
 
52
- def _gather_tokens(array, tokenizer, term_dict, term_doc, start_doc_id=0, trunc_posn=None):
64
+ def _gather_tokens(array, tokenizer,
65
+ term_dict, term_doc, start_doc_id=0, trunc_posn=None):
53
66
  all_terms = []
54
67
  all_docs = []
55
68
  all_posns = []
@@ -127,6 +140,7 @@ def _tokenize_batch(array, tokenizer, term_dict, batch_size, batch_beg, truncate
127
140
  if np.any(doc_lens > MAX_POSN):
128
141
  raise ValueError(f"Document length exceeds maximum of {MAX_POSN}")
129
142
 
143
+ logger.info("Batch tokenization complete")
130
144
  return batch_beg, term_doc, bit_posns, doc_lens
131
145
 
132
146
 
@@ -166,6 +180,7 @@ def _process_batches(term_doc, batch_size,
166
180
  logger.error(f"Batch {batch_beg} failed to tokenize")
167
181
  raise e
168
182
 
183
+ logger.info(f"(main thread) Processing {len(batch_results)} batch results")
169
184
  for result in batch_results:
170
185
  assert result is not None
171
186
  batch_beg, batch_term_doc, batch_bit_posns, batch_doc_lens = result
@@ -173,13 +188,16 @@ def _process_batches(term_doc, batch_size,
173
188
  if bit_posns is None:
174
189
  bit_posns = batch_bit_posns
175
190
  else:
191
+ logger.info("(main thread) Concatenating bit positions")
176
192
  bit_posns.concat(batch_bit_posns)
193
+ logger.info("(main thread) Concatenated bit positions... Done")
177
194
 
178
195
  doc_lens.append(batch_doc_lens)
179
196
  return bit_posns
180
197
 
181
198
 
182
199
  def build_index_from_tokenizer(array: Iterable, tokenizer, batch_size=10000,
200
+ data_dir: Optional[str] = None,
183
201
  truncate=False, workers=4):
184
202
  """Build index directly from tokenizing docs (array of string)."""
185
203
  term_dict = TermDict()
@@ -187,7 +205,7 @@ def build_index_from_tokenizer(array: Iterable, tokenizer, batch_size=10000,
187
205
  doc_lens: List[np.ndarray] = []
188
206
  bit_posns = None
189
207
 
190
- logger.info("Indexing begins")
208
+ logger.info(f"Indexing begins w/ {workers} workers")
191
209
  futures = []
192
210
  last_batch_beg_processed = 0
193
211
 
@@ -226,6 +244,12 @@ def build_index_from_tokenizer(array: Iterable, tokenizer, batch_size=10000,
226
244
 
227
245
  term_doc_built = RowViewableMatrix(term_doc.build())
228
246
  logger.info("Indexing from tokenization complete")
247
+ assert bit_posns is not None
248
+ # if data_dir is None:
249
+ # data_dir = searcharray_home()
250
+ if data_dir is not None:
251
+ logger.info(f"Memmapping bit positions to {data_dir}")
252
+ bit_posns.memmap(data_dir)
229
253
  return term_doc_built, bit_posns, term_dict, avg_doc_length, np.array(doc_lens)
230
254
 
231
255
 
@@ -0,0 +1,184 @@
1
+ import numpy as np
2
+ import os
3
+ from typing import Optional, List
4
+
5
+
6
+ def create_filename(data_dir: str):
7
+ """Create a filename for the memory-mapped array"""
8
+ # Ls path and get the number of files
9
+ files = os.listdir(data_dir)
10
+ num_files = len(files)
11
+ return os.path.join(data_dir, f'{num_files}')
12
+
13
+
14
+ class ArrayDict:
15
+ """An array of ints to np.array as contiguous memory"""
16
+
17
+ def __init__(self):
18
+ self.dtype = np.uint64
19
+ self.data = np.ndarray(0, dtype=self.dtype)
20
+ self.metadata = {}
21
+
22
+ @staticmethod
23
+ def from_arrays(arrays: List[np.ndarray]):
24
+ arr = ArrayDict()
25
+ metadata = {}
26
+ total_len = 0
27
+ for i, array in enumerate(arrays):
28
+ metadata[i] = {'offset': total_len, 'length': array.size}
29
+ total_len += array.size
30
+ data = np.concatenate(arrays)
31
+ arr.data = data
32
+ arr.metadata = metadata
33
+ return arr
34
+
35
+ @staticmethod
36
+ def from_array_with_boundaries(data: np.ndarray,
37
+ ids: np.ndarray,
38
+ boundaries: np.ndarray):
39
+ metadata = {}
40
+ for idx, (beg, end) in enumerate(zip(boundaries[:-1], boundaries[1:])):
41
+ offset = int(beg)
42
+ length = int(end - beg)
43
+ actual_id = ids[idx]
44
+ metadata[int(actual_id)] = {'offset': offset, 'length': length}
45
+ arr = ArrayDict()
46
+ arr.data = data
47
+ arr.metadata = metadata
48
+ return arr
49
+
50
+ @staticmethod
51
+ def concat(lhs: 'ArrayDict', rhs: 'ArrayDict', sort=True):
52
+ metadata = {}
53
+ offset = 0
54
+ for key, value in lhs.metadata.items():
55
+ metadata[key] = {'offset': offset, 'length': value['length']}
56
+ offset += value['length']
57
+ for key, value in rhs.metadata.items():
58
+ metadata[key] = {'offset': offset, 'length': value['length']}
59
+ offset += value['length']
60
+ data = np.concatenate((lhs.data, rhs.data))
61
+ if sort:
62
+ keys = sorted(metadata.keys())
63
+ metadata = {key: metadata[key] for key in keys}
64
+ arr = ArrayDict()
65
+ arr.data = data
66
+ arr.metadata = metadata
67
+ return arr
68
+
69
+ def __getitem__(self, key):
70
+ key = int(key)
71
+ if key in self.metadata:
72
+ offset = self.metadata[key]['offset']
73
+ length = self.metadata[key]['length']
74
+ return self.data[offset:offset + length]
75
+ else:
76
+ raise KeyError(f'Key {key} not found.')
77
+
78
+ def __setitem__(self, key, value):
79
+ key = int(key)
80
+ if value.dtype != self.dtype:
81
+ raise ValueError(f'Value must be of type {self.dtype}')
82
+ offset = self.data.size
83
+ length = value.size
84
+ self.metadata[key] = {'offset': offset, 'length': length}
85
+ self.data = np.append(self.data, value)
86
+
87
+ def __delitem__(self, key):
88
+ key = int(key)
89
+ if key in self.metadata:
90
+ del self.metadata[key]
91
+ else:
92
+ raise KeyError(f'Key {key} not found.')
93
+
94
+ def __len__(self):
95
+ return len(self.metadata)
96
+
97
+ def __iter__(self):
98
+ yield from self.metadata
99
+
100
+ def __contains__(self, key):
101
+ return key in self.metadata
102
+
103
+ def items(self):
104
+ for key, value in self.metadata.items():
105
+ yield key, self.data[value['offset']:value['offset'] + value['length']]
106
+
107
+ def keys(self):
108
+ return self.metadata.keys()
109
+
110
+ def compact(self):
111
+ """Compact the data array"""
112
+ new_data = np.ndarray(sum([value['length'] for value in self.metadata.values()]), dtype=self.dtype)
113
+ offset = 0
114
+ for value in self.metadata.values():
115
+ new_data[offset:offset + value['length']] = self.data[value['offset']:value['offset'] + value['length']]
116
+ value['offset'] = offset
117
+ offset += value['length']
118
+ self.data = new_data
119
+
120
+
121
+ class MemoryMappedArrays:
122
+ def __init__(self,
123
+ data_dir: str,
124
+ arrays: Optional[ArrayDict] = None):
125
+ root_filename = create_filename(data_dir)
126
+ self.filename = root_filename + '.dat'
127
+ self.fp = None
128
+ if arrays is not None:
129
+ self.arrays = arrays
130
+ self._initialize_file()
131
+ else:
132
+ self._load_arrays()
133
+
134
+ def _initialize_file(self):
135
+ """Memmap underlying array in ArrayDict."""
136
+ with open(self.filename, 'wb') as f:
137
+ self.arrays.data.tofile(f)
138
+
139
+ def _load_arrays(self):
140
+ """Memmap underlying array in ArrayDict from self.filename."""
141
+ self.arrays.data = np.memmap(self.filename, dtype=self.arrays.dtype, mode='r+')
142
+
143
+ def __getitem__(self, key):
144
+ # Use metadatat to get the offset and length of the array
145
+ return self.arrays[key]
146
+
147
+ def __setitem__(self, key, value):
148
+ """Append to file and update metadata. Not fast."""
149
+ self.arrays[key] = value
150
+ self._save_metadata()
151
+ self.fp = np.memmap(self.filename, dtype=self.dtype, mode='r+', shape=(self.fp.size + value.size,))
152
+
153
+ def __delitem__(self, key):
154
+ length = self.arrays[key].size
155
+ del self.arrays[key]
156
+ self.fp = np.memmap(self.filename, dtype=self.dtype, mode='r+', shape=(self.fp.size - length,))
157
+
158
+ def __len__(self):
159
+ return len(self.arrays)
160
+
161
+ def __iter__(self):
162
+ yield from self.arrays
163
+
164
+ def __contains__(self, key):
165
+ return key in self.arrays
166
+
167
+ def items(self):
168
+ yield from self.arrays.items()
169
+
170
+ def keys(self):
171
+ return self.arrays.keys()
172
+
173
+ def __getstate__(self):
174
+ """Return arrays without the numpy array."""
175
+ # Create arrays without data
176
+ arrays = ArrayDict()
177
+ arrays.metadata = self.arrays.metadata
178
+ return {'arrays': arrays,
179
+ 'filename': self.filename}
180
+
181
+ def __setstate__(self, state):
182
+ self.filename = state['filename']
183
+ self.arrays = state['arrays']
184
+ self._load_arrays()
@@ -9,6 +9,7 @@ import numpy as np
9
9
  from copy import deepcopy
10
10
  from typing import List, Tuple, Dict, Union, cast, Optional
11
11
  from searcharray.roaringish import RoaringishEncoder, convert_keys, merge
12
+ from searcharray.phrase.memmap_arrays import MemoryMappedArrays, ArrayDict
12
13
  from searcharray.phrase.bigram_freqs import bigram_freqs, Continuation
13
14
  from searcharray.phrase.spans import span_search
14
15
  import numbers
@@ -171,11 +172,9 @@ class PosnBitArrayFromFlatBuilder:
171
172
  payload=self.flat_array[2].view(np.uint64))
172
173
  term_ids = self.flat_array[0][term_boundaries[:-1]]
173
174
 
174
- encoded_term_posns = {}
175
- for into_terms, (beg_idx, end_idx) in enumerate(zip(enc_term_boundaries[:-1], enc_term_boundaries[1:])):
176
- sliced = encoded[beg_idx:end_idx]
177
- encoded_term_posns[term_ids[into_terms]] = sliced
178
-
175
+ encoded_term_posns = ArrayDict.from_array_with_boundaries(encoded,
176
+ boundaries=enc_term_boundaries,
177
+ ids=term_ids)
179
178
  largest_doc_id_with_term = np.max(self.flat_array[1])
180
179
  assert self.max_doc_id >= largest_doc_id_with_term
181
180
  return PosnBitArray(encoded_term_posns, self.max_doc_id)
@@ -197,7 +196,7 @@ class PosnBitArrayBuilder:
197
196
  self.max_doc_id = max(self.max_doc_id, doc_id)
198
197
 
199
198
  def build(self, check=False):
200
- encoded_term_posns = {}
199
+ encoded_term_posns = ArrayDict()
201
200
  for term_id, posns in self.term_posns.items():
202
201
  if len(posns) == 0:
203
202
  posns = np.asarray([], dtype=np.uint32).flatten()
@@ -294,12 +293,15 @@ class FilteredPosns(abc.Mapping):
294
293
 
295
294
  class PosnBitArray:
296
295
 
297
- def __init__(self, encoded_term_posns, max_doc_id: int):
296
+ def __init__(self, encoded_term_posns: Union[ArrayDict, FilteredPosns], max_doc_id: int):
298
297
  self.encoded_term_posns = encoded_term_posns
299
298
  self.max_doc_id = max_doc_id
300
299
  self.docfreq_cache : Dict[int, np.uint64] = {}
301
300
  self.termfreq_cache : Dict[int, Tuple[np.ndarray, np.ndarray]] = {}
302
301
 
302
+ def memmap(self, data_dir):
303
+ self.encoded_term_posns = MemoryMappedArrays(data_dir, self.encoded_term_posns)
304
+
303
305
  def warm(self):
304
306
  """Warm tf / df cache of most common terms."""
305
307
  for term_id, encoded in self.encoded_term_posns.items():
@@ -333,15 +335,7 @@ class PosnBitArray:
333
335
  Assumes other's doc ids are not overlapping with self's doc ids.
334
336
  """
335
337
  # Shared terms
336
- shared_terms = set(self.encoded_term_posns.keys()).intersection(set(other.encoded_term_posns.keys()))
337
- for term_id in shared_terms:
338
- # Append then sort
339
- self.encoded_term_posns[term_id] = np.concatenate([self.encoded_term_posns[term_id], other.encoded_term_posns[term_id]])
340
- self.encoded_term_posns[term_id].sort()
341
-
342
- only_other_terms = set(other.encoded_term_posns.keys()).difference(set(self.encoded_term_posns.keys()))
343
- for term_id in only_other_terms:
344
- self.encoded_term_posns[term_id] = other.encoded_term_posns[term_id]
338
+ self.encoded_term_posns = ArrayDict.concat(self.encoded_term_posns, other.encoded_term_posns)
345
339
  self.max_doc_id = max(self.max_doc_id, other.max_doc_id)
346
340
  self.clear_cache()
347
341
 
@@ -30,6 +30,29 @@ logger.addHandler(handler)
30
30
  logger.setLevel(logging.ERROR)
31
31
 
32
32
 
33
+ def bytes_to_human_readable(num_bytes):
34
+ """
35
+ Convert a number of bytes into a human-readable format (KB, MB, GB, etc.)
36
+
37
+ :param num_bytes: Number of bytes to be converted
38
+ :type num_bytes: int or float
39
+ :return: Human-readable string
40
+ :rtype: str
41
+ """
42
+ # Define the suffixes for byte conversion
43
+ suffixes = ['B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB']
44
+
45
+ if num_bytes == 0:
46
+ return "0B"
47
+
48
+ i = 0
49
+ while num_bytes >= 1024 and i < len(suffixes) - 1:
50
+ num_bytes /= 1024.
51
+ i += 1
52
+
53
+ return f"{num_bytes:.2f} {suffixes[i]}"
54
+
55
+
33
56
  class Terms:
34
57
  """An indexed search doc - a single bag of tokenized words and positions."""
35
58
 
@@ -229,6 +252,7 @@ class SearchArray(ExtensionArray):
229
252
  batch_size=100000,
230
253
  avoid_copies=True,
231
254
  workers=4,
255
+ data_dir: Optional[str] = None,
232
256
  autowarm=True) -> 'SearchArray':
233
257
  """Index an array of strings into SearchArray using tokenizer.
234
258
 
@@ -257,6 +281,7 @@ class SearchArray(ExtensionArray):
257
281
  term_mat, posns, term_dict, avg_doc_length, doc_lens =\
258
282
  build_index_from_tokenizer(array, tokenizer, batch_size=batch_size,
259
283
  truncate=truncate,
284
+ data_dir=data_dir,
260
285
  workers=workers)
261
286
 
262
287
  if autowarm:
@@ -539,7 +564,7 @@ class SearchArray(ExtensionArray):
539
564
  else:
540
565
  raise TypeError("Expected a string or list of strings for phrases")
541
566
 
542
- def memory_report(self):
567
+ def memory_report(self, N=1000):
543
568
  """Return a string with memory usage information."""
544
569
  term_mat_bytes = self.term_mat.nbytes
545
570
  posns_bytes = self.posns.nbytes
@@ -547,32 +572,32 @@ class SearchArray(ExtensionArray):
547
572
  num_terms = len(self.term_dict)
548
573
  top_10_terms_size = []
549
574
 
550
- for i in range(10):
575
+ for i in range(N):
551
576
  term_len = len(self.posns.encoded_term_posns[i])
552
577
  term = self.term_dict.get_term(i)
553
- top_10_terms_size.append((term, term_len))
578
+ term_bytes = term_len * 8
579
+ top_10_terms_size.append((term, term_bytes))
580
+
581
+ top_10_terms_size = sorted(top_10_terms_size, key=lambda x: x[1], reverse=True)
554
582
 
555
- return f"""
583
+ report = f"""
556
584
  SearchArray Memory Report
557
585
  -------------------------
558
586
  Number of Terms: {num_terms}
559
587
  -------------------------
560
- Term Matrix: {term_mat_bytes}
561
- Positions: {posns_bytes}
562
- Term Dictionary: {term_dict_bytes}
563
- --------------------------
564
- Term 0: {top_10_terms_size[0]}
565
- Term 1: {top_10_terms_size[1]}
566
- Term 2: {top_10_terms_size[2]}
567
- Term 3: {top_10_terms_size[3]}
568
- Term 4: {top_10_terms_size[4]}
569
- Term 5: {top_10_terms_size[5]}
570
- Term 6: {top_10_terms_size[6]}
571
- Term 7: {top_10_terms_size[7]}
572
- Term 8: {top_10_terms_size[8]}
573
- Term 9: {top_10_terms_size[9]}
588
+ Term Matrix: {bytes_to_human_readable(term_mat_bytes)}
589
+ Positions: {bytes_to_human_readable(posns_bytes)}
590
+ Term Dictionary: {bytes_to_human_readable(term_dict_bytes)}
591
+
574
592
  """
575
593
 
594
+ cum_sum = 0
595
+ for i in range(N):
596
+ term, term_bytes = top_10_terms_size[i]
597
+ cum_sum += term_bytes
598
+ report += f" Term {i}: {term} - {bytes_to_human_readable(term_bytes)} - Cumulative: {bytes_to_human_readable(cum_sum)}\n"
599
+ return report
600
+
576
601
  # ***********************************************************
577
602
  # Search API
578
603
  # ***********************************************************