searcharray 0.0.44__tar.gz → 0.0.45__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. {searcharray-0.0.44 → searcharray-0.0.45}/PKG-INFO +1 -1
  2. {searcharray-0.0.44 → searcharray-0.0.45}/searcharray/phrase/middle_out.py +6 -4
  3. {searcharray-0.0.44 → searcharray-0.0.45}/searcharray/roaringish/roaringish.py +24 -20
  4. {searcharray-0.0.44 → searcharray-0.0.45}/searcharray/roaringish/roaringish_ops.c +740 -124
  5. {searcharray-0.0.44 → searcharray-0.0.45}/searcharray/roaringish/roaringish_ops.pyx +25 -0
  6. {searcharray-0.0.44 → searcharray-0.0.45}/searcharray/roaringish/snp_ops.c +117 -117
  7. {searcharray-0.0.44 → searcharray-0.0.45}/searcharray.egg-info/PKG-INFO +1 -1
  8. {searcharray-0.0.44 → searcharray-0.0.45}/setup.py +1 -1
  9. {searcharray-0.0.44 → searcharray-0.0.45}/test/test_msmarco.py +32 -0
  10. {searcharray-0.0.44 → searcharray-0.0.45}/README.md +0 -0
  11. {searcharray-0.0.44 → searcharray-0.0.45}/pyproject.toml +0 -0
  12. {searcharray-0.0.44 → searcharray-0.0.45}/searcharray/__init__.py +0 -0
  13. {searcharray-0.0.44 → searcharray-0.0.45}/searcharray/indexing.py +0 -0
  14. {searcharray-0.0.44 → searcharray-0.0.45}/searcharray/phrase/__init__.py +0 -0
  15. {searcharray-0.0.44 → searcharray-0.0.45}/searcharray/phrase/bigram_freqs.py +0 -0
  16. {searcharray-0.0.44 → searcharray-0.0.45}/searcharray/phrase/posn_diffs.py +0 -0
  17. {searcharray-0.0.44 → searcharray-0.0.45}/searcharray/phrase/scan_merge.py +0 -0
  18. {searcharray-0.0.44 → searcharray-0.0.45}/searcharray/postings.py +0 -0
  19. {searcharray-0.0.44 → searcharray-0.0.45}/searcharray/roaringish/__init__.py +0 -0
  20. {searcharray-0.0.44 → searcharray-0.0.45}/searcharray/roaringish/snp_ops.pxd +0 -0
  21. {searcharray-0.0.44 → searcharray-0.0.45}/searcharray/roaringish/snp_ops.pyx +0 -0
  22. {searcharray-0.0.44 → searcharray-0.0.45}/searcharray/similarity.py +0 -0
  23. {searcharray-0.0.44 → searcharray-0.0.45}/searcharray/solr.py +0 -0
  24. {searcharray-0.0.44 → searcharray-0.0.45}/searcharray/term_dict.py +0 -0
  25. {searcharray-0.0.44 → searcharray-0.0.45}/searcharray/utils/__init__.py +0 -0
  26. {searcharray-0.0.44 → searcharray-0.0.45}/searcharray/utils/bitcount.py +0 -0
  27. {searcharray-0.0.44 → searcharray-0.0.45}/searcharray/utils/mat_set.py +0 -0
  28. {searcharray-0.0.44 → searcharray-0.0.45}/searcharray/utils/row_viewable_matrix.py +0 -0
  29. {searcharray-0.0.44 → searcharray-0.0.45}/searcharray.egg-info/SOURCES.txt +0 -0
  30. {searcharray-0.0.44 → searcharray-0.0.45}/searcharray.egg-info/dependency_links.txt +0 -0
  31. {searcharray-0.0.44 → searcharray-0.0.45}/searcharray.egg-info/entry_points.txt +0 -0
  32. {searcharray-0.0.44 → searcharray-0.0.45}/searcharray.egg-info/requires.txt +0 -0
  33. {searcharray-0.0.44 → searcharray-0.0.45}/searcharray.egg-info/top_level.txt +0 -0
  34. {searcharray-0.0.44 → searcharray-0.0.45}/setup.cfg +0 -0
  35. {searcharray-0.0.44 → searcharray-0.0.45}/test/test_bitcount64.py +0 -0
  36. {searcharray-0.0.44 → searcharray-0.0.45}/test/test_extension_array.py +0 -0
  37. {searcharray-0.0.44 → searcharray-0.0.45}/test/test_minmax_posns.py +0 -0
  38. {searcharray-0.0.44 → searcharray-0.0.45}/test/test_phrase_matches.py +0 -0
  39. {searcharray-0.0.44 → searcharray-0.0.45}/test/test_search.py +0 -0
  40. {searcharray-0.0.44 → searcharray-0.0.45}/test/test_snp_ops.py +0 -0
  41. {searcharray-0.0.44 → searcharray-0.0.45}/test/test_solr.py +0 -0
  42. {searcharray-0.0.44 → searcharray-0.0.45}/test/test_tmdb.py +0 -0
  43. {searcharray-0.0.44 → searcharray-0.0.45}/test/test_utils.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: searcharray
3
- Version: 0.0.44
3
+ Version: 0.0.45
4
4
  Summary: Searchable pandas text extension arrays for prototyping search
5
5
  Home-page: https://github.com/softwaredoug/searcharray
6
6
  Author: Doug Turnbull
@@ -344,8 +344,12 @@ class PosnBitArray:
344
344
  if phrase_freqs.shape[0] == self.max_doc_id + 1 and min_posn is None and max_posn is None:
345
345
  enc_term_posns = [self.encoded_term_posns[term_id] for term_id in term_ids]
346
346
  else:
347
+ keys = None
348
+ if phrase_freqs.shape[0] != self.max_doc_id + 1:
349
+ keys = doc_ids.view(np.uint64)
350
+
347
351
  enc_term_posns = [encoder.slice(self.encoded_term_posns[term_id],
348
- keys=doc_ids.view(np.uint64),
352
+ keys=keys,
349
353
  min_payload=min_posn,
350
354
  max_payload=max_posn) for term_id in term_ids]
351
355
  return compute_phrase_freqs(enc_term_posns, phrase_freqs)
@@ -390,13 +394,11 @@ class PosnBitArray:
390
394
  """Count term freqs using unique positions."""
391
395
  if doc_ids is None and max_posn is None and min_posn is None:
392
396
  return self._termfreqs_with_cache(term_id)
393
- if doc_ids is None:
394
- doc_ids = encoder.keys_unique(self.encoded_term_posns[term_id])
395
397
 
396
398
  encoded = self.encoded_term_posns[term_id]
397
399
  term_posns = encoded
398
400
  term_posns = encoder.slice(encoded,
399
- keys=doc_ids.astype(np.uint64),
401
+ keys=doc_ids,
400
402
  min_payload=min_posn,
401
403
  max_payload=max_posn)
402
404
 
@@ -8,7 +8,7 @@ import numbers
8
8
  from typing import Optional, Tuple, List, Union
9
9
 
10
10
  from searcharray.roaringish.snp_ops import intersect, unique, adjacent, merge
11
- from searcharray.roaringish.roaringish_ops import popcount64_reduce
11
+ from searcharray.roaringish.roaringish_ops import popcount64_reduce, payload_slice
12
12
 
13
13
  logger = logging.getLogger(__name__)
14
14
 
@@ -210,31 +210,35 @@ class RoaringishEncoder:
210
210
  lhs_idx, rhs_idx = intersect(lhs, rhs, mask=self.header_mask)
211
211
  return lhs[lhs_idx], rhs[rhs_idx]
212
212
 
213
- def slice(self, encoded: np.ndarray,
214
- keys: np.ndarray,
213
+ def slice(self,
214
+ encoded: np.ndarray,
215
+ keys: Optional[np.ndarray] = None,
215
216
  max_payload: Optional[int] = None,
216
217
  min_payload: Optional[int] = None) -> np.ndarray:
217
218
  """Get list of encoded that have values in keys."""
218
219
  # encoded_keys = encoded.view(np.uint64) >> (_64 - self.key_bits)
219
- encoded_keys = self.keys(encoded)
220
- idx_docs, idx_enc = intersect(keys, encoded_keys,
221
- drop_duplicates=False)
220
+ if keys is not None:
221
+ encoded_keys = self.keys(encoded)
222
+ idx_docs, idx_enc = intersect(keys.view(np.uint64),
223
+ encoded_keys.view(np.uint64),
224
+ drop_duplicates=False)
225
+ encoded = encoded[idx_enc]
222
226
  if max_payload is None and min_payload is None:
223
- return encoded[idx_enc]
227
+ return encoded
224
228
  else:
225
- gt_min_payload = np.ones(len(encoded), dtype=bool)
226
- lt_max_payload = np.ones(len(encoded), dtype=bool)
227
- if min_payload is not None:
228
- if min_payload % self.payload_lsb_bits != 0:
229
- raise ValueError(f"min_payload must be a multiple of {self.payload_lsb_bits}")
230
- min_payload_bit = min_payload // self.payload_lsb_bits
231
- gt_min_payload = self.payload_msb(encoded[idx_enc]) >= min_payload_bit
232
- if max_payload is not None:
233
- if max_payload % self.payload_lsb_bits != self.payload_lsb_bits - 1:
234
- raise ValueError(f"max_payload must be a multiple of {self.payload_lsb_bits} - 1")
235
- max_payload_bit = max_payload // self.payload_lsb_bits
236
- lt_max_payload = self.payload_msb(encoded[idx_enc]) <= max_payload_bit
237
- return encoded[lt_max_payload & gt_min_payload]
229
+ if min_payload is not None and min_payload % self.payload_lsb_bits != 0:
230
+ raise ValueError(f"min_payload must be a multiple of {self.payload_lsb_bits}")
231
+ if max_payload is not None and max_payload % self.payload_lsb_bits != self.payload_lsb_bits - 1:
232
+ raise ValueError(f"max_payload must be a multiple of {self.payload_lsb_bits} - 1")
233
+ min_payload = 0 if min_payload is None else min_payload
234
+ max_payload = 0xFFFFFFFFFFFFFFFF if max_payload is None else max_payload
235
+
236
+ def payl_slice():
237
+ return payload_slice(encoded, self.payload_msb_mask,
238
+ min_payload // self.payload_lsb_bits,
239
+ max_payload // self.payload_lsb_bits)
240
+
241
+ return payl_slice()
238
242
 
239
243
 
240
244
  def convert_keys(keys) -> np.ndarray: