searcharray 0.0.43__tar.gz → 0.0.45__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. {searcharray-0.0.43 → searcharray-0.0.45}/PKG-INFO +1 -1
  2. {searcharray-0.0.43 → searcharray-0.0.45}/searcharray/phrase/bigram_freqs.py +2 -2
  3. {searcharray-0.0.43 → searcharray-0.0.45}/searcharray/phrase/middle_out.py +9 -16
  4. {searcharray-0.0.43 → searcharray-0.0.45}/searcharray/postings.py +4 -2
  5. searcharray-0.0.45/searcharray/roaringish/__init__.py +3 -0
  6. {searcharray-0.0.43/searcharray/utils → searcharray-0.0.45/searcharray/roaringish}/roaringish.py +29 -20
  7. searcharray-0.0.45/searcharray/roaringish/roaringish_ops.c +30895 -0
  8. searcharray-0.0.45/searcharray/roaringish/roaringish_ops.pyx +80 -0
  9. {searcharray-0.0.43/searcharray/utils → searcharray-0.0.45/searcharray/roaringish}/snp_ops.c +1614 -2116
  10. searcharray-0.0.45/searcharray/roaringish/snp_ops.pxd +22 -0
  11. {searcharray-0.0.43/searcharray/utils → searcharray-0.0.45/searcharray/roaringish}/snp_ops.pyx +5 -19
  12. {searcharray-0.0.43 → searcharray-0.0.45}/searcharray.egg-info/PKG-INFO +1 -1
  13. {searcharray-0.0.43 → searcharray-0.0.45}/searcharray.egg-info/SOURCES.txt +7 -3
  14. {searcharray-0.0.43 → searcharray-0.0.45}/setup.py +6 -4
  15. {searcharray-0.0.43 → searcharray-0.0.45}/test/test_bitcount64.py +1 -1
  16. {searcharray-0.0.43 → searcharray-0.0.45}/test/test_msmarco.py +34 -0
  17. {searcharray-0.0.43 → searcharray-0.0.45}/test/test_snp_ops.py +1 -1
  18. {searcharray-0.0.43 → searcharray-0.0.45}/README.md +0 -0
  19. {searcharray-0.0.43 → searcharray-0.0.45}/pyproject.toml +0 -0
  20. {searcharray-0.0.43 → searcharray-0.0.45}/searcharray/__init__.py +0 -0
  21. {searcharray-0.0.43 → searcharray-0.0.45}/searcharray/indexing.py +0 -0
  22. {searcharray-0.0.43 → searcharray-0.0.45}/searcharray/phrase/__init__.py +0 -0
  23. {searcharray-0.0.43 → searcharray-0.0.45}/searcharray/phrase/posn_diffs.py +0 -0
  24. {searcharray-0.0.43 → searcharray-0.0.45}/searcharray/phrase/scan_merge.py +0 -0
  25. {searcharray-0.0.43 → searcharray-0.0.45}/searcharray/similarity.py +0 -0
  26. {searcharray-0.0.43 → searcharray-0.0.45}/searcharray/solr.py +0 -0
  27. {searcharray-0.0.43 → searcharray-0.0.45}/searcharray/term_dict.py +0 -0
  28. {searcharray-0.0.43 → searcharray-0.0.45}/searcharray/utils/__init__.py +0 -0
  29. {searcharray-0.0.43 → searcharray-0.0.45}/searcharray/utils/bitcount.py +0 -0
  30. {searcharray-0.0.43 → searcharray-0.0.45}/searcharray/utils/mat_set.py +0 -0
  31. {searcharray-0.0.43 → searcharray-0.0.45}/searcharray/utils/row_viewable_matrix.py +0 -0
  32. {searcharray-0.0.43 → searcharray-0.0.45}/searcharray.egg-info/dependency_links.txt +0 -0
  33. {searcharray-0.0.43 → searcharray-0.0.45}/searcharray.egg-info/entry_points.txt +0 -0
  34. {searcharray-0.0.43 → searcharray-0.0.45}/searcharray.egg-info/requires.txt +0 -0
  35. {searcharray-0.0.43 → searcharray-0.0.45}/searcharray.egg-info/top_level.txt +0 -0
  36. {searcharray-0.0.43 → searcharray-0.0.45}/setup.cfg +0 -0
  37. {searcharray-0.0.43 → searcharray-0.0.45}/test/test_extension_array.py +0 -0
  38. {searcharray-0.0.43 → searcharray-0.0.45}/test/test_minmax_posns.py +0 -0
  39. {searcharray-0.0.43 → searcharray-0.0.45}/test/test_phrase_matches.py +0 -0
  40. {searcharray-0.0.43 → searcharray-0.0.45}/test/test_search.py +0 -0
  41. {searcharray-0.0.43 → searcharray-0.0.45}/test/test_solr.py +0 -0
  42. {searcharray-0.0.43 → searcharray-0.0.45}/test/test_tmdb.py +0 -0
  43. {searcharray-0.0.43 → searcharray-0.0.45}/test/test_utils.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: searcharray
3
- Version: 0.0.43
3
+ Version: 0.0.45
4
4
  Summary: Searchable pandas text extension arrays for prototyping search
5
5
  Home-page: https://github.com/softwaredoug/searcharray
6
6
  Author: Doug Turnbull
@@ -1,11 +1,11 @@
1
1
  """Given two roaringish encoded arrays, count bigram matches between them."""
2
2
  import numpy as np
3
3
  from typing import Tuple, Optional
4
- from searcharray.utils.roaringish import RoaringishEncoder
4
+ from searcharray.roaringish import RoaringishEncoder
5
5
  import logging
6
6
  from enum import Enum
7
7
 
8
- from searcharray.utils.snp_ops import intersect, popcount64, merge
8
+ from searcharray.roaringish import intersect, popcount64, merge
9
9
 
10
10
 
11
11
  logger = logging.getLogger(__name__)
@@ -8,14 +8,12 @@ https://colab.research.google.com/drive/10tIEkdlCE_1J_CcgEcV0jkLfBc-0H4am?authus
8
8
  import numpy as np
9
9
  from copy import deepcopy
10
10
  from typing import List, Tuple, Dict, Union, cast, Optional
11
- from searcharray.utils.roaringish import RoaringishEncoder, convert_keys, sorted_unique
11
+ from searcharray.roaringish import RoaringishEncoder, convert_keys, merge
12
12
  from searcharray.phrase.bigram_freqs import bigram_freqs, Continuation
13
13
  import numbers
14
14
  import logging
15
15
  from collections import defaultdict
16
16
 
17
- from searcharray.utils.snp_ops import popcount64, merge
18
-
19
17
 
20
18
  logger = logging.getLogger(__name__)
21
19
 
@@ -346,8 +344,12 @@ class PosnBitArray:
346
344
  if phrase_freqs.shape[0] == self.max_doc_id + 1 and min_posn is None and max_posn is None:
347
345
  enc_term_posns = [self.encoded_term_posns[term_id] for term_id in term_ids]
348
346
  else:
347
+ keys = None
348
+ if phrase_freqs.shape[0] != self.max_doc_id + 1:
349
+ keys = doc_ids.view(np.uint64)
350
+
349
351
  enc_term_posns = [encoder.slice(self.encoded_term_posns[term_id],
350
- keys=doc_ids.view(np.uint64),
352
+ keys=keys,
351
353
  min_payload=min_posn,
352
354
  max_payload=max_posn) for term_id in term_ids]
353
355
  return compute_phrase_freqs(enc_term_posns, phrase_freqs)
@@ -392,27 +394,18 @@ class PosnBitArray:
392
394
  """Count term freqs using unique positions."""
393
395
  if doc_ids is None and max_posn is None and min_posn is None:
394
396
  return self._termfreqs_with_cache(term_id)
395
- if doc_ids is None:
396
- doc_ids = encoder.keys_unique(self.encoded_term_posns[term_id])
397
397
 
398
398
  encoded = self.encoded_term_posns[term_id]
399
399
  term_posns = encoded
400
400
  term_posns = encoder.slice(encoded,
401
- keys=doc_ids.astype(np.uint64),
401
+ keys=doc_ids,
402
402
  min_payload=min_posn,
403
403
  max_payload=max_posn)
404
404
 
405
405
  return self._computed_term_freqs(term_posns)
406
406
 
407
407
  def _computed_term_freqs(self, term_posns) -> Tuple[np.ndarray, np.ndarray]:
408
- doc_ids = encoder.keys(term_posns)
409
- change_indices = np.nonzero(np.diff(doc_ids))[0]
410
- change_indices = np.concatenate((np.asarray([0]), change_indices + 1))
411
- posns = term_posns & encoder.payload_lsb_mask
412
- bit_counts = popcount64(posns)
413
-
414
- term_freqs = np.add.reduceat(bit_counts, change_indices, dtype=np.float32)
415
- return sorted_unique(doc_ids), term_freqs
408
+ return encoder.num_values_per_key(term_posns)
416
409
 
417
410
  def _termfreqs_with_cache(self, term_id: int) -> Tuple[np.ndarray, np.ndarray]:
418
411
  try:
@@ -431,7 +424,7 @@ class PosnBitArray:
431
424
  return self.docfreq_cache[term_id]
432
425
 
433
426
  def _maybe_cache_docfreq(self, term_id: int, docfreq: np.uint64):
434
- if self.max_doc_id >= 100000 and docfreq > (self.max_doc_id // 100):
427
+ if self.max_doc_id >= 99999 and docfreq > (self.max_doc_id // 100):
435
428
  self.docfreq_cache[term_id] = docfreq
436
429
 
437
430
  def docfreq(self, term_id: int) -> np.uint64:
@@ -567,7 +567,9 @@ class SearchArray(ExtensionArray):
567
567
  term_freq = self.termfreqs(token)
568
568
  return term_freq > 0
569
569
 
570
- def score(self, token: Union[str, List[str]], similarity: Similarity = default_bm25) -> np.ndarray:
570
+ def score(self, token: Union[str, List[str]], similarity: Similarity = default_bm25,
571
+ min_posn: Optional[int] = None,
572
+ max_posn: Optional[int] = None) -> np.ndarray:
571
573
  """Score each doc using a similarity function.
572
574
 
573
575
  Parameters
@@ -583,7 +585,7 @@ class SearchArray(ExtensionArray):
583
585
  tokens_l = [token] if isinstance(token, str) else token
584
586
  all_dfs = np.asarray([self.docfreq(token) for token in tokens_l])
585
587
 
586
- tfs = self.termfreqs(token)
588
+ tfs = self.termfreqs(token, min_posn=min_posn, max_posn=max_posn)
587
589
  token = self._check_token_arg(token)
588
590
  doc_lens = self.doclengths()
589
591
  scores = similarity(term_freqs=tfs, doc_freqs=all_dfs,
@@ -0,0 +1,3 @@
1
+ from searcharray.roaringish.roaringish import RoaringishEncoder, convert_keys # noqa
2
+ from searcharray.roaringish.snp_ops import popcount64 # noqa
3
+ from searcharray.roaringish.snp_ops import merge, intersect, unique, adjacent # noqa
@@ -7,7 +7,8 @@ import logging
7
7
  import numbers
8
8
  from typing import Optional, Tuple, List, Union
9
9
 
10
- from searcharray.utils.snp_ops import intersect, unique, adjacent, merge
10
+ from searcharray.roaringish.snp_ops import intersect, unique, adjacent, merge
11
+ from searcharray.roaringish.roaringish_ops import popcount64_reduce, payload_slice
11
12
 
12
13
  logger = logging.getLogger(__name__)
13
14
 
@@ -157,6 +158,10 @@ class RoaringishEncoder:
157
158
  else:
158
159
  return grouped
159
160
 
161
+ def num_values_per_key(self, encoded: np.ndarray) -> Tuple[np.ndarray, np.ndarray]:
162
+ pcr = popcount64_reduce(encoded, (_64 - self.key_bits), self.payload_lsb_mask)
163
+ return pcr
164
+
160
165
  def keys(self, encoded: np.ndarray) -> np.ndarray:
161
166
  """Return keys from encoded."""
162
167
  return encoded >> (_64 - self.key_bits)
@@ -205,31 +210,35 @@ class RoaringishEncoder:
205
210
  lhs_idx, rhs_idx = intersect(lhs, rhs, mask=self.header_mask)
206
211
  return lhs[lhs_idx], rhs[rhs_idx]
207
212
 
208
- def slice(self, encoded: np.ndarray,
209
- keys: np.ndarray,
213
+ def slice(self,
214
+ encoded: np.ndarray,
215
+ keys: Optional[np.ndarray] = None,
210
216
  max_payload: Optional[int] = None,
211
217
  min_payload: Optional[int] = None) -> np.ndarray:
212
218
  """Get list of encoded that have values in keys."""
213
219
  # encoded_keys = encoded.view(np.uint64) >> (_64 - self.key_bits)
214
- encoded_keys = self.keys(encoded)
215
- idx_docs, idx_enc = intersect(keys, encoded_keys,
216
- drop_duplicates=False)
220
+ if keys is not None:
221
+ encoded_keys = self.keys(encoded)
222
+ idx_docs, idx_enc = intersect(keys.view(np.uint64),
223
+ encoded_keys.view(np.uint64),
224
+ drop_duplicates=False)
225
+ encoded = encoded[idx_enc]
217
226
  if max_payload is None and min_payload is None:
218
- return encoded[idx_enc]
227
+ return encoded
219
228
  else:
220
- gt_min_payload = np.ones(len(encoded), dtype=bool)
221
- lt_max_payload = np.ones(len(encoded), dtype=bool)
222
- if min_payload is not None:
223
- if min_payload % self.payload_lsb_bits != 0:
224
- raise ValueError(f"min_payload must be a multiple of {self.payload_lsb_bits}")
225
- min_payload_bit = min_payload // self.payload_lsb_bits
226
- gt_min_payload = self.payload_msb(encoded[idx_enc]) >= min_payload_bit
227
- if max_payload is not None:
228
- if max_payload % self.payload_lsb_bits != self.payload_lsb_bits - 1:
229
- raise ValueError(f"max_payload must be a multiple of {self.payload_lsb_bits} - 1")
230
- max_payload_bit = max_payload // self.payload_lsb_bits
231
- lt_max_payload = self.payload_msb(encoded[idx_enc]) <= max_payload_bit
232
- return encoded[lt_max_payload & gt_min_payload]
229
+ if min_payload is not None and min_payload % self.payload_lsb_bits != 0:
230
+ raise ValueError(f"min_payload must be a multiple of {self.payload_lsb_bits}")
231
+ if max_payload is not None and max_payload % self.payload_lsb_bits != self.payload_lsb_bits - 1:
232
+ raise ValueError(f"max_payload must be a multiple of {self.payload_lsb_bits} - 1")
233
+ min_payload = 0 if min_payload is None else min_payload
234
+ max_payload = 0xFFFFFFFFFFFFFFFF if max_payload is None else max_payload
235
+
236
+ def payl_slice():
237
+ return payload_slice(encoded, self.payload_msb_mask,
238
+ min_payload // self.payload_lsb_bits,
239
+ max_payload // self.payload_lsb_bits)
240
+
241
+ return payl_slice()
233
242
 
234
243
 
235
244
  def convert_keys(keys) -> np.ndarray: