searcharray 0.0.57__tar.gz → 0.0.58__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. {searcharray-0.0.57 → searcharray-0.0.58}/PKG-INFO +1 -1
  2. {searcharray-0.0.57 → searcharray-0.0.58}/searcharray/indexing.py +1 -1
  3. {searcharray-0.0.57 → searcharray-0.0.58}/searcharray/phrase/middle_out.py +6 -2
  4. {searcharray-0.0.57 → searcharray-0.0.58}/searcharray/postings.py +9 -0
  5. {searcharray-0.0.57 → searcharray-0.0.58}/searcharray/roaringish/intersect.c +1666 -1113
  6. {searcharray-0.0.57 → searcharray-0.0.58}/searcharray/roaringish/merge.c +949 -347
  7. {searcharray-0.0.57 → searcharray-0.0.58}/searcharray/roaringish/popcount.c +865 -257
  8. {searcharray-0.0.57 → searcharray-0.0.58}/searcharray/roaringish/roaringish_ops.c +865 -257
  9. {searcharray-0.0.57 → searcharray-0.0.58}/searcharray/roaringish/search.c +970 -373
  10. {searcharray-0.0.57 → searcharray-0.0.58}/searcharray/roaringish/spans.c +865 -257
  11. {searcharray-0.0.57 → searcharray-0.0.58}/searcharray/roaringish/unique.c +947 -342
  12. {searcharray-0.0.57 → searcharray-0.0.58}/searcharray.egg-info/PKG-INFO +1 -1
  13. {searcharray-0.0.57 → searcharray-0.0.58}/setup.py +1 -1
  14. {searcharray-0.0.57 → searcharray-0.0.58}/test/test_tmdb.py +14 -2
  15. {searcharray-0.0.57 → searcharray-0.0.58}/README.md +0 -0
  16. {searcharray-0.0.57 → searcharray-0.0.58}/pyproject.toml +0 -0
  17. {searcharray-0.0.57 → searcharray-0.0.58}/searcharray/__init__.py +0 -0
  18. {searcharray-0.0.57 → searcharray-0.0.58}/searcharray/phrase/__init__.py +0 -0
  19. {searcharray-0.0.57 → searcharray-0.0.58}/searcharray/phrase/bigram_freqs.py +0 -0
  20. {searcharray-0.0.57 → searcharray-0.0.58}/searcharray/phrase/spans.py +0 -0
  21. {searcharray-0.0.57 → searcharray-0.0.58}/searcharray/roaringish/__init__.py +0 -0
  22. {searcharray-0.0.57 → searcharray-0.0.58}/searcharray/roaringish/intersect.pyi +0 -0
  23. {searcharray-0.0.57 → searcharray-0.0.58}/searcharray/roaringish/intersect.pyx +0 -0
  24. {searcharray-0.0.57 → searcharray-0.0.58}/searcharray/roaringish/merge.pyi +0 -0
  25. {searcharray-0.0.57 → searcharray-0.0.58}/searcharray/roaringish/merge.pyx +0 -0
  26. {searcharray-0.0.57 → searcharray-0.0.58}/searcharray/roaringish/popcount.pyi +0 -0
  27. {searcharray-0.0.57 → searcharray-0.0.58}/searcharray/roaringish/popcount.pyx +0 -0
  28. {searcharray-0.0.57 → searcharray-0.0.58}/searcharray/roaringish/roaringish.py +0 -0
  29. {searcharray-0.0.57 → searcharray-0.0.58}/searcharray/roaringish/roaringish_ops.pyx +0 -0
  30. {searcharray-0.0.57 → searcharray-0.0.58}/searcharray/roaringish/search.pyi +0 -0
  31. {searcharray-0.0.57 → searcharray-0.0.58}/searcharray/roaringish/search.pyx +0 -0
  32. {searcharray-0.0.57 → searcharray-0.0.58}/searcharray/roaringish/snp_ops.pxd +0 -0
  33. {searcharray-0.0.57 → searcharray-0.0.58}/searcharray/roaringish/spans.pyx +0 -0
  34. {searcharray-0.0.57 → searcharray-0.0.58}/searcharray/roaringish/unique.pyi +0 -0
  35. {searcharray-0.0.57 → searcharray-0.0.58}/searcharray/roaringish/unique.pyx +0 -0
  36. {searcharray-0.0.57 → searcharray-0.0.58}/searcharray/similarity.py +0 -0
  37. {searcharray-0.0.57 → searcharray-0.0.58}/searcharray/solr.py +0 -0
  38. {searcharray-0.0.57 → searcharray-0.0.58}/searcharray/term_dict.py +0 -0
  39. {searcharray-0.0.57 → searcharray-0.0.58}/searcharray/utils/__init__.py +0 -0
  40. {searcharray-0.0.57 → searcharray-0.0.58}/searcharray/utils/bitcount.py +0 -0
  41. {searcharray-0.0.57 → searcharray-0.0.58}/searcharray/utils/mat_set.py +0 -0
  42. {searcharray-0.0.57 → searcharray-0.0.58}/searcharray/utils/row_viewable_matrix.py +0 -0
  43. {searcharray-0.0.57 → searcharray-0.0.58}/searcharray/utils/sort.py +0 -0
  44. {searcharray-0.0.57 → searcharray-0.0.58}/searcharray.egg-info/SOURCES.txt +0 -0
  45. {searcharray-0.0.57 → searcharray-0.0.58}/searcharray.egg-info/dependency_links.txt +0 -0
  46. {searcharray-0.0.57 → searcharray-0.0.58}/searcharray.egg-info/entry_points.txt +0 -0
  47. {searcharray-0.0.57 → searcharray-0.0.58}/searcharray.egg-info/requires.txt +0 -0
  48. {searcharray-0.0.57 → searcharray-0.0.58}/searcharray.egg-info/top_level.txt +0 -0
  49. {searcharray-0.0.57 → searcharray-0.0.58}/setup.cfg +0 -0
  50. {searcharray-0.0.57 → searcharray-0.0.58}/test/test_bitcount64.py +0 -0
  51. {searcharray-0.0.57 → searcharray-0.0.58}/test/test_extension_array.py +0 -0
  52. {searcharray-0.0.57 → searcharray-0.0.58}/test/test_minmax_posns.py +0 -0
  53. {searcharray-0.0.57 → searcharray-0.0.58}/test/test_msmarco.py +0 -0
  54. {searcharray-0.0.57 → searcharray-0.0.58}/test/test_phrase_matches.py +0 -0
  55. {searcharray-0.0.57 → searcharray-0.0.58}/test/test_search.py +0 -0
  56. {searcharray-0.0.57 → searcharray-0.0.58}/test/test_similarity.py +0 -0
  57. {searcharray-0.0.57 → searcharray-0.0.58}/test/test_slop_matches.py +0 -0
  58. {searcharray-0.0.57 → searcharray-0.0.58}/test/test_snp_ops.py +0 -0
  59. {searcharray-0.0.57 → searcharray-0.0.58}/test/test_solr.py +0 -0
  60. {searcharray-0.0.57 → searcharray-0.0.58}/test/test_utils.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: searcharray
3
- Version: 0.0.57
3
+ Version: 0.0.58
4
4
  Summary: Searchable pandas text extension arrays for prototyping search
5
5
  Home-page: https://github.com/softwaredoug/searcharray
6
6
  Author: Doug Turnbull
@@ -121,7 +121,7 @@ def _tokenize_batch(array, tokenizer, term_dict, batch_size, batch_beg, truncate
121
121
 
122
122
  # Encode posns to bit array
123
123
  logger.info("Encoding positions to bit array")
124
- posns = PosnBitArrayFromFlatBuilder(terms_w_posns)
124
+ posns = PosnBitArrayFromFlatBuilder(terms_w_posns, max_doc_id=(batch_beg + len(array) - 1))
125
125
  bit_posns = posns.build()
126
126
 
127
127
  if np.any(doc_lens > MAX_POSN):
@@ -158,8 +158,9 @@ class PosnBitArrayFromFlatBuilder:
158
158
 
159
159
  """
160
160
 
161
- def __init__(self, flat_array: np.ndarray):
161
+ def __init__(self, flat_array: np.ndarray, max_doc_id: int) -> None:
162
162
  self.flat_array = flat_array
163
+ self.max_doc_id = max_doc_id
163
164
 
164
165
  def build(self):
165
166
  """Slice the flat array into a 2d array of doc ids and posns."""
@@ -179,7 +180,9 @@ class PosnBitArrayFromFlatBuilder:
179
180
  sliced = encoded[beg_idx:end_idx]
180
181
  encoded_term_posns[term_ids[into_terms]] = sliced
181
182
 
182
- return PosnBitArray(encoded_term_posns, self.flat_array[1].max())
183
+ largest_doc_id_with_term = np.max(self.flat_array[1])
184
+ assert self.max_doc_id >= largest_doc_id_with_term
185
+ return PosnBitArray(encoded_term_posns, self.max_doc_id)
183
186
 
184
187
 
185
188
  class PosnBitArrayBuilder:
@@ -390,6 +393,7 @@ class PosnBitArray:
390
393
  min_posn: Optional[int] = None,
391
394
  max_posn: Optional[int] = None) -> np.ndarray:
392
395
  phrase_freqs = self.empty_buffer()
396
+
393
397
  if len(term_ids) < 2:
394
398
  raise ValueError("Must have at least two terms")
395
399
  if phrase_freqs.shape[0] == self.max_doc_id + 1 and min_posn is None and max_posn is None and doc_ids is None:
@@ -469,6 +469,15 @@ class SearchArray(ExtensionArray):
469
469
  logger.warning("Unique called on SearchArray. This is not supported.")
470
470
  return self[:]
471
471
 
472
+ def __iter__(self):
473
+ if len(self) > 10000:
474
+ warning_text = """Iterating over SearchArray is very slow and not reccomended.
475
+
476
+ If you're looping a dataframe, slice out the non-SearchArray columns first."""
477
+ logger.warning(warning_text)
478
+ warnings.warn(warning_text)
479
+ return super().__iter__()
480
+
472
481
  def take(self, indices, allow_fill=False, fill_value=None):
473
482
  # Want to take rows of term freqs
474
483
  row_indices = np.arange(len(self.term_mat.rows))