searcharray 0.0.56__tar.gz → 0.0.58__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. {searcharray-0.0.56 → searcharray-0.0.58}/PKG-INFO +1 -1
  2. {searcharray-0.0.56 → searcharray-0.0.58}/pyproject.toml +3 -0
  3. {searcharray-0.0.56 → searcharray-0.0.58}/searcharray/indexing.py +1 -1
  4. {searcharray-0.0.56 → searcharray-0.0.58}/searcharray/phrase/middle_out.py +6 -2
  5. {searcharray-0.0.56 → searcharray-0.0.58}/searcharray/postings.py +14 -0
  6. {searcharray-0.0.56 → searcharray-0.0.58}/searcharray/roaringish/intersect.c +1875 -1342
  7. {searcharray-0.0.56 → searcharray-0.0.58}/searcharray/roaringish/intersect.pyx +17 -22
  8. {searcharray-0.0.56 → searcharray-0.0.58}/searcharray/roaringish/merge.c +1267 -665
  9. {searcharray-0.0.56 → searcharray-0.0.58}/searcharray/roaringish/merge.pyx +0 -2
  10. {searcharray-0.0.56 → searcharray-0.0.58}/searcharray/roaringish/popcount.c +1001 -403
  11. {searcharray-0.0.56 → searcharray-0.0.58}/searcharray/roaringish/popcount.pyx +1 -2
  12. {searcharray-0.0.56 → searcharray-0.0.58}/searcharray/roaringish/roaringish_ops.c +997 -422
  13. {searcharray-0.0.56 → searcharray-0.0.58}/searcharray/roaringish/roaringish_ops.pyx +0 -1
  14. {searcharray-0.0.56 → searcharray-0.0.58}/searcharray/roaringish/search.c +1323 -732
  15. {searcharray-0.0.56 → searcharray-0.0.58}/searcharray/roaringish/search.pyx +5 -6
  16. {searcharray-0.0.56 → searcharray-0.0.58}/searcharray/roaringish/spans.c +950 -375
  17. {searcharray-0.0.56 → searcharray-0.0.58}/searcharray/roaringish/spans.pyx +4 -11
  18. {searcharray-0.0.56 → searcharray-0.0.58}/searcharray/roaringish/unique.c +1222 -617
  19. {searcharray-0.0.56 → searcharray-0.0.58}/searcharray/roaringish/unique.pyx +1 -3
  20. {searcharray-0.0.56 → searcharray-0.0.58}/searcharray.egg-info/PKG-INFO +1 -1
  21. {searcharray-0.0.56 → searcharray-0.0.58}/setup.py +1 -1
  22. {searcharray-0.0.56 → searcharray-0.0.58}/test/test_extension_array.py +8 -1
  23. {searcharray-0.0.56 → searcharray-0.0.58}/test/test_tmdb.py +27 -2
  24. {searcharray-0.0.56 → searcharray-0.0.58}/README.md +0 -0
  25. {searcharray-0.0.56 → searcharray-0.0.58}/searcharray/__init__.py +0 -0
  26. {searcharray-0.0.56 → searcharray-0.0.58}/searcharray/phrase/__init__.py +0 -0
  27. {searcharray-0.0.56 → searcharray-0.0.58}/searcharray/phrase/bigram_freqs.py +0 -0
  28. {searcharray-0.0.56 → searcharray-0.0.58}/searcharray/phrase/spans.py +0 -0
  29. {searcharray-0.0.56 → searcharray-0.0.58}/searcharray/roaringish/__init__.py +0 -0
  30. {searcharray-0.0.56 → searcharray-0.0.58}/searcharray/roaringish/intersect.pyi +0 -0
  31. {searcharray-0.0.56 → searcharray-0.0.58}/searcharray/roaringish/merge.pyi +0 -0
  32. {searcharray-0.0.56 → searcharray-0.0.58}/searcharray/roaringish/popcount.pyi +0 -0
  33. {searcharray-0.0.56 → searcharray-0.0.58}/searcharray/roaringish/roaringish.py +0 -0
  34. {searcharray-0.0.56 → searcharray-0.0.58}/searcharray/roaringish/search.pyi +0 -0
  35. {searcharray-0.0.56 → searcharray-0.0.58}/searcharray/roaringish/snp_ops.pxd +0 -0
  36. {searcharray-0.0.56 → searcharray-0.0.58}/searcharray/roaringish/unique.pyi +0 -0
  37. {searcharray-0.0.56 → searcharray-0.0.58}/searcharray/similarity.py +0 -0
  38. {searcharray-0.0.56 → searcharray-0.0.58}/searcharray/solr.py +0 -0
  39. {searcharray-0.0.56 → searcharray-0.0.58}/searcharray/term_dict.py +0 -0
  40. {searcharray-0.0.56 → searcharray-0.0.58}/searcharray/utils/__init__.py +0 -0
  41. {searcharray-0.0.56 → searcharray-0.0.58}/searcharray/utils/bitcount.py +0 -0
  42. {searcharray-0.0.56 → searcharray-0.0.58}/searcharray/utils/mat_set.py +0 -0
  43. {searcharray-0.0.56 → searcharray-0.0.58}/searcharray/utils/row_viewable_matrix.py +0 -0
  44. {searcharray-0.0.56 → searcharray-0.0.58}/searcharray/utils/sort.py +0 -0
  45. {searcharray-0.0.56 → searcharray-0.0.58}/searcharray.egg-info/SOURCES.txt +0 -0
  46. {searcharray-0.0.56 → searcharray-0.0.58}/searcharray.egg-info/dependency_links.txt +0 -0
  47. {searcharray-0.0.56 → searcharray-0.0.58}/searcharray.egg-info/entry_points.txt +0 -0
  48. {searcharray-0.0.56 → searcharray-0.0.58}/searcharray.egg-info/requires.txt +0 -0
  49. {searcharray-0.0.56 → searcharray-0.0.58}/searcharray.egg-info/top_level.txt +0 -0
  50. {searcharray-0.0.56 → searcharray-0.0.58}/setup.cfg +0 -0
  51. {searcharray-0.0.56 → searcharray-0.0.58}/test/test_bitcount64.py +0 -0
  52. {searcharray-0.0.56 → searcharray-0.0.58}/test/test_minmax_posns.py +0 -0
  53. {searcharray-0.0.56 → searcharray-0.0.58}/test/test_msmarco.py +0 -0
  54. {searcharray-0.0.56 → searcharray-0.0.58}/test/test_phrase_matches.py +0 -0
  55. {searcharray-0.0.56 → searcharray-0.0.58}/test/test_search.py +0 -0
  56. {searcharray-0.0.56 → searcharray-0.0.58}/test/test_similarity.py +0 -0
  57. {searcharray-0.0.56 → searcharray-0.0.58}/test/test_slop_matches.py +0 -0
  58. {searcharray-0.0.56 → searcharray-0.0.58}/test/test_snp_ops.py +0 -0
  59. {searcharray-0.0.56 → searcharray-0.0.58}/test/test_solr.py +0 -0
  60. {searcharray-0.0.56 → searcharray-0.0.58}/test/test_utils.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: searcharray
3
- Version: 0.0.56
3
+ Version: 0.0.58
4
4
  Summary: Searchable pandas text extension arrays for prototyping search
5
5
  Home-page: https://github.com/softwaredoug/searcharray
6
6
  Author: Doug Turnbull
@@ -3,3 +3,6 @@
3
3
  # https://pip.pypa.io/en/stable/reference/pip/#pep-517-and-518-support
4
4
  requires = ["setuptools>=43.0.0", "wheel", "numpy", "cython"]
5
5
  build-backend = "setuptools.build_meta"
6
+
7
+ [tool.cython-lint]
8
+ max-line-length = 120
@@ -121,7 +121,7 @@ def _tokenize_batch(array, tokenizer, term_dict, batch_size, batch_beg, truncate
121
121
 
122
122
  # Encode posns to bit array
123
123
  logger.info("Encoding positions to bit array")
124
- posns = PosnBitArrayFromFlatBuilder(terms_w_posns)
124
+ posns = PosnBitArrayFromFlatBuilder(terms_w_posns, max_doc_id=(batch_beg + len(array) - 1))
125
125
  bit_posns = posns.build()
126
126
 
127
127
  if np.any(doc_lens > MAX_POSN):
@@ -158,8 +158,9 @@ class PosnBitArrayFromFlatBuilder:
158
158
 
159
159
  """
160
160
 
161
- def __init__(self, flat_array: np.ndarray):
161
+ def __init__(self, flat_array: np.ndarray, max_doc_id: int) -> None:
162
162
  self.flat_array = flat_array
163
+ self.max_doc_id = max_doc_id
163
164
 
164
165
  def build(self):
165
166
  """Slice the flat array into a 2d array of doc ids and posns."""
@@ -179,7 +180,9 @@ class PosnBitArrayFromFlatBuilder:
179
180
  sliced = encoded[beg_idx:end_idx]
180
181
  encoded_term_posns[term_ids[into_terms]] = sliced
181
182
 
182
- return PosnBitArray(encoded_term_posns, self.flat_array[1].max())
183
+ largest_doc_id_with_term = np.max(self.flat_array[1])
184
+ assert self.max_doc_id >= largest_doc_id_with_term
185
+ return PosnBitArray(encoded_term_posns, self.max_doc_id)
183
186
 
184
187
 
185
188
  class PosnBitArrayBuilder:
@@ -390,6 +393,7 @@ class PosnBitArray:
390
393
  min_posn: Optional[int] = None,
391
394
  max_posn: Optional[int] = None) -> np.ndarray:
392
395
  phrase_freqs = self.empty_buffer()
396
+
393
397
  if len(term_ids) < 2:
394
398
  raise ValueError("Must have at least two terms")
395
399
  if phrase_freqs.shape[0] == self.max_doc_id + 1 and min_posn is None and max_posn is None and doc_ids is None:
@@ -464,6 +464,20 @@ class SearchArray(ExtensionArray):
464
464
  empties = self.doc_lens == 0
465
465
  return empties
466
466
 
467
+ def unique(self):
468
+ """This is a hack for colab which wants to visit every element of the array."""
469
+ logger.warning("Unique called on SearchArray. This is not supported.")
470
+ return self[:]
471
+
472
+ def __iter__(self):
473
+ if len(self) > 10000:
474
+ warning_text = """Iterating over SearchArray is very slow and not reccomended.
475
+
476
+ If you're looping a dataframe, slice out the non-SearchArray columns first."""
477
+ logger.warning(warning_text)
478
+ warnings.warn(warning_text)
479
+ return super().__iter__()
480
+
467
481
  def take(self, indices, allow_fill=False, fill_value=None):
468
482
  # Want to take rows of term freqs
469
483
  row_indices = np.arange(len(self.term_mat.rows))