searcharray 0.0.52__tar.gz → 0.0.54__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. {searcharray-0.0.52 → searcharray-0.0.54}/PKG-INFO +1 -1
  2. {searcharray-0.0.52 → searcharray-0.0.54}/searcharray/phrase/bigram_freqs.py +2 -9
  3. {searcharray-0.0.52 → searcharray-0.0.54}/searcharray/postings.py +34 -0
  4. {searcharray-0.0.52 → searcharray-0.0.54}/searcharray/roaringish/__init__.py +1 -1
  5. {searcharray-0.0.52 → searcharray-0.0.54}/searcharray/roaringish/roaringish_ops.c +117 -117
  6. {searcharray-0.0.52 → searcharray-0.0.54}/searcharray/roaringish/snp_ops.c +3797 -2749
  7. {searcharray-0.0.52 → searcharray-0.0.54}/searcharray/roaringish/snp_ops.pyx +193 -149
  8. {searcharray-0.0.52 → searcharray-0.0.54}/searcharray/roaringish/spans.c +117 -117
  9. {searcharray-0.0.52 → searcharray-0.0.54}/searcharray.egg-info/PKG-INFO +1 -1
  10. {searcharray-0.0.52 → searcharray-0.0.54}/setup.py +1 -1
  11. {searcharray-0.0.52 → searcharray-0.0.54}/test/test_msmarco.py +99 -141
  12. {searcharray-0.0.52 → searcharray-0.0.54}/test/test_snp_ops.py +36 -5
  13. {searcharray-0.0.52 → searcharray-0.0.54}/test/test_utils.py +3 -2
  14. {searcharray-0.0.52 → searcharray-0.0.54}/README.md +0 -0
  15. {searcharray-0.0.52 → searcharray-0.0.54}/pyproject.toml +0 -0
  16. {searcharray-0.0.52 → searcharray-0.0.54}/searcharray/__init__.py +0 -0
  17. {searcharray-0.0.52 → searcharray-0.0.54}/searcharray/indexing.py +0 -0
  18. {searcharray-0.0.52 → searcharray-0.0.54}/searcharray/phrase/__init__.py +0 -0
  19. {searcharray-0.0.52 → searcharray-0.0.54}/searcharray/phrase/middle_out.py +0 -0
  20. {searcharray-0.0.52 → searcharray-0.0.54}/searcharray/phrase/posn_diffs.py +0 -0
  21. {searcharray-0.0.52 → searcharray-0.0.54}/searcharray/phrase/scan_merge.py +0 -0
  22. {searcharray-0.0.52 → searcharray-0.0.54}/searcharray/phrase/spans.py +0 -0
  23. {searcharray-0.0.52 → searcharray-0.0.54}/searcharray/roaringish/roaringish.py +0 -0
  24. {searcharray-0.0.52 → searcharray-0.0.54}/searcharray/roaringish/roaringish_ops.pyx +0 -0
  25. {searcharray-0.0.52 → searcharray-0.0.54}/searcharray/roaringish/snp_ops.pxd +0 -0
  26. {searcharray-0.0.52 → searcharray-0.0.54}/searcharray/roaringish/spans.pyx +0 -0
  27. {searcharray-0.0.52 → searcharray-0.0.54}/searcharray/similarity.py +0 -0
  28. {searcharray-0.0.52 → searcharray-0.0.54}/searcharray/solr.py +0 -0
  29. {searcharray-0.0.52 → searcharray-0.0.54}/searcharray/term_dict.py +0 -0
  30. {searcharray-0.0.52 → searcharray-0.0.54}/searcharray/utils/__init__.py +0 -0
  31. {searcharray-0.0.52 → searcharray-0.0.54}/searcharray/utils/bitcount.py +0 -0
  32. {searcharray-0.0.52 → searcharray-0.0.54}/searcharray/utils/mat_set.py +0 -0
  33. {searcharray-0.0.52 → searcharray-0.0.54}/searcharray/utils/row_viewable_matrix.py +0 -0
  34. {searcharray-0.0.52 → searcharray-0.0.54}/searcharray/utils/sort.py +0 -0
  35. {searcharray-0.0.52 → searcharray-0.0.54}/searcharray.egg-info/SOURCES.txt +0 -0
  36. {searcharray-0.0.52 → searcharray-0.0.54}/searcharray.egg-info/dependency_links.txt +0 -0
  37. {searcharray-0.0.52 → searcharray-0.0.54}/searcharray.egg-info/entry_points.txt +0 -0
  38. {searcharray-0.0.52 → searcharray-0.0.54}/searcharray.egg-info/requires.txt +0 -0
  39. {searcharray-0.0.52 → searcharray-0.0.54}/searcharray.egg-info/top_level.txt +0 -0
  40. {searcharray-0.0.52 → searcharray-0.0.54}/setup.cfg +0 -0
  41. {searcharray-0.0.52 → searcharray-0.0.54}/test/test_bitcount64.py +0 -0
  42. {searcharray-0.0.52 → searcharray-0.0.54}/test/test_extension_array.py +0 -0
  43. {searcharray-0.0.52 → searcharray-0.0.54}/test/test_minmax_posns.py +0 -0
  44. {searcharray-0.0.52 → searcharray-0.0.54}/test/test_phrase_matches.py +0 -0
  45. {searcharray-0.0.52 → searcharray-0.0.54}/test/test_search.py +0 -0
  46. {searcharray-0.0.52 → searcharray-0.0.54}/test/test_similarity.py +0 -0
  47. {searcharray-0.0.52 → searcharray-0.0.54}/test/test_slop_matches.py +0 -0
  48. {searcharray-0.0.52 → searcharray-0.0.54}/test/test_solr.py +0 -0
  49. {searcharray-0.0.52 → searcharray-0.0.54}/test/test_tmdb.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: searcharray
3
- Version: 0.0.52
3
+ Version: 0.0.54
4
4
  Summary: Searchable pandas text extension arrays for prototyping search
5
5
  Home-page: https://github.com/softwaredoug/searcharray
6
6
  Author: Doug Turnbull
@@ -5,7 +5,7 @@ from searcharray.roaringish import RoaringishEncoder
5
5
  import logging
6
6
  from enum import Enum
7
7
 
8
- from searcharray.roaringish import intersect, popcount64, merge
8
+ from searcharray.roaringish import intersect, popcount64, merge, popcount_reduce_at
9
9
 
10
10
 
11
11
  logger = logging.getLogger(__name__)
@@ -144,14 +144,7 @@ def _inner_bigram_freqs(lhs: np.ndarray, rhs: np.ndarray,
144
144
  lhs_next = overlap_bits.copy()
145
145
  lhs_next |= (lhs_int & (encoder.key_mask | encoder.payload_msb_mask))
146
146
 
147
- matches2 = overlap_bits > 0
148
- if np.any(matches2):
149
- transitions = np.argwhere(np.diff(lhs_doc_ids[matches2]) != 0).flatten() + 1
150
- transitions = np.insert(transitions, 0, 0)
151
- counted_bits = popcount64(overlap_bits[matches2])
152
- reduced = np.add.reduceat(counted_bits,
153
- transitions)
154
- phrase_freqs[np.unique(lhs_doc_ids[matches2])] += reduced
147
+ popcount_reduce_at(lhs_doc_ids, overlap_bits, phrase_freqs)
155
148
  return phrase_freqs, (lhs_next, rhs_next)
156
149
 
157
150
 
@@ -525,6 +525,40 @@ class SearchArray(ExtensionArray):
525
525
  else:
526
526
  raise TypeError("Expected a string or list of strings for phrases")
527
527
 
528
+ def memory_report(self):
529
+ """Return a string with memory usage information."""
530
+ term_mat_bytes = self.term_mat.nbytes
531
+ posns_bytes = self.posns.nbytes
532
+ term_dict_bytes = self.term_dict.nbytes
533
+ num_terms = len(self.term_dict)
534
+ top_10_terms_size = []
535
+
536
+ for i in range(10):
537
+ term_len = len(self.posns.encoded_term_posns[i])
538
+ term = self.term_dict.get_term(i)
539
+ top_10_terms_size.append((term, term_len))
540
+
541
+ return f"""
542
+ SearchArray Memory Report
543
+ -------------------------
544
+ Number of Terms: {num_terms}
545
+ -------------------------
546
+ Term Matrix: {term_mat_bytes}
547
+ Positions: {posns_bytes}
548
+ Term Dictionary: {term_dict_bytes}
549
+ --------------------------
550
+ Term 0: {top_10_terms_size[0]}
551
+ Term 1: {top_10_terms_size[1]}
552
+ Term 2: {top_10_terms_size[2]}
553
+ Term 3: {top_10_terms_size[3]}
554
+ Term 4: {top_10_terms_size[4]}
555
+ Term 5: {top_10_terms_size[5]}
556
+ Term 6: {top_10_terms_size[6]}
557
+ Term 7: {top_10_terms_size[7]}
558
+ Term 8: {top_10_terms_size[8]}
559
+ Term 9: {top_10_terms_size[9]}
560
+ """
561
+
528
562
  # ***********************************************************
529
563
  # Search functionality
530
564
  # ***********************************************************
@@ -1,4 +1,4 @@
1
1
  from searcharray.roaringish.roaringish import RoaringishEncoder, convert_keys # noqa
2
2
  from searcharray.roaringish.snp_ops import popcount64 # noqa
3
- from searcharray.roaringish.snp_ops import merge, intersect, unique, adjacent # noqa
3
+ from searcharray.roaringish.snp_ops import merge, intersect, unique, adjacent, popcount_reduce_at # noqa
4
4
  from searcharray.roaringish.spans import span_search # noqa