searcharray 0.0.52__tar.gz → 0.0.54__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {searcharray-0.0.52 → searcharray-0.0.54}/PKG-INFO +1 -1
- {searcharray-0.0.52 → searcharray-0.0.54}/searcharray/phrase/bigram_freqs.py +2 -9
- {searcharray-0.0.52 → searcharray-0.0.54}/searcharray/postings.py +34 -0
- {searcharray-0.0.52 → searcharray-0.0.54}/searcharray/roaringish/__init__.py +1 -1
- {searcharray-0.0.52 → searcharray-0.0.54}/searcharray/roaringish/roaringish_ops.c +117 -117
- {searcharray-0.0.52 → searcharray-0.0.54}/searcharray/roaringish/snp_ops.c +3797 -2749
- {searcharray-0.0.52 → searcharray-0.0.54}/searcharray/roaringish/snp_ops.pyx +193 -149
- {searcharray-0.0.52 → searcharray-0.0.54}/searcharray/roaringish/spans.c +117 -117
- {searcharray-0.0.52 → searcharray-0.0.54}/searcharray.egg-info/PKG-INFO +1 -1
- {searcharray-0.0.52 → searcharray-0.0.54}/setup.py +1 -1
- {searcharray-0.0.52 → searcharray-0.0.54}/test/test_msmarco.py +99 -141
- {searcharray-0.0.52 → searcharray-0.0.54}/test/test_snp_ops.py +36 -5
- {searcharray-0.0.52 → searcharray-0.0.54}/test/test_utils.py +3 -2
- {searcharray-0.0.52 → searcharray-0.0.54}/README.md +0 -0
- {searcharray-0.0.52 → searcharray-0.0.54}/pyproject.toml +0 -0
- {searcharray-0.0.52 → searcharray-0.0.54}/searcharray/__init__.py +0 -0
- {searcharray-0.0.52 → searcharray-0.0.54}/searcharray/indexing.py +0 -0
- {searcharray-0.0.52 → searcharray-0.0.54}/searcharray/phrase/__init__.py +0 -0
- {searcharray-0.0.52 → searcharray-0.0.54}/searcharray/phrase/middle_out.py +0 -0
- {searcharray-0.0.52 → searcharray-0.0.54}/searcharray/phrase/posn_diffs.py +0 -0
- {searcharray-0.0.52 → searcharray-0.0.54}/searcharray/phrase/scan_merge.py +0 -0
- {searcharray-0.0.52 → searcharray-0.0.54}/searcharray/phrase/spans.py +0 -0
- {searcharray-0.0.52 → searcharray-0.0.54}/searcharray/roaringish/roaringish.py +0 -0
- {searcharray-0.0.52 → searcharray-0.0.54}/searcharray/roaringish/roaringish_ops.pyx +0 -0
- {searcharray-0.0.52 → searcharray-0.0.54}/searcharray/roaringish/snp_ops.pxd +0 -0
- {searcharray-0.0.52 → searcharray-0.0.54}/searcharray/roaringish/spans.pyx +0 -0
- {searcharray-0.0.52 → searcharray-0.0.54}/searcharray/similarity.py +0 -0
- {searcharray-0.0.52 → searcharray-0.0.54}/searcharray/solr.py +0 -0
- {searcharray-0.0.52 → searcharray-0.0.54}/searcharray/term_dict.py +0 -0
- {searcharray-0.0.52 → searcharray-0.0.54}/searcharray/utils/__init__.py +0 -0
- {searcharray-0.0.52 → searcharray-0.0.54}/searcharray/utils/bitcount.py +0 -0
- {searcharray-0.0.52 → searcharray-0.0.54}/searcharray/utils/mat_set.py +0 -0
- {searcharray-0.0.52 → searcharray-0.0.54}/searcharray/utils/row_viewable_matrix.py +0 -0
- {searcharray-0.0.52 → searcharray-0.0.54}/searcharray/utils/sort.py +0 -0
- {searcharray-0.0.52 → searcharray-0.0.54}/searcharray.egg-info/SOURCES.txt +0 -0
- {searcharray-0.0.52 → searcharray-0.0.54}/searcharray.egg-info/dependency_links.txt +0 -0
- {searcharray-0.0.52 → searcharray-0.0.54}/searcharray.egg-info/entry_points.txt +0 -0
- {searcharray-0.0.52 → searcharray-0.0.54}/searcharray.egg-info/requires.txt +0 -0
- {searcharray-0.0.52 → searcharray-0.0.54}/searcharray.egg-info/top_level.txt +0 -0
- {searcharray-0.0.52 → searcharray-0.0.54}/setup.cfg +0 -0
- {searcharray-0.0.52 → searcharray-0.0.54}/test/test_bitcount64.py +0 -0
- {searcharray-0.0.52 → searcharray-0.0.54}/test/test_extension_array.py +0 -0
- {searcharray-0.0.52 → searcharray-0.0.54}/test/test_minmax_posns.py +0 -0
- {searcharray-0.0.52 → searcharray-0.0.54}/test/test_phrase_matches.py +0 -0
- {searcharray-0.0.52 → searcharray-0.0.54}/test/test_search.py +0 -0
- {searcharray-0.0.52 → searcharray-0.0.54}/test/test_similarity.py +0 -0
- {searcharray-0.0.52 → searcharray-0.0.54}/test/test_slop_matches.py +0 -0
- {searcharray-0.0.52 → searcharray-0.0.54}/test/test_solr.py +0 -0
- {searcharray-0.0.52 → searcharray-0.0.54}/test/test_tmdb.py +0 -0
|
@@ -5,7 +5,7 @@ from searcharray.roaringish import RoaringishEncoder
|
|
|
5
5
|
import logging
|
|
6
6
|
from enum import Enum
|
|
7
7
|
|
|
8
|
-
from searcharray.roaringish import intersect, popcount64, merge
|
|
8
|
+
from searcharray.roaringish import intersect, popcount64, merge, popcount_reduce_at
|
|
9
9
|
|
|
10
10
|
|
|
11
11
|
logger = logging.getLogger(__name__)
|
|
@@ -144,14 +144,7 @@ def _inner_bigram_freqs(lhs: np.ndarray, rhs: np.ndarray,
|
|
|
144
144
|
lhs_next = overlap_bits.copy()
|
|
145
145
|
lhs_next |= (lhs_int & (encoder.key_mask | encoder.payload_msb_mask))
|
|
146
146
|
|
|
147
|
-
|
|
148
|
-
if np.any(matches2):
|
|
149
|
-
transitions = np.argwhere(np.diff(lhs_doc_ids[matches2]) != 0).flatten() + 1
|
|
150
|
-
transitions = np.insert(transitions, 0, 0)
|
|
151
|
-
counted_bits = popcount64(overlap_bits[matches2])
|
|
152
|
-
reduced = np.add.reduceat(counted_bits,
|
|
153
|
-
transitions)
|
|
154
|
-
phrase_freqs[np.unique(lhs_doc_ids[matches2])] += reduced
|
|
147
|
+
popcount_reduce_at(lhs_doc_ids, overlap_bits, phrase_freqs)
|
|
155
148
|
return phrase_freqs, (lhs_next, rhs_next)
|
|
156
149
|
|
|
157
150
|
|
|
@@ -525,6 +525,40 @@ class SearchArray(ExtensionArray):
|
|
|
525
525
|
else:
|
|
526
526
|
raise TypeError("Expected a string or list of strings for phrases")
|
|
527
527
|
|
|
528
|
+
def memory_report(self):
|
|
529
|
+
"""Return a string with memory usage information."""
|
|
530
|
+
term_mat_bytes = self.term_mat.nbytes
|
|
531
|
+
posns_bytes = self.posns.nbytes
|
|
532
|
+
term_dict_bytes = self.term_dict.nbytes
|
|
533
|
+
num_terms = len(self.term_dict)
|
|
534
|
+
top_10_terms_size = []
|
|
535
|
+
|
|
536
|
+
for i in range(10):
|
|
537
|
+
term_len = len(self.posns.encoded_term_posns[i])
|
|
538
|
+
term = self.term_dict.get_term(i)
|
|
539
|
+
top_10_terms_size.append((term, term_len))
|
|
540
|
+
|
|
541
|
+
return f"""
|
|
542
|
+
SearchArray Memory Report
|
|
543
|
+
-------------------------
|
|
544
|
+
Number of Terms: {num_terms}
|
|
545
|
+
-------------------------
|
|
546
|
+
Term Matrix: {term_mat_bytes}
|
|
547
|
+
Positions: {posns_bytes}
|
|
548
|
+
Term Dictionary: {term_dict_bytes}
|
|
549
|
+
--------------------------
|
|
550
|
+
Term 0: {top_10_terms_size[0]}
|
|
551
|
+
Term 1: {top_10_terms_size[1]}
|
|
552
|
+
Term 2: {top_10_terms_size[2]}
|
|
553
|
+
Term 3: {top_10_terms_size[3]}
|
|
554
|
+
Term 4: {top_10_terms_size[4]}
|
|
555
|
+
Term 5: {top_10_terms_size[5]}
|
|
556
|
+
Term 6: {top_10_terms_size[6]}
|
|
557
|
+
Term 7: {top_10_terms_size[7]}
|
|
558
|
+
Term 8: {top_10_terms_size[8]}
|
|
559
|
+
Term 9: {top_10_terms_size[9]}
|
|
560
|
+
"""
|
|
561
|
+
|
|
528
562
|
# ***********************************************************
|
|
529
563
|
# Search functionality
|
|
530
564
|
# ***********************************************************
|
|
@@ -1,4 +1,4 @@
|
|
|
1
1
|
from searcharray.roaringish.roaringish import RoaringishEncoder, convert_keys # noqa
|
|
2
2
|
from searcharray.roaringish.snp_ops import popcount64 # noqa
|
|
3
|
-
from searcharray.roaringish.snp_ops import merge, intersect, unique, adjacent # noqa
|
|
3
|
+
from searcharray.roaringish.snp_ops import merge, intersect, unique, adjacent, popcount_reduce_at # noqa
|
|
4
4
|
from searcharray.roaringish.spans import span_search # noqa
|