searcharray 0.0.8__tar.gz → 0.0.10__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {searcharray-0.0.8 → searcharray-0.0.10}/PKG-INFO +1 -1
- {searcharray-0.0.8 → searcharray-0.0.10}/searcharray/postings.py +60 -14
- {searcharray-0.0.8 → searcharray-0.0.10}/searcharray.egg-info/PKG-INFO +1 -1
- {searcharray-0.0.8 → searcharray-0.0.10}/setup.py +1 -1
- {searcharray-0.0.8 → searcharray-0.0.10}/test/test_phrase_matches.py +16 -1
- {searcharray-0.0.8 → searcharray-0.0.10}/README.md +0 -0
- {searcharray-0.0.8 → searcharray-0.0.10}/pyproject.toml +0 -0
- {searcharray-0.0.8 → searcharray-0.0.10}/searcharray/__init__.py +0 -0
- {searcharray-0.0.8 → searcharray-0.0.10}/searcharray/stemmers.py +0 -0
- {searcharray-0.0.8 → searcharray-0.0.10}/searcharray/term_dict.py +0 -0
- {searcharray-0.0.8 → searcharray-0.0.10}/searcharray.egg-info/SOURCES.txt +0 -0
- {searcharray-0.0.8 → searcharray-0.0.10}/searcharray.egg-info/dependency_links.txt +0 -0
- {searcharray-0.0.8 → searcharray-0.0.10}/searcharray.egg-info/entry_points.txt +0 -0
- {searcharray-0.0.8 → searcharray-0.0.10}/searcharray.egg-info/requires.txt +0 -0
- {searcharray-0.0.8 → searcharray-0.0.10}/searcharray.egg-info/top_level.txt +0 -0
- {searcharray-0.0.8 → searcharray-0.0.10}/setup.cfg +0 -0
- {searcharray-0.0.8 → searcharray-0.0.10}/test/test_extension_array.py +0 -0
- {searcharray-0.0.8 → searcharray-0.0.10}/test/test_msmarco.py +0 -0
- {searcharray-0.0.8 → searcharray-0.0.10}/test/test_tmdb.py +0 -0
- {searcharray-0.0.8 → searcharray-0.0.10}/test/test_utils.py +0 -0
|
@@ -19,7 +19,7 @@ from searcharray.term_dict import TermDict, TermMissingError
|
|
|
19
19
|
from scipy.sparse import dok_matrix, csr_matrix
|
|
20
20
|
|
|
21
21
|
root = logging.getLogger()
|
|
22
|
-
root.setLevel(logging.
|
|
22
|
+
root.setLevel(logging.WARNING)
|
|
23
23
|
|
|
24
24
|
# import sys
|
|
25
25
|
# handler = logging.StreamHandler(sys.stdout)
|
|
@@ -29,6 +29,19 @@ root.setLevel(logging.INFO)
|
|
|
29
29
|
# root.addHandler(handler)
|
|
30
30
|
|
|
31
31
|
|
|
32
|
+
def vstack_with_pad(arrays, width=10, pad=-100):
|
|
33
|
+
vstacked = np.zeros((len(arrays), width), dtype=arrays[0].dtype) + pad
|
|
34
|
+
for idx, array in enumerate(arrays):
|
|
35
|
+
# Resize if needed, padding with pad
|
|
36
|
+
if len(array) > width:
|
|
37
|
+
logging.info(f"Resizing from {width} to {len(array)}")
|
|
38
|
+
vstack_padded = np.pad(vstacked, ((0, 0), (0, len(array) - width)), constant_values=pad)
|
|
39
|
+
width = len(array)
|
|
40
|
+
vstacked = vstack_padded
|
|
41
|
+
vstacked[idx, :len(array)] = array
|
|
42
|
+
return vstacked
|
|
43
|
+
|
|
44
|
+
|
|
32
45
|
class PostingsRow:
|
|
33
46
|
"""Wrapper around a row of a postings matrix.
|
|
34
47
|
|
|
@@ -742,11 +755,56 @@ class PostingsArray(ExtensionArray):
|
|
|
742
755
|
return mask
|
|
743
756
|
|
|
744
757
|
def phrase_freq(self, tokens, slop=1):
|
|
758
|
+
from time import perf_counter
|
|
759
|
+
start = perf_counter()
|
|
760
|
+
|
|
761
|
+
mask = self.and_query(tokens)
|
|
762
|
+
|
|
763
|
+
if np.sum(mask) == 0:
|
|
764
|
+
return mask
|
|
765
|
+
|
|
766
|
+
# Any identical terms, default to shitty algo for now
|
|
767
|
+
if len(tokens) != len(set(tokens)):
|
|
768
|
+
return self.phrase_freq_shitty(tokens, slop=slop)
|
|
769
|
+
|
|
770
|
+
# Iterate each phrase with its next term
|
|
771
|
+
prior_term = tokens[0]
|
|
772
|
+
prior_posns = self.positions(prior_term, mask)
|
|
773
|
+
phrase_freqs = np.zeros(len(self))
|
|
774
|
+
for term in tokens[1:]:
|
|
775
|
+
term_posns = self.positions(term, mask)
|
|
776
|
+
|
|
777
|
+
assert len(prior_posns) == len(term_posns)
|
|
778
|
+
# RIPE for optimization -> vectorize in C
|
|
779
|
+
bigram_freqs = np.zeros(len(term_posns))
|
|
780
|
+
cont_posns = []
|
|
781
|
+
term_start = perf_counter()
|
|
782
|
+
for idx in range(len(term_posns)):
|
|
783
|
+
# Find insert position of every next term in prior term's positions
|
|
784
|
+
# Intuition:
|
|
785
|
+
# https://colab.research.google.com/drive/1EeqHYuCiqyptd-awS67Re78pqVdTfH4A
|
|
786
|
+
ins_posns = np.searchsorted(prior_posns[idx], term_posns[idx], side='right')
|
|
787
|
+
prior_adjacents = prior_posns[idx][ins_posns - 1]
|
|
788
|
+
adjacents = term_posns[idx] - prior_adjacents
|
|
789
|
+
bigram_freqs[idx] = np.sum(adjacents <= slop)
|
|
790
|
+
|
|
791
|
+
cont_indices = np.argwhere(adjacents <= slop).flatten()
|
|
792
|
+
cont_posn = term_posns[idx][cont_indices]
|
|
793
|
+
cont_posns.append(cont_posn)
|
|
794
|
+
phrase_freqs[mask] = bigram_freqs
|
|
795
|
+
prior_term = term
|
|
796
|
+
prior_posns = cont_posns
|
|
797
|
+
logging.info(f"Term Time: {perf_counter() - term_start:.4f}s")
|
|
798
|
+
|
|
799
|
+
logging.info(f"Phrase Search Time: {perf_counter() - start:.4f}s")
|
|
800
|
+
|
|
801
|
+
return phrase_freqs
|
|
802
|
+
|
|
803
|
+
def phrase_freq_shitty(self, tokens, slop=1):
|
|
745
804
|
"""Return number of occurences of a phrase."""
|
|
746
805
|
from time import perf_counter
|
|
747
806
|
# Start with docs with all terms
|
|
748
807
|
start = perf_counter()
|
|
749
|
-
pad = -1000
|
|
750
808
|
mask = self.and_query(tokens)
|
|
751
809
|
# For detailed documentation of this algorithm, see this ChatGPT4 discussion
|
|
752
810
|
# https://chat.openai.com/share/31affaad-dc91-4757-b31c-e85bdb5a0eb6
|
|
@@ -754,18 +812,6 @@ class PostingsArray(ExtensionArray):
|
|
|
754
812
|
if np.sum(mask) == 0:
|
|
755
813
|
return mask
|
|
756
814
|
|
|
757
|
-
def vstack_with_pad(arrays, width=10):
|
|
758
|
-
vstacked = np.zeros((len(arrays), width), dtype=arrays[0].dtype) + pad
|
|
759
|
-
for idx, array in enumerate(arrays):
|
|
760
|
-
# Resize if needed, padding with pad
|
|
761
|
-
if len(array) > width:
|
|
762
|
-
logging.info(f"Resizing from {width} to {len(array)}")
|
|
763
|
-
vstack_padded = np.pad(vstacked, ((0, 0), (0, len(array) - width)), constant_values=pad)
|
|
764
|
-
width = len(array)
|
|
765
|
-
vstacked = vstack_padded
|
|
766
|
-
vstacked[idx, :len(array)] = array
|
|
767
|
-
return vstacked
|
|
768
|
-
|
|
769
815
|
# Pad for easy difference computation
|
|
770
816
|
term_posns = []
|
|
771
817
|
for term in tokens:
|
|
@@ -36,7 +36,7 @@ setup(
|
|
|
36
36
|
# For a discussion on single-sourcing the version across setup.py and the
|
|
37
37
|
# project code, see
|
|
38
38
|
# https://packaging.python.org/guides/single-sourcing-package-version/
|
|
39
|
-
version="0.0.
|
|
39
|
+
version="0.0.10", # Required
|
|
40
40
|
# This is a one-line description or tagline of what your project does. This
|
|
41
41
|
# corresponds to the "Summary" metadata field:
|
|
42
42
|
# https://packaging.python.org/specifications/core-metadata/#summary
|
|
@@ -38,6 +38,13 @@ perf_scenarios = {
|
|
|
38
38
|
"phrase": ["foo", "bar"],
|
|
39
39
|
"expected": [True, False, False, False, False, True] * 100000,
|
|
40
40
|
},
|
|
41
|
+
"many_docs_and_positions": {
|
|
42
|
+
"docs": lambda: PostingsArray.index([" ".join(["foo bar bar baz foo foo bar foo"] * 100),
|
|
43
|
+
" ".join(["what is the foo bar doing in the bar foo?"] * 100)] * 100000),
|
|
44
|
+
"phrase": ["foo", "bar"],
|
|
45
|
+
"expected": [True, True] * 100000
|
|
46
|
+
}
|
|
47
|
+
|
|
41
48
|
}
|
|
42
49
|
|
|
43
50
|
|
|
@@ -52,6 +59,11 @@ scenarios = {
|
|
|
52
59
|
"phrase": ["foo", "bar"],
|
|
53
60
|
"expected": [1, 0, 0, 0] * 25,
|
|
54
61
|
},
|
|
62
|
+
"term_repeats": {
|
|
63
|
+
"docs": lambda: PostingsArray.index(["foo foo bar bar baz", "data2", "data3 bar", "bunny funny wunny"] * 25),
|
|
64
|
+
"phrase": ["foo", "bar"],
|
|
65
|
+
"expected": [1, 0, 0, 0] * 25,
|
|
66
|
+
},
|
|
55
67
|
"multi_term_one_doc": {
|
|
56
68
|
"docs": lambda: PostingsArray.index(["foo bar bar bar foo", "data2", "data3 bar", "bunny funny wunny"] * 25),
|
|
57
69
|
"phrase": ["foo", "bar"],
|
|
@@ -150,6 +162,9 @@ def test_phrase(docs, phrase, expected):
|
|
|
150
162
|
matches = docs.match(phrase)
|
|
151
163
|
assert (term_freqs == expected).all()
|
|
152
164
|
assert (matches == expected_matches).all()
|
|
165
|
+
if len(phrase) > 1:
|
|
166
|
+
phrase_matches = docs.phrase_freq(phrase)
|
|
167
|
+
assert (expected == phrase_matches).all()
|
|
153
168
|
assert (docs == docs_before).all(), "The phrase_match method should not modify the original array"
|
|
154
169
|
bm25 = docs.bm25(phrase)
|
|
155
170
|
assert (np.argsort(bm25) == np.argsort(expected)).all()
|
|
@@ -161,7 +176,7 @@ def test_phrase_performance(docs, phrase, expected):
|
|
|
161
176
|
start = perf_counter()
|
|
162
177
|
docs = docs()
|
|
163
178
|
matches = docs.match(phrase)
|
|
164
|
-
print(f"phrase_match took {perf_counter() - start} seconds | {len(docs)} docs")
|
|
179
|
+
print(f"phrase_match 1 took {perf_counter() - start} seconds | {len(docs)} docs")
|
|
165
180
|
assert (matches == expected).all()
|
|
166
181
|
|
|
167
182
|
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|