searcharray 0.0.8__tar.gz → 0.0.10__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: searcharray
3
- Version: 0.0.8
3
+ Version: 0.0.10
4
4
  Summary: Searchable pandas text extension arrays for prototyping search
5
5
  Home-page: https://github.com/softwaredoug/searcharray
6
6
  Author: Doug Turnbull
@@ -19,7 +19,7 @@ from searcharray.term_dict import TermDict, TermMissingError
19
19
  from scipy.sparse import dok_matrix, csr_matrix
20
20
 
21
21
  root = logging.getLogger()
22
- root.setLevel(logging.INFO)
22
+ root.setLevel(logging.WARNING)
23
23
 
24
24
  # import sys
25
25
  # handler = logging.StreamHandler(sys.stdout)
@@ -29,6 +29,19 @@ root.setLevel(logging.INFO)
29
29
  # root.addHandler(handler)
30
30
 
31
31
 
32
+ def vstack_with_pad(arrays, width=10, pad=-100):
33
+ vstacked = np.zeros((len(arrays), width), dtype=arrays[0].dtype) + pad
34
+ for idx, array in enumerate(arrays):
35
+ # Resize if needed, padding with pad
36
+ if len(array) > width:
37
+ logging.info(f"Resizing from {width} to {len(array)}")
38
+ vstack_padded = np.pad(vstacked, ((0, 0), (0, len(array) - width)), constant_values=pad)
39
+ width = len(array)
40
+ vstacked = vstack_padded
41
+ vstacked[idx, :len(array)] = array
42
+ return vstacked
43
+
44
+
32
45
  class PostingsRow:
33
46
  """Wrapper around a row of a postings matrix.
34
47
 
@@ -742,11 +755,56 @@ class PostingsArray(ExtensionArray):
742
755
  return mask
743
756
 
744
757
  def phrase_freq(self, tokens, slop=1):
758
+ from time import perf_counter
759
+ start = perf_counter()
760
+
761
+ mask = self.and_query(tokens)
762
+
763
+ if np.sum(mask) == 0:
764
+ return mask
765
+
766
+ # Any identical terms, default to shitty algo for now
767
+ if len(tokens) != len(set(tokens)):
768
+ return self.phrase_freq_shitty(tokens, slop=slop)
769
+
770
+ # Iterate each phrase with its next term
771
+ prior_term = tokens[0]
772
+ prior_posns = self.positions(prior_term, mask)
773
+ phrase_freqs = np.zeros(len(self))
774
+ for term in tokens[1:]:
775
+ term_posns = self.positions(term, mask)
776
+
777
+ assert len(prior_posns) == len(term_posns)
778
+ # RIPE for optimization -> vectorize in C
779
+ bigram_freqs = np.zeros(len(term_posns))
780
+ cont_posns = []
781
+ term_start = perf_counter()
782
+ for idx in range(len(term_posns)):
783
+ # Find insert position of every next term in prior term's positions
784
+ # Intuition:
785
+ # https://colab.research.google.com/drive/1EeqHYuCiqyptd-awS67Re78pqVdTfH4A
786
+ ins_posns = np.searchsorted(prior_posns[idx], term_posns[idx], side='right')
787
+ prior_adjacents = prior_posns[idx][ins_posns - 1]
788
+ adjacents = term_posns[idx] - prior_adjacents
789
+ bigram_freqs[idx] = np.sum(adjacents <= slop)
790
+
791
+ cont_indices = np.argwhere(adjacents <= slop).flatten()
792
+ cont_posn = term_posns[idx][cont_indices]
793
+ cont_posns.append(cont_posn)
794
+ phrase_freqs[mask] = bigram_freqs
795
+ prior_term = term
796
+ prior_posns = cont_posns
797
+ logging.info(f"Term Time: {perf_counter() - term_start:.4f}s")
798
+
799
+ logging.info(f"Phrase Search Time: {perf_counter() - start:.4f}s")
800
+
801
+ return phrase_freqs
802
+
803
+ def phrase_freq_shitty(self, tokens, slop=1):
745
804
  """Return number of occurences of a phrase."""
746
805
  from time import perf_counter
747
806
  # Start with docs with all terms
748
807
  start = perf_counter()
749
- pad = -1000
750
808
  mask = self.and_query(tokens)
751
809
  # For detailed documentation of this algorithm, see this ChatGPT4 discussion
752
810
  # https://chat.openai.com/share/31affaad-dc91-4757-b31c-e85bdb5a0eb6
@@ -754,18 +812,6 @@ class PostingsArray(ExtensionArray):
754
812
  if np.sum(mask) == 0:
755
813
  return mask
756
814
 
757
- def vstack_with_pad(arrays, width=10):
758
- vstacked = np.zeros((len(arrays), width), dtype=arrays[0].dtype) + pad
759
- for idx, array in enumerate(arrays):
760
- # Resize if needed, padding with pad
761
- if len(array) > width:
762
- logging.info(f"Resizing from {width} to {len(array)}")
763
- vstack_padded = np.pad(vstacked, ((0, 0), (0, len(array) - width)), constant_values=pad)
764
- width = len(array)
765
- vstacked = vstack_padded
766
- vstacked[idx, :len(array)] = array
767
- return vstacked
768
-
769
815
  # Pad for easy difference computation
770
816
  term_posns = []
771
817
  for term in tokens:
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: searcharray
3
- Version: 0.0.8
3
+ Version: 0.0.10
4
4
  Summary: Searchable pandas text extension arrays for prototyping search
5
5
  Home-page: https://github.com/softwaredoug/searcharray
6
6
  Author: Doug Turnbull
@@ -36,7 +36,7 @@ setup(
36
36
  # For a discussion on single-sourcing the version across setup.py and the
37
37
  # project code, see
38
38
  # https://packaging.python.org/guides/single-sourcing-package-version/
39
- version="0.0.8", # Required
39
+ version="0.0.10", # Required
40
40
  # This is a one-line description or tagline of what your project does. This
41
41
  # corresponds to the "Summary" metadata field:
42
42
  # https://packaging.python.org/specifications/core-metadata/#summary
@@ -38,6 +38,13 @@ perf_scenarios = {
38
38
  "phrase": ["foo", "bar"],
39
39
  "expected": [True, False, False, False, False, True] * 100000,
40
40
  },
41
+ "many_docs_and_positions": {
42
+ "docs": lambda: PostingsArray.index([" ".join(["foo bar bar baz foo foo bar foo"] * 100),
43
+ " ".join(["what is the foo bar doing in the bar foo?"] * 100)] * 100000),
44
+ "phrase": ["foo", "bar"],
45
+ "expected": [True, True] * 100000
46
+ }
47
+
41
48
  }
42
49
 
43
50
 
@@ -52,6 +59,11 @@ scenarios = {
52
59
  "phrase": ["foo", "bar"],
53
60
  "expected": [1, 0, 0, 0] * 25,
54
61
  },
62
+ "term_repeats": {
63
+ "docs": lambda: PostingsArray.index(["foo foo bar bar baz", "data2", "data3 bar", "bunny funny wunny"] * 25),
64
+ "phrase": ["foo", "bar"],
65
+ "expected": [1, 0, 0, 0] * 25,
66
+ },
55
67
  "multi_term_one_doc": {
56
68
  "docs": lambda: PostingsArray.index(["foo bar bar bar foo", "data2", "data3 bar", "bunny funny wunny"] * 25),
57
69
  "phrase": ["foo", "bar"],
@@ -150,6 +162,9 @@ def test_phrase(docs, phrase, expected):
150
162
  matches = docs.match(phrase)
151
163
  assert (term_freqs == expected).all()
152
164
  assert (matches == expected_matches).all()
165
+ if len(phrase) > 1:
166
+ phrase_matches = docs.phrase_freq(phrase)
167
+ assert (expected == phrase_matches).all()
153
168
  assert (docs == docs_before).all(), "The phrase_match method should not modify the original array"
154
169
  bm25 = docs.bm25(phrase)
155
170
  assert (np.argsort(bm25) == np.argsort(expected)).all()
@@ -161,7 +176,7 @@ def test_phrase_performance(docs, phrase, expected):
161
176
  start = perf_counter()
162
177
  docs = docs()
163
178
  matches = docs.match(phrase)
164
- print(f"phrase_match took {perf_counter() - start} seconds | {len(docs)} docs")
179
+ print(f"phrase_match 1 took {perf_counter() - start} seconds | {len(docs)} docs")
165
180
  assert (matches == expected).all()
166
181
 
167
182
 
File without changes
File without changes
File without changes