searcharray 0.0.46__tar.gz → 0.0.48__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. {searcharray-0.0.46 → searcharray-0.0.48}/PKG-INFO +1 -1
  2. {searcharray-0.0.46 → searcharray-0.0.48}/searcharray/indexing.py +1 -1
  3. {searcharray-0.0.46 → searcharray-0.0.48}/searcharray/phrase/bigram_freqs.py +2 -2
  4. {searcharray-0.0.46 → searcharray-0.0.48}/searcharray/phrase/middle_out.py +7 -1
  5. {searcharray-0.0.46 → searcharray-0.0.48}/searcharray/phrase/scan_merge.py +1 -1
  6. searcharray-0.0.48/searcharray/phrase/spans.py +132 -0
  7. {searcharray-0.0.46 → searcharray-0.0.48}/searcharray/postings.py +22 -34
  8. {searcharray-0.0.46 → searcharray-0.0.48}/searcharray/roaringish/__init__.py +1 -0
  9. {searcharray-0.0.46 → searcharray-0.0.48}/searcharray/roaringish/roaringish_ops.c +193 -163
  10. {searcharray-0.0.46 → searcharray-0.0.48}/searcharray/roaringish/snp_ops.c +192 -162
  11. {searcharray-0.0.46 → searcharray-0.0.48}/searcharray/roaringish/snp_ops.pxd +1 -0
  12. {searcharray-0.0.46 → searcharray-0.0.48}/searcharray/roaringish/snp_ops.pyx +2 -2
  13. searcharray-0.0.48/searcharray/roaringish/spans.c +30312 -0
  14. searcharray-0.0.48/searcharray/roaringish/spans.pyx +78 -0
  15. {searcharray-0.0.46 → searcharray-0.0.48}/searcharray/similarity.py +16 -8
  16. {searcharray-0.0.46 → searcharray-0.0.48}/searcharray/solr.py +19 -8
  17. {searcharray-0.0.46 → searcharray-0.0.48}/searcharray.egg-info/PKG-INFO +1 -1
  18. {searcharray-0.0.46 → searcharray-0.0.48}/searcharray.egg-info/SOURCES.txt +5 -0
  19. {searcharray-0.0.46 → searcharray-0.0.48}/setup.py +1 -1
  20. {searcharray-0.0.46 → searcharray-0.0.48}/test/test_search.py +17 -0
  21. searcharray-0.0.48/test/test_similarity.py +44 -0
  22. searcharray-0.0.48/test/test_slop_matches.py +52 -0
  23. {searcharray-0.0.46 → searcharray-0.0.48}/test/test_snp_ops.py +25 -7
  24. {searcharray-0.0.46 → searcharray-0.0.48}/test/test_solr.py +22 -2
  25. {searcharray-0.0.46 → searcharray-0.0.48}/README.md +0 -0
  26. {searcharray-0.0.46 → searcharray-0.0.48}/pyproject.toml +0 -0
  27. {searcharray-0.0.46 → searcharray-0.0.48}/searcharray/__init__.py +0 -0
  28. {searcharray-0.0.46 → searcharray-0.0.48}/searcharray/phrase/__init__.py +0 -0
  29. {searcharray-0.0.46 → searcharray-0.0.48}/searcharray/phrase/posn_diffs.py +0 -0
  30. {searcharray-0.0.46 → searcharray-0.0.48}/searcharray/roaringish/roaringish.py +0 -0
  31. {searcharray-0.0.46 → searcharray-0.0.48}/searcharray/roaringish/roaringish_ops.pyx +0 -0
  32. {searcharray-0.0.46 → searcharray-0.0.48}/searcharray/term_dict.py +0 -0
  33. {searcharray-0.0.46 → searcharray-0.0.48}/searcharray/utils/__init__.py +0 -0
  34. {searcharray-0.0.46 → searcharray-0.0.48}/searcharray/utils/bitcount.py +0 -0
  35. {searcharray-0.0.46 → searcharray-0.0.48}/searcharray/utils/mat_set.py +0 -0
  36. {searcharray-0.0.46 → searcharray-0.0.48}/searcharray/utils/row_viewable_matrix.py +0 -0
  37. {searcharray-0.0.46 → searcharray-0.0.48}/searcharray.egg-info/dependency_links.txt +0 -0
  38. {searcharray-0.0.46 → searcharray-0.0.48}/searcharray.egg-info/entry_points.txt +0 -0
  39. {searcharray-0.0.46 → searcharray-0.0.48}/searcharray.egg-info/requires.txt +0 -0
  40. {searcharray-0.0.46 → searcharray-0.0.48}/searcharray.egg-info/top_level.txt +0 -0
  41. {searcharray-0.0.46 → searcharray-0.0.48}/setup.cfg +0 -0
  42. {searcharray-0.0.46 → searcharray-0.0.48}/test/test_bitcount64.py +0 -0
  43. {searcharray-0.0.46 → searcharray-0.0.48}/test/test_extension_array.py +0 -0
  44. {searcharray-0.0.46 → searcharray-0.0.48}/test/test_minmax_posns.py +0 -0
  45. {searcharray-0.0.46 → searcharray-0.0.48}/test/test_msmarco.py +0 -0
  46. {searcharray-0.0.46 → searcharray-0.0.48}/test/test_phrase_matches.py +0 -0
  47. {searcharray-0.0.46 → searcharray-0.0.48}/test/test_tmdb.py +0 -0
  48. {searcharray-0.0.46 → searcharray-0.0.48}/test/test_utils.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: searcharray
3
- Version: 0.0.46
3
+ Version: 0.0.48
4
4
  Summary: Searchable pandas text extension arrays for prototyping search
5
5
  Home-page: https://github.com/softwaredoug/searcharray
6
6
  Author: Doug Turnbull
@@ -59,7 +59,7 @@ def _gather_tokens(array, tokenizer, term_dict, term_doc, start_doc_id=0, trunc_
59
59
  term_doc.append(np.unique(terms))
60
60
 
61
61
  if idx % 10000 == 0 and idx > 0:
62
- logger.info(f"Tokenized {doc_id} ({100.0 * (idx/len(array))}%)")
62
+ logger.info(f"Tokenized {doc_id} ({100.0 * (idx / len(array))}%)")
63
63
 
64
64
  # Flatten each
65
65
  all_terms = np.concatenate(all_terms)
@@ -58,7 +58,7 @@ def _adj_to_phrase_freq(overlap: np.ndarray, adjacents: np.ndarray) -> np.ndarra
58
58
  # [foo foo] [foo foo] mal [foo foo] -> 4 adjs, phrase freqs = 3
59
59
  consecutive_ones = encoder.payload_lsb(overlap & (overlap << _1))
60
60
  consecutive_ones = popcount64(consecutive_ones)
61
- adjacents -= -np.floor_divide(consecutive_ones, -2).astype(np.int64)
61
+ adjacents -= -np.floor_divide(consecutive_ones, -2, dtype=np.int64)
62
62
  return adjacents
63
63
 
64
64
 
@@ -80,7 +80,7 @@ def _inner_bigram_same_term(lhs_int: np.ndarray, rhs_int: np.ndarray,
80
80
  # Count these bits...
81
81
  overlap = lhs_int & rhs_shift
82
82
  adj_count = encoder.payload_lsb(overlap)
83
- adjacents = popcount64(adj_count).astype(np.int64)
83
+ adjacents = popcount64(adj_count).view(np.int64)
84
84
 
85
85
  phrase_freqs[lhs_doc_ids] += _adj_to_phrase_freq(overlap, adjacents)
86
86
  # Continue with ?? ?? foo foo
@@ -10,6 +10,7 @@ from copy import deepcopy
10
10
  from typing import List, Tuple, Dict, Union, cast, Optional
11
11
  from searcharray.roaringish import RoaringishEncoder, convert_keys, merge
12
12
  from searcharray.phrase.bigram_freqs import bigram_freqs, Continuation
13
+ from searcharray.phrase.spans import span_search
13
14
  import numbers
14
15
  import logging
15
16
  from collections import defaultdict
@@ -337,6 +338,7 @@ class PosnBitArray:
337
338
 
338
339
  def phrase_freqs(self, term_ids: List[int], phrase_freqs: np.ndarray,
339
340
  doc_ids: np.ndarray,
341
+ slop: int = 0,
340
342
  min_posn: Optional[int] = None,
341
343
  max_posn: Optional[int] = None) -> np.ndarray:
342
344
  if len(term_ids) < 2:
@@ -352,7 +354,11 @@ class PosnBitArray:
352
354
  keys=keys,
353
355
  min_payload=min_posn,
354
356
  max_payload=max_posn) for term_id in term_ids]
355
- return compute_phrase_freqs(enc_term_posns, phrase_freqs)
357
+
358
+ if slop == 0:
359
+ return compute_phrase_freqs(enc_term_posns, phrase_freqs)
360
+ else:
361
+ return span_search(enc_term_posns, phrase_freqs, slop)
356
362
 
357
363
  def positions(self, term_id: int, doc_ids) -> Union[List[np.ndarray], np.ndarray]:
358
364
  if isinstance(doc_ids, numbers.Number):
@@ -24,7 +24,7 @@ def _self_adjs(prior_posns, next_posns):
24
24
 
25
25
 
26
26
  def scan_merge_ins(term_posns: List[List[np.ndarray]],
27
- phrase_freqs: np.ndarray, slop=1) -> np.ndarray:
27
+ phrase_freqs: np.ndarray, slop=0) -> np.ndarray:
28
28
  """Merge bigram, by bigram, using np.searchsorted to find if insert posns match slop.
29
29
 
30
30
  Description:
@@ -0,0 +1,132 @@
1
+ """Find span matches up to PAYLOAD_LSB bits span distance."""
2
+ # Colab notebooks
3
+ # https://colab.research.google.com/drive/1H4g7eeaYaHT1dJ5dmcmZUcwKNT-14tYs?authuser=1
4
+ # Chat GPT convo
5
+ # https://chat.openai.com/share/b97c61e6-95d0-40e7-a8cc-447dc1495314
6
+ import numpy as np
7
+ from typing import List
8
+ from searcharray.roaringish import intersect, adjacent, merge, span_search as r_span_search
9
+ # from searcharary.roaringish import intersect_all
10
+ from searcharray.roaringish.roaringish import RoaringishEncoder
11
+
12
+ encoder = RoaringishEncoder()
13
+
14
+ # To not constantly type coerce
15
+ _64 = np.uint64(64)
16
+ _2 = np.uint64(2)
17
+ _1 = np.uint64(1)
18
+ _0 = np.uint64(0)
19
+ _neg1 = np.int64(-1)
20
+ _upper_bit = _1 << (encoder.payload_lsb_bits - _1)
21
+
22
+
23
+ # ---
24
+ # How to find max spans in roaringish
25
+ # MSB
26
+ # term A |10| | 12| 13| | 18 * Intersect A+B -> 10,12
27
+ # term B |10| | 12| | 14 15 17 | * RHS intersect -> 13 -> 14
28
+ # term C | 11 14
29
+ # term D 10 | 15
30
+ # term E | 11 12
31
+ # ------
32
+ #
33
+ #
34
+ # term A |10| 11 | 12| 13| * Intersect A+B -> 10,12
35
+ # term B |10| 11 | 12| 13| * Intersect A+B -> 10,12
36
+ # term C 9 | | 11 |
37
+ #
38
+ # ----
39
+ #
40
+ # Finally: X X X X <- 4 words selected from each to find max span
41
+ #
42
+ #
43
+ def _intersect_all(posns_encoded: List[np.ndarray]):
44
+ """Intersect all encoded positions at roaringish MSBs."""
45
+ if len(posns_encoded) < 2:
46
+ raise ValueError("Need at least two positions to intersect")
47
+
48
+ lhs_int = posns_encoded[0]
49
+ lhs_to_left = posns_encoded[0]
50
+ lhs_to_right = posns_encoded[0]
51
+
52
+ for term_idx, posns_next in enumerate(posns_encoded[1:]):
53
+ _, rhs_int = intersect(lhs_int, posns_next, mask=encoder.header_mask)
54
+ _, rhs_ls = adjacent(lhs_to_left, posns_next, mask=encoder.header_mask)
55
+ rhs_rs, _ = adjacent(lhs_to_right, posns_next, mask=encoder.header_mask)
56
+
57
+ # Update LHS to rhs_int + rhs_ls + rhs_rs indices
58
+ lhs_int = posns_next[rhs_int]
59
+ lhs_to_left = posns_next[rhs_ls]
60
+ lhs_to_right = posns_next[rhs_rs]
61
+
62
+ # Get active MSBs now
63
+ # Merge all the rest to grab them
64
+ int_header = encoder.header(lhs_int)
65
+ to_left_header = encoder.header(lhs_to_left)
66
+ to_right_header = encoder.header(lhs_to_right)
67
+ merged = merge(int_header, to_left_header, drop_duplicates=True)
68
+ merged = merge(merged, to_right_header, drop_duplicates=True)
69
+
70
+ # Slice only matches at MSBs
71
+ for i in range(len(posns_encoded)):
72
+ posns_encoded[i] = encoder.slice(posns_encoded[i], merged)
73
+
74
+
75
+ # Picking up from intersections:
76
+ #
77
+ # Finally: X X X X X X <- 6 words selected from each to find max span
78
+ #
79
+ # Each X may be adjacent or not
80
+ #
81
+ # Looking at LSBs:
82
+ # MSB: 9 10 13 14 15 20
83
+ # Term A 01001 00000 00000 01000 00001 00100
84
+ # Term B 00010 00010 00010 00000 01000 00010
85
+ # Term C 10000 00001 00000 00001 00000 00001
86
+ #
87
+ # Limit to two adj words at a time (slop must be < LSB bits)
88
+ #
89
+ # MSB: 9 10 13 14 15 20
90
+ # Term A 01001 00000 00000 01000 00001 00100
91
+ # Term B 00010 00010 00010 00000 01000 00010
92
+ # Term C 10000 00001 00000 00001 00000 00001
93
+ # -----------
94
+ # (has span?)
95
+ # -----------
96
+ # (has span?)
97
+ # -----------
98
+ # (has span?)
99
+ # -----
100
+ # (has span?)
101
+ #
102
+ # Has span algo, really finding minimum sized spans
103
+ #
104
+ # Concat to single 64 bit word
105
+ # 0100100000
106
+ # 0001000010
107
+ # 1000000001
108
+ #
109
+ # Search w/ mask
110
+ # ffff000000
111
+ # And'd, now check if > 0:
112
+ # 0100000000 > 0 -> 1 0 0
113
+ # 0001000000 > 0 -> 1 1 0
114
+ # 1000000000 > 0 -> 1 1 0
115
+ #
116
+ # has_span (ANDED): 1
117
+ #
118
+ # Can we shrink the mask?
119
+ # If any hase leading or trailing 0s in mask, then shrink
120
+ def span_search(posns_encoded: List[np.ndarray],
121
+ phrase_freqs: np.ndarray,
122
+ slop: int) -> np.ndarray:
123
+ """Find span matches up to PAYLOAD_LSB bits span distance."""
124
+ # Find inner span candidates
125
+ _intersect_all(posns_encoded)
126
+
127
+ # Populate phrase freqs with matches of slop
128
+ r_span_search(posns_encoded, phrase_freqs, slop,
129
+ encoder.key_mask,
130
+ encoder.header_mask,
131
+ encoder.payload_lsb_bits)
132
+ return phrase_freqs
@@ -15,7 +15,7 @@ import numpy as np
15
15
  from searcharray.phrase.scan_merge import scan_merge_ins
16
16
  from searcharray.phrase.posn_diffs import compute_phrase_freqs
17
17
  from searcharray.phrase.middle_out import PosnBitArray
18
- from searcharray.similarity import Similarity, default_bm25, ScoringContext
18
+ from searcharray.similarity import Similarity, default_bm25
19
19
  from searcharray.indexing import build_index_from_tokenizer, build_index_from_terms_list
20
20
  from searcharray.term_dict import TermMissingError
21
21
  from searcharray.roaringish.roaringish_ops import as_dense
@@ -217,7 +217,6 @@ class SearchArray(ExtensionArray):
217
217
  if not is_list_like(postings):
218
218
  raise TypeError("Expected list-like object, got {}".format(type(postings)))
219
219
 
220
- self.scoring_context = None
221
220
  self.avoid_copies = avoid_copies
222
221
  self.tokenizer = tokenizer
223
222
  self.term_mat, self.posns, \
@@ -495,11 +494,9 @@ class SearchArray(ExtensionArray):
495
494
  postings_arr.posns = self.posns
496
495
  postings_arr.term_dict = self.term_dict
497
496
  postings_arr.avg_doc_length = self.avg_doc_length
498
- postings_arr.scoring_context = self.scoring_context
499
497
  postings_arr.corpus_size = self.corpus_size
500
498
 
501
499
  if not self.avoid_copies:
502
- postings_arr.scoring_context = None
503
500
  postings_arr.posns = self.posns.copy()
504
501
  postings_arr.term_dict = self.term_dict.copy()
505
502
  return postings_arr
@@ -532,11 +529,12 @@ class SearchArray(ExtensionArray):
532
529
  # Search functionality
533
530
  # ***********************************************************
534
531
  def termfreqs(self, token: Union[List[str], str],
532
+ slop: int = 0,
535
533
  min_posn: Optional[int] = None,
536
534
  max_posn: Optional[int] = None) -> np.ndarray:
537
535
  token = self._check_token_arg(token)
538
536
  if isinstance(token, list):
539
- return self.phrase_freq(token, min_posn=min_posn, max_posn=max_posn)
537
+ return self.phrase_freq(token, slop=slop, min_posn=min_posn, max_posn=max_posn)
540
538
 
541
539
  try:
542
540
  term_id = self.term_dict.get_term_id(token)
@@ -552,7 +550,6 @@ class SearchArray(ExtensionArray):
552
550
  else:
553
551
  doc_ids, termfreqs = self.posns.termfreqs(term_id,
554
552
  doc_ids=slice_of_rows, min_posn=min_posn, max_posn=max_posn)
555
- # import pdb; pdb.set_trace()
556
553
  # matches[doc_ids] = termfreqs
557
554
  return as_dense(doc_ids, termfreqs, len(self))
558
555
  except TermMissingError:
@@ -570,11 +567,11 @@ class SearchArray(ExtensionArray):
570
567
  def doclengths(self) -> np.ndarray:
571
568
  return self.doc_lens
572
569
 
573
- def match(self, token: Union[List[str], str], slop: int = 1) -> np.ndarray:
570
+ def match(self, token: Union[List[str], str], slop: int = 0) -> np.ndarray:
574
571
  """Return a boolean numpy array indicating which elements contain the given term."""
575
572
  token = self._check_token_arg(token)
576
573
  if isinstance(token, list):
577
- term_freq = self.phrase_freq(token)
574
+ term_freq = self.phrase_freq(token, slop=slop)
578
575
  else:
579
576
  term_freq = self.termfreqs(token)
580
577
  return term_freq > 0
@@ -601,14 +598,7 @@ class SearchArray(ExtensionArray):
601
598
  token = self._check_token_arg(token)
602
599
  doc_lens = self.doclengths()
603
600
 
604
- context = ScoringContext(doc_lens=doc_lens,
605
- avg_doc_lens=self.avg_doc_length,
606
- num_docs=self.corpus_size)
607
-
608
- if not context.same_as(self.scoring_context):
609
- self.scoring_context = context
610
-
611
- scores = similarity(tfs, all_dfs, self.scoring_context)
601
+ scores = similarity(tfs, all_dfs, doc_lens, self.avg_doc_length, self.corpus_size)
612
602
  return scores
613
603
 
614
604
  def positions(self, token: str, key=None) -> List[np.ndarray]:
@@ -633,24 +623,22 @@ class SearchArray(ExtensionArray):
633
623
  return mask
634
624
 
635
625
  def phrase_freq(self, tokens: List[str],
636
- slop=1,
626
+ slop=0,
637
627
  min_posn: Optional[int] = None,
638
628
  max_posn: Optional[int] = None) -> np.ndarray:
639
- if slop == 1:
640
- phrase_freqs = np.zeros(len(self))
641
- try:
642
- doc_ids = self.term_mat.rows
643
- term_ids = [self.term_dict.get_term_id(token) for token in tokens]
644
- return self.posns.phrase_freqs(term_ids, doc_ids=doc_ids,
645
- phrase_freqs=phrase_freqs,
646
- min_posn=min_posn,
647
- max_posn=max_posn)
648
- except TermMissingError:
649
- return phrase_freqs
650
- else:
651
- return self.phrase_freq_every_diff(tokens, slop=slop)
629
+ phrase_freqs = np.zeros(len(self))
630
+ try:
631
+ doc_ids = self.term_mat.rows
632
+ term_ids = [self.term_dict.get_term_id(token) for token in tokens]
633
+ return self.posns.phrase_freqs(term_ids, doc_ids=doc_ids,
634
+ phrase_freqs=phrase_freqs,
635
+ slop=slop,
636
+ min_posn=min_posn,
637
+ max_posn=max_posn)
638
+ except TermMissingError:
639
+ return phrase_freqs
652
640
 
653
- def phrase_freq_scan(self, tokens: List[str], mask=None, slop=1) -> np.ndarray:
641
+ def phrase_freq_scan(self, tokens: List[str], mask=None, slop=0) -> np.ndarray:
654
642
  if mask is None:
655
643
  mask = self.and_query(tokens)
656
644
 
@@ -661,10 +649,10 @@ class SearchArray(ExtensionArray):
661
649
  posns = [self.positions(token, mask) for token in tokens]
662
650
  phrase_freqs = np.zeros(len(self))
663
651
 
664
- phrase_freqs[mask] = scan_merge_ins(posns, phrase_freqs[mask], slop=slop)
652
+ phrase_freqs[mask] = scan_merge_ins(posns, phrase_freqs[mask], slop=slop + 1)
665
653
  return phrase_freqs
666
654
 
667
- def phrase_freq_every_diff(self, tokens: List[str], slop=1) -> np.ndarray:
655
+ def phrase_freq_every_diff(self, tokens: List[str], slop=0) -> np.ndarray:
668
656
  phrase_freqs = -np.ones(len(self))
669
657
 
670
658
  mask = self.and_query(tokens)
@@ -676,7 +664,7 @@ class SearchArray(ExtensionArray):
676
664
  for width in [10, 20, 30, 40]:
677
665
  phrase_freqs[mask] = compute_phrase_freqs(term_posns,
678
666
  phrase_freqs[mask],
679
- slop=slop,
667
+ slop=slop + 1,
680
668
  width=width)
681
669
 
682
670
  remaining_mask = phrase_freqs == -1
@@ -1,3 +1,4 @@
1
1
  from searcharray.roaringish.roaringish import RoaringishEncoder, convert_keys # noqa
2
2
  from searcharray.roaringish.snp_ops import popcount64 # noqa
3
3
  from searcharray.roaringish.snp_ops import merge, intersect, unique, adjacent # noqa
4
+ from searcharray.roaringish.spans import span_search # noqa