searcharray 0.0.45__tar.gz → 0.0.47__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. {searcharray-0.0.45 → searcharray-0.0.47}/PKG-INFO +1 -1
  2. {searcharray-0.0.45 → searcharray-0.0.47}/searcharray/indexing.py +1 -1
  3. {searcharray-0.0.45 → searcharray-0.0.47}/searcharray/phrase/bigram_freqs.py +2 -2
  4. {searcharray-0.0.45 → searcharray-0.0.47}/searcharray/phrase/middle_out.py +7 -1
  5. {searcharray-0.0.45 → searcharray-0.0.47}/searcharray/phrase/scan_merge.py +1 -1
  6. searcharray-0.0.47/searcharray/phrase/spans.py +132 -0
  7. {searcharray-0.0.45 → searcharray-0.0.47}/searcharray/postings.py +35 -29
  8. {searcharray-0.0.45 → searcharray-0.0.47}/searcharray/roaringish/__init__.py +1 -0
  9. {searcharray-0.0.45 → searcharray-0.0.47}/searcharray/roaringish/roaringish_ops.c +895 -281
  10. {searcharray-0.0.45 → searcharray-0.0.47}/searcharray/roaringish/roaringish_ops.pyx +26 -2
  11. {searcharray-0.0.45 → searcharray-0.0.47}/searcharray/roaringish/snp_ops.c +192 -162
  12. {searcharray-0.0.45 → searcharray-0.0.47}/searcharray/roaringish/snp_ops.pxd +1 -0
  13. {searcharray-0.0.45 → searcharray-0.0.47}/searcharray/roaringish/snp_ops.pyx +2 -2
  14. searcharray-0.0.47/searcharray/roaringish/spans.c +30312 -0
  15. searcharray-0.0.47/searcharray/roaringish/spans.pyx +78 -0
  16. searcharray-0.0.47/searcharray/similarity.py +110 -0
  17. {searcharray-0.0.45 → searcharray-0.0.47}/searcharray.egg-info/PKG-INFO +1 -1
  18. {searcharray-0.0.45 → searcharray-0.0.47}/searcharray.egg-info/SOURCES.txt +5 -0
  19. {searcharray-0.0.45 → searcharray-0.0.47}/setup.py +1 -1
  20. {searcharray-0.0.45 → searcharray-0.0.47}/test/test_msmarco.py +7 -4
  21. {searcharray-0.0.45 → searcharray-0.0.47}/test/test_search.py +17 -0
  22. searcharray-0.0.47/test/test_similarity.py +44 -0
  23. searcharray-0.0.47/test/test_slop_matches.py +52 -0
  24. {searcharray-0.0.45 → searcharray-0.0.47}/test/test_snp_ops.py +25 -7
  25. {searcharray-0.0.45 → searcharray-0.0.47}/test/test_solr.py +3 -2
  26. {searcharray-0.0.45 → searcharray-0.0.47}/test/test_tmdb.py +4 -1
  27. searcharray-0.0.45/searcharray/similarity.py +0 -82
  28. {searcharray-0.0.45 → searcharray-0.0.47}/README.md +0 -0
  29. {searcharray-0.0.45 → searcharray-0.0.47}/pyproject.toml +0 -0
  30. {searcharray-0.0.45 → searcharray-0.0.47}/searcharray/__init__.py +0 -0
  31. {searcharray-0.0.45 → searcharray-0.0.47}/searcharray/phrase/__init__.py +0 -0
  32. {searcharray-0.0.45 → searcharray-0.0.47}/searcharray/phrase/posn_diffs.py +0 -0
  33. {searcharray-0.0.45 → searcharray-0.0.47}/searcharray/roaringish/roaringish.py +0 -0
  34. {searcharray-0.0.45 → searcharray-0.0.47}/searcharray/solr.py +0 -0
  35. {searcharray-0.0.45 → searcharray-0.0.47}/searcharray/term_dict.py +0 -0
  36. {searcharray-0.0.45 → searcharray-0.0.47}/searcharray/utils/__init__.py +0 -0
  37. {searcharray-0.0.45 → searcharray-0.0.47}/searcharray/utils/bitcount.py +0 -0
  38. {searcharray-0.0.45 → searcharray-0.0.47}/searcharray/utils/mat_set.py +0 -0
  39. {searcharray-0.0.45 → searcharray-0.0.47}/searcharray/utils/row_viewable_matrix.py +0 -0
  40. {searcharray-0.0.45 → searcharray-0.0.47}/searcharray.egg-info/dependency_links.txt +0 -0
  41. {searcharray-0.0.45 → searcharray-0.0.47}/searcharray.egg-info/entry_points.txt +0 -0
  42. {searcharray-0.0.45 → searcharray-0.0.47}/searcharray.egg-info/requires.txt +0 -0
  43. {searcharray-0.0.45 → searcharray-0.0.47}/searcharray.egg-info/top_level.txt +0 -0
  44. {searcharray-0.0.45 → searcharray-0.0.47}/setup.cfg +0 -0
  45. {searcharray-0.0.45 → searcharray-0.0.47}/test/test_bitcount64.py +0 -0
  46. {searcharray-0.0.45 → searcharray-0.0.47}/test/test_extension_array.py +0 -0
  47. {searcharray-0.0.45 → searcharray-0.0.47}/test/test_minmax_posns.py +0 -0
  48. {searcharray-0.0.45 → searcharray-0.0.47}/test/test_phrase_matches.py +0 -0
  49. {searcharray-0.0.45 → searcharray-0.0.47}/test/test_utils.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: searcharray
3
- Version: 0.0.45
3
+ Version: 0.0.47
4
4
  Summary: Searchable pandas text extension arrays for prototyping search
5
5
  Home-page: https://github.com/softwaredoug/searcharray
6
6
  Author: Doug Turnbull
@@ -59,7 +59,7 @@ def _gather_tokens(array, tokenizer, term_dict, term_doc, start_doc_id=0, trunc_
59
59
  term_doc.append(np.unique(terms))
60
60
 
61
61
  if idx % 10000 == 0 and idx > 0:
62
- logger.info(f"Tokenized {doc_id} ({100.0 * (idx/len(array))}%)")
62
+ logger.info(f"Tokenized {doc_id} ({100.0 * (idx / len(array))}%)")
63
63
 
64
64
  # Flatten each
65
65
  all_terms = np.concatenate(all_terms)
@@ -58,7 +58,7 @@ def _adj_to_phrase_freq(overlap: np.ndarray, adjacents: np.ndarray) -> np.ndarra
58
58
  # [foo foo] [foo foo] mal [foo foo] -> 4 adjs, phrase freqs = 3
59
59
  consecutive_ones = encoder.payload_lsb(overlap & (overlap << _1))
60
60
  consecutive_ones = popcount64(consecutive_ones)
61
- adjacents -= -np.floor_divide(consecutive_ones, -2).astype(np.int64)
61
+ adjacents -= -np.floor_divide(consecutive_ones, -2, dtype=np.int64)
62
62
  return adjacents
63
63
 
64
64
 
@@ -80,7 +80,7 @@ def _inner_bigram_same_term(lhs_int: np.ndarray, rhs_int: np.ndarray,
80
80
  # Count these bits...
81
81
  overlap = lhs_int & rhs_shift
82
82
  adj_count = encoder.payload_lsb(overlap)
83
- adjacents = popcount64(adj_count).astype(np.int64)
83
+ adjacents = popcount64(adj_count).view(np.int64)
84
84
 
85
85
  phrase_freqs[lhs_doc_ids] += _adj_to_phrase_freq(overlap, adjacents)
86
86
  # Continue with ?? ?? foo foo
@@ -10,6 +10,7 @@ from copy import deepcopy
10
10
  from typing import List, Tuple, Dict, Union, cast, Optional
11
11
  from searcharray.roaringish import RoaringishEncoder, convert_keys, merge
12
12
  from searcharray.phrase.bigram_freqs import bigram_freqs, Continuation
13
+ from searcharray.phrase.spans import span_search
13
14
  import numbers
14
15
  import logging
15
16
  from collections import defaultdict
@@ -337,6 +338,7 @@ class PosnBitArray:
337
338
 
338
339
  def phrase_freqs(self, term_ids: List[int], phrase_freqs: np.ndarray,
339
340
  doc_ids: np.ndarray,
341
+ slop: int = 0,
340
342
  min_posn: Optional[int] = None,
341
343
  max_posn: Optional[int] = None) -> np.ndarray:
342
344
  if len(term_ids) < 2:
@@ -352,7 +354,11 @@ class PosnBitArray:
352
354
  keys=keys,
353
355
  min_payload=min_posn,
354
356
  max_payload=max_posn) for term_id in term_ids]
355
- return compute_phrase_freqs(enc_term_posns, phrase_freqs)
357
+
358
+ if slop == 0:
359
+ return compute_phrase_freqs(enc_term_posns, phrase_freqs)
360
+ else:
361
+ return span_search(enc_term_posns, phrase_freqs, slop)
356
362
 
357
363
  def positions(self, term_id: int, doc_ids) -> Union[List[np.ndarray], np.ndarray]:
358
364
  if isinstance(doc_ids, numbers.Number):
@@ -24,7 +24,7 @@ def _self_adjs(prior_posns, next_posns):
24
24
 
25
25
 
26
26
  def scan_merge_ins(term_posns: List[List[np.ndarray]],
27
- phrase_freqs: np.ndarray, slop=1) -> np.ndarray:
27
+ phrase_freqs: np.ndarray, slop=0) -> np.ndarray:
28
28
  """Merge bigram, by bigram, using np.searchsorted to find if insert posns match slop.
29
29
 
30
30
  Description:
@@ -0,0 +1,132 @@
1
+ """Find span matches up to PAYLOAD_LSB bits span distance."""
2
+ # Colab notebooks
3
+ # https://colab.research.google.com/drive/1H4g7eeaYaHT1dJ5dmcmZUcwKNT-14tYs?authuser=1
4
+ # Chat GPT convo
5
+ # https://chat.openai.com/share/b97c61e6-95d0-40e7-a8cc-447dc1495314
6
+ import numpy as np
7
+ from typing import List
8
+ from searcharray.roaringish import intersect, adjacent, merge, span_search as r_span_search
9
+ # from searcharary.roaringish import intersect_all
10
+ from searcharray.roaringish.roaringish import RoaringishEncoder
11
+
12
+ encoder = RoaringishEncoder()
13
+
14
+ # To not constantly type coerce
15
+ _64 = np.uint64(64)
16
+ _2 = np.uint64(2)
17
+ _1 = np.uint64(1)
18
+ _0 = np.uint64(0)
19
+ _neg1 = np.int64(-1)
20
+ _upper_bit = _1 << (encoder.payload_lsb_bits - _1)
21
+
22
+
23
+ # ---
24
+ # How to find max spans in roaringish
25
+ # MSB
26
+ # term A |10| | 12| 13| | 18 * Intersect A+B -> 10,12
27
+ # term B |10| | 12| | 14 15 17 | * RHS intersect -> 13 -> 14
28
+ # term C | 11 14
29
+ # term D 10 | 15
30
+ # term E | 11 12
31
+ # ------
32
+ #
33
+ #
34
+ # term A |10| 11 | 12| 13| * Intersect A+B -> 10,12
35
+ # term B |10| 11 | 12| 13| * Intersect A+B -> 10,12
36
+ # term C 9 | | 11 |
37
+ #
38
+ # ----
39
+ #
40
+ # Finally: X X X X <- 4 words selected from each to find max span
41
+ #
42
+ #
43
+ def _intersect_all(posns_encoded: List[np.ndarray]):
44
+ """Intersect all encoded positions at roaringish MSBs."""
45
+ if len(posns_encoded) < 2:
46
+ raise ValueError("Need at least two positions to intersect")
47
+
48
+ lhs_int = posns_encoded[0]
49
+ lhs_to_left = posns_encoded[0]
50
+ lhs_to_right = posns_encoded[0]
51
+
52
+ for term_idx, posns_next in enumerate(posns_encoded[1:]):
53
+ _, rhs_int = intersect(lhs_int, posns_next, mask=encoder.header_mask)
54
+ _, rhs_ls = adjacent(lhs_to_left, posns_next, mask=encoder.header_mask)
55
+ rhs_rs, _ = adjacent(lhs_to_right, posns_next, mask=encoder.header_mask)
56
+
57
+ # Update LHS to rhs_int + rhs_ls + rhs_rs indices
58
+ lhs_int = posns_next[rhs_int]
59
+ lhs_to_left = posns_next[rhs_ls]
60
+ lhs_to_right = posns_next[rhs_rs]
61
+
62
+ # Get active MSBs now
63
+ # Merge all the rest to grab them
64
+ int_header = encoder.header(lhs_int)
65
+ to_left_header = encoder.header(lhs_to_left)
66
+ to_right_header = encoder.header(lhs_to_right)
67
+ merged = merge(int_header, to_left_header, drop_duplicates=True)
68
+ merged = merge(merged, to_right_header, drop_duplicates=True)
69
+
70
+ # Slice only matches at MSBs
71
+ for i in range(len(posns_encoded)):
72
+ posns_encoded[i] = encoder.slice(posns_encoded[i], merged)
73
+
74
+
75
+ # Picking up from intersections:
76
+ #
77
+ # Finally: X X X X X X <- 6 words selected from each to find max span
78
+ #
79
+ # Each X may be adjacent or not
80
+ #
81
+ # Looking at LSBs:
82
+ # MSB: 9 10 13 14 15 20
83
+ # Term A 01001 00000 00000 01000 00001 00100
84
+ # Term B 00010 00010 00010 00000 01000 00010
85
+ # Term C 10000 00001 00000 00001 00000 00001
86
+ #
87
+ # Limit to two adj words at a time (slop must be < LSB bits)
88
+ #
89
+ # MSB: 9 10 13 14 15 20
90
+ # Term A 01001 00000 00000 01000 00001 00100
91
+ # Term B 00010 00010 00010 00000 01000 00010
92
+ # Term C 10000 00001 00000 00001 00000 00001
93
+ # -----------
94
+ # (has span?)
95
+ # -----------
96
+ # (has span?)
97
+ # -----------
98
+ # (has span?)
99
+ # -----
100
+ # (has span?)
101
+ #
102
+ # Has span algo, really finding minimum sized spans
103
+ #
104
+ # Concat to single 64 bit word
105
+ # 0100100000
106
+ # 0001000010
107
+ # 1000000001
108
+ #
109
+ # Search w/ mask
110
+ # ffff000000
111
+ # And'd, now check if > 0:
112
+ # 0100000000 > 0 -> 1 0 0
113
+ # 0001000000 > 0 -> 1 1 0
114
+ # 1000000000 > 0 -> 1 1 0
115
+ #
116
+ # has_span (ANDED): 1
117
+ #
118
+ # Can we shrink the mask?
119
+ # If any hase leading or trailing 0s in mask, then shrink
120
+ def span_search(posns_encoded: List[np.ndarray],
121
+ phrase_freqs: np.ndarray,
122
+ slop: int) -> np.ndarray:
123
+ """Find span matches up to PAYLOAD_LSB bits span distance."""
124
+ # Find inner span candidates
125
+ _intersect_all(posns_encoded)
126
+
127
+ # Populate phrase freqs with matches of slop
128
+ r_span_search(posns_encoded, phrase_freqs, slop,
129
+ encoder.key_mask,
130
+ encoder.header_mask,
131
+ encoder.payload_lsb_bits)
132
+ return phrase_freqs
@@ -18,6 +18,7 @@ from searcharray.phrase.middle_out import PosnBitArray
18
18
  from searcharray.similarity import Similarity, default_bm25
19
19
  from searcharray.indexing import build_index_from_tokenizer, build_index_from_terms_list
20
20
  from searcharray.term_dict import TermMissingError
21
+ from searcharray.roaringish.roaringish_ops import as_dense
21
22
 
22
23
  logger = logging.getLogger(__name__)
23
24
 
@@ -221,6 +222,7 @@ class SearchArray(ExtensionArray):
221
222
  self.term_mat, self.posns, \
222
223
  self.term_dict, self.avg_doc_length, \
223
224
  self.doc_lens = build_index_from_terms_list(postings, Terms)
225
+ self.corpus_size = len(self.doc_lens)
224
226
 
225
227
  @classmethod
226
228
  def index(cls, array: Iterable,
@@ -266,6 +268,7 @@ class SearchArray(ExtensionArray):
266
268
  postings.term_dict = term_dict
267
269
  postings.avg_doc_length = avg_doc_length
268
270
  postings.doc_lens = doc_lens
271
+ postings.corpus_size = len(doc_lens)
269
272
  return postings
270
273
 
271
274
  def warm(self):
@@ -312,13 +315,18 @@ class SearchArray(ExtensionArray):
312
315
  else:
313
316
  # Construct a sliced view of this array
314
317
  sliced_tfs = self.term_mat.slice(key)
315
- sliced_posns = self.posns.slice(sliced_tfs.rows) if not self.avoid_copies else self.posns
318
+ if not self.avoid_copies:
319
+ sliced_posns = self.posns.slice(sliced_tfs.rows) if not self.avoid_copies else self.posns
320
+ else:
321
+ sliced_posns = self.posns
322
+
316
323
  arr = SearchArray([], tokenizer=self.tokenizer)
317
324
  arr.term_mat = sliced_tfs
318
325
  arr.doc_lens = self.doc_lens[key]
319
326
  arr.posns = sliced_posns
320
327
  arr.term_dict = self.term_dict
321
328
  arr.avg_doc_length = self.avg_doc_length
329
+ arr.corpus_size = self.corpus_size
322
330
  return arr
323
331
 
324
332
  def __setitem__(self, key, value):
@@ -486,6 +494,7 @@ class SearchArray(ExtensionArray):
486
494
  postings_arr.posns = self.posns
487
495
  postings_arr.term_dict = self.term_dict
488
496
  postings_arr.avg_doc_length = self.avg_doc_length
497
+ postings_arr.corpus_size = self.corpus_size
489
498
 
490
499
  if not self.avoid_copies:
491
500
  postings_arr.posns = self.posns.copy()
@@ -520,17 +529,18 @@ class SearchArray(ExtensionArray):
520
529
  # Search functionality
521
530
  # ***********************************************************
522
531
  def termfreqs(self, token: Union[List[str], str],
532
+ slop: int = 0,
523
533
  min_posn: Optional[int] = None,
524
534
  max_posn: Optional[int] = None) -> np.ndarray:
525
535
  token = self._check_token_arg(token)
526
536
  if isinstance(token, list):
527
- return self.phrase_freq(token, min_posn=min_posn, max_posn=max_posn)
537
+ return self.phrase_freq(token, slop=slop, min_posn=min_posn, max_posn=max_posn)
528
538
 
529
539
  try:
530
540
  term_id = self.term_dict.get_term_id(token)
531
- matches = np.zeros(len(self), dtype=np.float32)
532
541
  slice_of_rows = None
533
542
  if self.term_mat.subset:
543
+ matches = np.zeros(len(self), dtype=np.float32)
534
544
  slice_of_rows = self.term_mat.rows
535
545
  doc_ids, termfreqs = self.posns.termfreqs(term_id,
536
546
  doc_ids=slice_of_rows, min_posn=min_posn, max_posn=max_posn)
@@ -540,9 +550,8 @@ class SearchArray(ExtensionArray):
540
550
  else:
541
551
  doc_ids, termfreqs = self.posns.termfreqs(term_id,
542
552
  doc_ids=slice_of_rows, min_posn=min_posn, max_posn=max_posn)
543
-
544
- matches[doc_ids] = termfreqs
545
- return matches
553
+ # matches[doc_ids] = termfreqs
554
+ return as_dense(doc_ids, termfreqs, len(self))
546
555
  except TermMissingError:
547
556
  return np.zeros(len(self), dtype=np.float32)
548
557
 
@@ -558,11 +567,11 @@ class SearchArray(ExtensionArray):
558
567
  def doclengths(self) -> np.ndarray:
559
568
  return self.doc_lens
560
569
 
561
- def match(self, token: Union[List[str], str], slop: int = 1) -> np.ndarray:
570
+ def match(self, token: Union[List[str], str], slop: int = 0) -> np.ndarray:
562
571
  """Return a boolean numpy array indicating which elements contain the given term."""
563
572
  token = self._check_token_arg(token)
564
573
  if isinstance(token, list):
565
- term_freq = self.phrase_freq(token)
574
+ term_freq = self.phrase_freq(token, slop=slop)
566
575
  else:
567
576
  term_freq = self.termfreqs(token)
568
577
  return term_freq > 0
@@ -588,9 +597,8 @@ class SearchArray(ExtensionArray):
588
597
  tfs = self.termfreqs(token, min_posn=min_posn, max_posn=max_posn)
589
598
  token = self._check_token_arg(token)
590
599
  doc_lens = self.doclengths()
591
- scores = similarity(term_freqs=tfs, doc_freqs=all_dfs,
592
- doc_lens=doc_lens, avg_doc_lens=self.avg_doc_length,
593
- num_docs=len(self))
600
+
601
+ scores = similarity(tfs, all_dfs, doc_lens, self.avg_doc_length, self.corpus_size)
594
602
  return scores
595
603
 
596
604
  def positions(self, token: str, key=None) -> List[np.ndarray]:
@@ -615,24 +623,22 @@ class SearchArray(ExtensionArray):
615
623
  return mask
616
624
 
617
625
  def phrase_freq(self, tokens: List[str],
618
- slop=1,
626
+ slop=0,
619
627
  min_posn: Optional[int] = None,
620
628
  max_posn: Optional[int] = None) -> np.ndarray:
621
- if slop == 1:
622
- phrase_freqs = np.zeros(len(self))
623
- try:
624
- doc_ids = self.term_mat.rows
625
- term_ids = [self.term_dict.get_term_id(token) for token in tokens]
626
- return self.posns.phrase_freqs(term_ids, doc_ids=doc_ids,
627
- phrase_freqs=phrase_freqs,
628
- min_posn=min_posn,
629
- max_posn=max_posn)
630
- except TermMissingError:
631
- return phrase_freqs
632
- else:
633
- return self.phrase_freq_every_diff(tokens, slop=slop)
629
+ phrase_freqs = np.zeros(len(self))
630
+ try:
631
+ doc_ids = self.term_mat.rows
632
+ term_ids = [self.term_dict.get_term_id(token) for token in tokens]
633
+ return self.posns.phrase_freqs(term_ids, doc_ids=doc_ids,
634
+ phrase_freqs=phrase_freqs,
635
+ slop=slop,
636
+ min_posn=min_posn,
637
+ max_posn=max_posn)
638
+ except TermMissingError:
639
+ return phrase_freqs
634
640
 
635
- def phrase_freq_scan(self, tokens: List[str], mask=None, slop=1) -> np.ndarray:
641
+ def phrase_freq_scan(self, tokens: List[str], mask=None, slop=0) -> np.ndarray:
636
642
  if mask is None:
637
643
  mask = self.and_query(tokens)
638
644
 
@@ -643,10 +649,10 @@ class SearchArray(ExtensionArray):
643
649
  posns = [self.positions(token, mask) for token in tokens]
644
650
  phrase_freqs = np.zeros(len(self))
645
651
 
646
- phrase_freqs[mask] = scan_merge_ins(posns, phrase_freqs[mask], slop=slop)
652
+ phrase_freqs[mask] = scan_merge_ins(posns, phrase_freqs[mask], slop=slop + 1)
647
653
  return phrase_freqs
648
654
 
649
- def phrase_freq_every_diff(self, tokens: List[str], slop=1) -> np.ndarray:
655
+ def phrase_freq_every_diff(self, tokens: List[str], slop=0) -> np.ndarray:
650
656
  phrase_freqs = -np.ones(len(self))
651
657
 
652
658
  mask = self.and_query(tokens)
@@ -658,7 +664,7 @@ class SearchArray(ExtensionArray):
658
664
  for width in [10, 20, 30, 40]:
659
665
  phrase_freqs[mask] = compute_phrase_freqs(term_posns,
660
666
  phrase_freqs[mask],
661
- slop=slop,
667
+ slop=slop + 1,
662
668
  width=width)
663
669
 
664
670
  remaining_mask = phrase_freqs == -1
@@ -1,3 +1,4 @@
1
1
  from searcharray.roaringish.roaringish import RoaringishEncoder, convert_keys # noqa
2
2
  from searcharray.roaringish.snp_ops import popcount64 # noqa
3
3
  from searcharray.roaringish.snp_ops import merge, intersect, unique, adjacent # noqa
4
+ from searcharray.roaringish.spans import span_search # noqa