searcharray 0.0.46__tar.gz → 0.0.47__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {searcharray-0.0.46 → searcharray-0.0.47}/PKG-INFO +1 -1
- {searcharray-0.0.46 → searcharray-0.0.47}/searcharray/indexing.py +1 -1
- {searcharray-0.0.46 → searcharray-0.0.47}/searcharray/phrase/bigram_freqs.py +2 -2
- {searcharray-0.0.46 → searcharray-0.0.47}/searcharray/phrase/middle_out.py +7 -1
- {searcharray-0.0.46 → searcharray-0.0.47}/searcharray/phrase/scan_merge.py +1 -1
- searcharray-0.0.47/searcharray/phrase/spans.py +132 -0
- {searcharray-0.0.46 → searcharray-0.0.47}/searcharray/postings.py +22 -34
- {searcharray-0.0.46 → searcharray-0.0.47}/searcharray/roaringish/__init__.py +1 -0
- {searcharray-0.0.46 → searcharray-0.0.47}/searcharray/roaringish/roaringish_ops.c +193 -163
- {searcharray-0.0.46 → searcharray-0.0.47}/searcharray/roaringish/snp_ops.c +192 -162
- {searcharray-0.0.46 → searcharray-0.0.47}/searcharray/roaringish/snp_ops.pxd +1 -0
- {searcharray-0.0.46 → searcharray-0.0.47}/searcharray/roaringish/snp_ops.pyx +2 -2
- searcharray-0.0.47/searcharray/roaringish/spans.c +30312 -0
- searcharray-0.0.47/searcharray/roaringish/spans.pyx +78 -0
- {searcharray-0.0.46 → searcharray-0.0.47}/searcharray/similarity.py +16 -8
- {searcharray-0.0.46 → searcharray-0.0.47}/searcharray.egg-info/PKG-INFO +1 -1
- {searcharray-0.0.46 → searcharray-0.0.47}/searcharray.egg-info/SOURCES.txt +5 -0
- {searcharray-0.0.46 → searcharray-0.0.47}/setup.py +1 -1
- {searcharray-0.0.46 → searcharray-0.0.47}/test/test_search.py +17 -0
- searcharray-0.0.47/test/test_similarity.py +44 -0
- searcharray-0.0.47/test/test_slop_matches.py +52 -0
- {searcharray-0.0.46 → searcharray-0.0.47}/test/test_snp_ops.py +25 -7
- {searcharray-0.0.46 → searcharray-0.0.47}/test/test_solr.py +2 -2
- {searcharray-0.0.46 → searcharray-0.0.47}/README.md +0 -0
- {searcharray-0.0.46 → searcharray-0.0.47}/pyproject.toml +0 -0
- {searcharray-0.0.46 → searcharray-0.0.47}/searcharray/__init__.py +0 -0
- {searcharray-0.0.46 → searcharray-0.0.47}/searcharray/phrase/__init__.py +0 -0
- {searcharray-0.0.46 → searcharray-0.0.47}/searcharray/phrase/posn_diffs.py +0 -0
- {searcharray-0.0.46 → searcharray-0.0.47}/searcharray/roaringish/roaringish.py +0 -0
- {searcharray-0.0.46 → searcharray-0.0.47}/searcharray/roaringish/roaringish_ops.pyx +0 -0
- {searcharray-0.0.46 → searcharray-0.0.47}/searcharray/solr.py +0 -0
- {searcharray-0.0.46 → searcharray-0.0.47}/searcharray/term_dict.py +0 -0
- {searcharray-0.0.46 → searcharray-0.0.47}/searcharray/utils/__init__.py +0 -0
- {searcharray-0.0.46 → searcharray-0.0.47}/searcharray/utils/bitcount.py +0 -0
- {searcharray-0.0.46 → searcharray-0.0.47}/searcharray/utils/mat_set.py +0 -0
- {searcharray-0.0.46 → searcharray-0.0.47}/searcharray/utils/row_viewable_matrix.py +0 -0
- {searcharray-0.0.46 → searcharray-0.0.47}/searcharray.egg-info/dependency_links.txt +0 -0
- {searcharray-0.0.46 → searcharray-0.0.47}/searcharray.egg-info/entry_points.txt +0 -0
- {searcharray-0.0.46 → searcharray-0.0.47}/searcharray.egg-info/requires.txt +0 -0
- {searcharray-0.0.46 → searcharray-0.0.47}/searcharray.egg-info/top_level.txt +0 -0
- {searcharray-0.0.46 → searcharray-0.0.47}/setup.cfg +0 -0
- {searcharray-0.0.46 → searcharray-0.0.47}/test/test_bitcount64.py +0 -0
- {searcharray-0.0.46 → searcharray-0.0.47}/test/test_extension_array.py +0 -0
- {searcharray-0.0.46 → searcharray-0.0.47}/test/test_minmax_posns.py +0 -0
- {searcharray-0.0.46 → searcharray-0.0.47}/test/test_msmarco.py +0 -0
- {searcharray-0.0.46 → searcharray-0.0.47}/test/test_phrase_matches.py +0 -0
- {searcharray-0.0.46 → searcharray-0.0.47}/test/test_tmdb.py +0 -0
- {searcharray-0.0.46 → searcharray-0.0.47}/test/test_utils.py +0 -0
|
@@ -59,7 +59,7 @@ def _gather_tokens(array, tokenizer, term_dict, term_doc, start_doc_id=0, trunc_
|
|
|
59
59
|
term_doc.append(np.unique(terms))
|
|
60
60
|
|
|
61
61
|
if idx % 10000 == 0 and idx > 0:
|
|
62
|
-
logger.info(f"Tokenized {doc_id} ({100.0 * (idx/len(array))}%)")
|
|
62
|
+
logger.info(f"Tokenized {doc_id} ({100.0 * (idx / len(array))}%)")
|
|
63
63
|
|
|
64
64
|
# Flatten each
|
|
65
65
|
all_terms = np.concatenate(all_terms)
|
|
@@ -58,7 +58,7 @@ def _adj_to_phrase_freq(overlap: np.ndarray, adjacents: np.ndarray) -> np.ndarra
|
|
|
58
58
|
# [foo foo] [foo foo] mal [foo foo] -> 4 adjs, phrase freqs = 3
|
|
59
59
|
consecutive_ones = encoder.payload_lsb(overlap & (overlap << _1))
|
|
60
60
|
consecutive_ones = popcount64(consecutive_ones)
|
|
61
|
-
adjacents -= -np.floor_divide(consecutive_ones, -2
|
|
61
|
+
adjacents -= -np.floor_divide(consecutive_ones, -2, dtype=np.int64)
|
|
62
62
|
return adjacents
|
|
63
63
|
|
|
64
64
|
|
|
@@ -80,7 +80,7 @@ def _inner_bigram_same_term(lhs_int: np.ndarray, rhs_int: np.ndarray,
|
|
|
80
80
|
# Count these bits...
|
|
81
81
|
overlap = lhs_int & rhs_shift
|
|
82
82
|
adj_count = encoder.payload_lsb(overlap)
|
|
83
|
-
adjacents = popcount64(adj_count).
|
|
83
|
+
adjacents = popcount64(adj_count).view(np.int64)
|
|
84
84
|
|
|
85
85
|
phrase_freqs[lhs_doc_ids] += _adj_to_phrase_freq(overlap, adjacents)
|
|
86
86
|
# Continue with ?? ?? foo foo
|
|
@@ -10,6 +10,7 @@ from copy import deepcopy
|
|
|
10
10
|
from typing import List, Tuple, Dict, Union, cast, Optional
|
|
11
11
|
from searcharray.roaringish import RoaringishEncoder, convert_keys, merge
|
|
12
12
|
from searcharray.phrase.bigram_freqs import bigram_freqs, Continuation
|
|
13
|
+
from searcharray.phrase.spans import span_search
|
|
13
14
|
import numbers
|
|
14
15
|
import logging
|
|
15
16
|
from collections import defaultdict
|
|
@@ -337,6 +338,7 @@ class PosnBitArray:
|
|
|
337
338
|
|
|
338
339
|
def phrase_freqs(self, term_ids: List[int], phrase_freqs: np.ndarray,
|
|
339
340
|
doc_ids: np.ndarray,
|
|
341
|
+
slop: int = 0,
|
|
340
342
|
min_posn: Optional[int] = None,
|
|
341
343
|
max_posn: Optional[int] = None) -> np.ndarray:
|
|
342
344
|
if len(term_ids) < 2:
|
|
@@ -352,7 +354,11 @@ class PosnBitArray:
|
|
|
352
354
|
keys=keys,
|
|
353
355
|
min_payload=min_posn,
|
|
354
356
|
max_payload=max_posn) for term_id in term_ids]
|
|
355
|
-
|
|
357
|
+
|
|
358
|
+
if slop == 0:
|
|
359
|
+
return compute_phrase_freqs(enc_term_posns, phrase_freqs)
|
|
360
|
+
else:
|
|
361
|
+
return span_search(enc_term_posns, phrase_freqs, slop)
|
|
356
362
|
|
|
357
363
|
def positions(self, term_id: int, doc_ids) -> Union[List[np.ndarray], np.ndarray]:
|
|
358
364
|
if isinstance(doc_ids, numbers.Number):
|
|
@@ -24,7 +24,7 @@ def _self_adjs(prior_posns, next_posns):
|
|
|
24
24
|
|
|
25
25
|
|
|
26
26
|
def scan_merge_ins(term_posns: List[List[np.ndarray]],
|
|
27
|
-
phrase_freqs: np.ndarray, slop=
|
|
27
|
+
phrase_freqs: np.ndarray, slop=0) -> np.ndarray:
|
|
28
28
|
"""Merge bigram, by bigram, using np.searchsorted to find if insert posns match slop.
|
|
29
29
|
|
|
30
30
|
Description:
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
"""Find span matches up to PAYLOAD_LSB bits span distance."""
|
|
2
|
+
# Colab notebooks
|
|
3
|
+
# https://colab.research.google.com/drive/1H4g7eeaYaHT1dJ5dmcmZUcwKNT-14tYs?authuser=1
|
|
4
|
+
# Chat GPT convo
|
|
5
|
+
# https://chat.openai.com/share/b97c61e6-95d0-40e7-a8cc-447dc1495314
|
|
6
|
+
import numpy as np
|
|
7
|
+
from typing import List
|
|
8
|
+
from searcharray.roaringish import intersect, adjacent, merge, span_search as r_span_search
|
|
9
|
+
# from searcharary.roaringish import intersect_all
|
|
10
|
+
from searcharray.roaringish.roaringish import RoaringishEncoder
|
|
11
|
+
|
|
12
|
+
encoder = RoaringishEncoder()
|
|
13
|
+
|
|
14
|
+
# To not constantly type coerce
|
|
15
|
+
_64 = np.uint64(64)
|
|
16
|
+
_2 = np.uint64(2)
|
|
17
|
+
_1 = np.uint64(1)
|
|
18
|
+
_0 = np.uint64(0)
|
|
19
|
+
_neg1 = np.int64(-1)
|
|
20
|
+
_upper_bit = _1 << (encoder.payload_lsb_bits - _1)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
# ---
|
|
24
|
+
# How to find max spans in roaringish
|
|
25
|
+
# MSB
|
|
26
|
+
# term A |10| | 12| 13| | 18 * Intersect A+B -> 10,12
|
|
27
|
+
# term B |10| | 12| | 14 15 17 | * RHS intersect -> 13 -> 14
|
|
28
|
+
# term C | 11 14
|
|
29
|
+
# term D 10 | 15
|
|
30
|
+
# term E | 11 12
|
|
31
|
+
# ------
|
|
32
|
+
#
|
|
33
|
+
#
|
|
34
|
+
# term A |10| 11 | 12| 13| * Intersect A+B -> 10,12
|
|
35
|
+
# term B |10| 11 | 12| 13| * Intersect A+B -> 10,12
|
|
36
|
+
# term C 9 | | 11 |
|
|
37
|
+
#
|
|
38
|
+
# ----
|
|
39
|
+
#
|
|
40
|
+
# Finally: X X X X <- 4 words selected from each to find max span
|
|
41
|
+
#
|
|
42
|
+
#
|
|
43
|
+
def _intersect_all(posns_encoded: List[np.ndarray]):
|
|
44
|
+
"""Intersect all encoded positions at roaringish MSBs."""
|
|
45
|
+
if len(posns_encoded) < 2:
|
|
46
|
+
raise ValueError("Need at least two positions to intersect")
|
|
47
|
+
|
|
48
|
+
lhs_int = posns_encoded[0]
|
|
49
|
+
lhs_to_left = posns_encoded[0]
|
|
50
|
+
lhs_to_right = posns_encoded[0]
|
|
51
|
+
|
|
52
|
+
for term_idx, posns_next in enumerate(posns_encoded[1:]):
|
|
53
|
+
_, rhs_int = intersect(lhs_int, posns_next, mask=encoder.header_mask)
|
|
54
|
+
_, rhs_ls = adjacent(lhs_to_left, posns_next, mask=encoder.header_mask)
|
|
55
|
+
rhs_rs, _ = adjacent(lhs_to_right, posns_next, mask=encoder.header_mask)
|
|
56
|
+
|
|
57
|
+
# Update LHS to rhs_int + rhs_ls + rhs_rs indices
|
|
58
|
+
lhs_int = posns_next[rhs_int]
|
|
59
|
+
lhs_to_left = posns_next[rhs_ls]
|
|
60
|
+
lhs_to_right = posns_next[rhs_rs]
|
|
61
|
+
|
|
62
|
+
# Get active MSBs now
|
|
63
|
+
# Merge all the rest to grab them
|
|
64
|
+
int_header = encoder.header(lhs_int)
|
|
65
|
+
to_left_header = encoder.header(lhs_to_left)
|
|
66
|
+
to_right_header = encoder.header(lhs_to_right)
|
|
67
|
+
merged = merge(int_header, to_left_header, drop_duplicates=True)
|
|
68
|
+
merged = merge(merged, to_right_header, drop_duplicates=True)
|
|
69
|
+
|
|
70
|
+
# Slice only matches at MSBs
|
|
71
|
+
for i in range(len(posns_encoded)):
|
|
72
|
+
posns_encoded[i] = encoder.slice(posns_encoded[i], merged)
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
# Picking up from intersections:
|
|
76
|
+
#
|
|
77
|
+
# Finally: X X X X X X <- 6 words selected from each to find max span
|
|
78
|
+
#
|
|
79
|
+
# Each X may be adjacent or not
|
|
80
|
+
#
|
|
81
|
+
# Looking at LSBs:
|
|
82
|
+
# MSB: 9 10 13 14 15 20
|
|
83
|
+
# Term A 01001 00000 00000 01000 00001 00100
|
|
84
|
+
# Term B 00010 00010 00010 00000 01000 00010
|
|
85
|
+
# Term C 10000 00001 00000 00001 00000 00001
|
|
86
|
+
#
|
|
87
|
+
# Limit to two adj words at a time (slop must be < LSB bits)
|
|
88
|
+
#
|
|
89
|
+
# MSB: 9 10 13 14 15 20
|
|
90
|
+
# Term A 01001 00000 00000 01000 00001 00100
|
|
91
|
+
# Term B 00010 00010 00010 00000 01000 00010
|
|
92
|
+
# Term C 10000 00001 00000 00001 00000 00001
|
|
93
|
+
# -----------
|
|
94
|
+
# (has span?)
|
|
95
|
+
# -----------
|
|
96
|
+
# (has span?)
|
|
97
|
+
# -----------
|
|
98
|
+
# (has span?)
|
|
99
|
+
# -----
|
|
100
|
+
# (has span?)
|
|
101
|
+
#
|
|
102
|
+
# Has span algo, really finding minimum sized spans
|
|
103
|
+
#
|
|
104
|
+
# Concat to single 64 bit word
|
|
105
|
+
# 0100100000
|
|
106
|
+
# 0001000010
|
|
107
|
+
# 1000000001
|
|
108
|
+
#
|
|
109
|
+
# Search w/ mask
|
|
110
|
+
# ffff000000
|
|
111
|
+
# And'd, now check if > 0:
|
|
112
|
+
# 0100000000 > 0 -> 1 0 0
|
|
113
|
+
# 0001000000 > 0 -> 1 1 0
|
|
114
|
+
# 1000000000 > 0 -> 1 1 0
|
|
115
|
+
#
|
|
116
|
+
# has_span (ANDED): 1
|
|
117
|
+
#
|
|
118
|
+
# Can we shrink the mask?
|
|
119
|
+
# If any hase leading or trailing 0s in mask, then shrink
|
|
120
|
+
def span_search(posns_encoded: List[np.ndarray],
|
|
121
|
+
phrase_freqs: np.ndarray,
|
|
122
|
+
slop: int) -> np.ndarray:
|
|
123
|
+
"""Find span matches up to PAYLOAD_LSB bits span distance."""
|
|
124
|
+
# Find inner span candidates
|
|
125
|
+
_intersect_all(posns_encoded)
|
|
126
|
+
|
|
127
|
+
# Populate phrase freqs with matches of slop
|
|
128
|
+
r_span_search(posns_encoded, phrase_freqs, slop,
|
|
129
|
+
encoder.key_mask,
|
|
130
|
+
encoder.header_mask,
|
|
131
|
+
encoder.payload_lsb_bits)
|
|
132
|
+
return phrase_freqs
|
|
@@ -15,7 +15,7 @@ import numpy as np
|
|
|
15
15
|
from searcharray.phrase.scan_merge import scan_merge_ins
|
|
16
16
|
from searcharray.phrase.posn_diffs import compute_phrase_freqs
|
|
17
17
|
from searcharray.phrase.middle_out import PosnBitArray
|
|
18
|
-
from searcharray.similarity import Similarity, default_bm25
|
|
18
|
+
from searcharray.similarity import Similarity, default_bm25
|
|
19
19
|
from searcharray.indexing import build_index_from_tokenizer, build_index_from_terms_list
|
|
20
20
|
from searcharray.term_dict import TermMissingError
|
|
21
21
|
from searcharray.roaringish.roaringish_ops import as_dense
|
|
@@ -217,7 +217,6 @@ class SearchArray(ExtensionArray):
|
|
|
217
217
|
if not is_list_like(postings):
|
|
218
218
|
raise TypeError("Expected list-like object, got {}".format(type(postings)))
|
|
219
219
|
|
|
220
|
-
self.scoring_context = None
|
|
221
220
|
self.avoid_copies = avoid_copies
|
|
222
221
|
self.tokenizer = tokenizer
|
|
223
222
|
self.term_mat, self.posns, \
|
|
@@ -495,11 +494,9 @@ class SearchArray(ExtensionArray):
|
|
|
495
494
|
postings_arr.posns = self.posns
|
|
496
495
|
postings_arr.term_dict = self.term_dict
|
|
497
496
|
postings_arr.avg_doc_length = self.avg_doc_length
|
|
498
|
-
postings_arr.scoring_context = self.scoring_context
|
|
499
497
|
postings_arr.corpus_size = self.corpus_size
|
|
500
498
|
|
|
501
499
|
if not self.avoid_copies:
|
|
502
|
-
postings_arr.scoring_context = None
|
|
503
500
|
postings_arr.posns = self.posns.copy()
|
|
504
501
|
postings_arr.term_dict = self.term_dict.copy()
|
|
505
502
|
return postings_arr
|
|
@@ -532,11 +529,12 @@ class SearchArray(ExtensionArray):
|
|
|
532
529
|
# Search functionality
|
|
533
530
|
# ***********************************************************
|
|
534
531
|
def termfreqs(self, token: Union[List[str], str],
|
|
532
|
+
slop: int = 0,
|
|
535
533
|
min_posn: Optional[int] = None,
|
|
536
534
|
max_posn: Optional[int] = None) -> np.ndarray:
|
|
537
535
|
token = self._check_token_arg(token)
|
|
538
536
|
if isinstance(token, list):
|
|
539
|
-
return self.phrase_freq(token, min_posn=min_posn, max_posn=max_posn)
|
|
537
|
+
return self.phrase_freq(token, slop=slop, min_posn=min_posn, max_posn=max_posn)
|
|
540
538
|
|
|
541
539
|
try:
|
|
542
540
|
term_id = self.term_dict.get_term_id(token)
|
|
@@ -552,7 +550,6 @@ class SearchArray(ExtensionArray):
|
|
|
552
550
|
else:
|
|
553
551
|
doc_ids, termfreqs = self.posns.termfreqs(term_id,
|
|
554
552
|
doc_ids=slice_of_rows, min_posn=min_posn, max_posn=max_posn)
|
|
555
|
-
# import pdb; pdb.set_trace()
|
|
556
553
|
# matches[doc_ids] = termfreqs
|
|
557
554
|
return as_dense(doc_ids, termfreqs, len(self))
|
|
558
555
|
except TermMissingError:
|
|
@@ -570,11 +567,11 @@ class SearchArray(ExtensionArray):
|
|
|
570
567
|
def doclengths(self) -> np.ndarray:
|
|
571
568
|
return self.doc_lens
|
|
572
569
|
|
|
573
|
-
def match(self, token: Union[List[str], str], slop: int =
|
|
570
|
+
def match(self, token: Union[List[str], str], slop: int = 0) -> np.ndarray:
|
|
574
571
|
"""Return a boolean numpy array indicating which elements contain the given term."""
|
|
575
572
|
token = self._check_token_arg(token)
|
|
576
573
|
if isinstance(token, list):
|
|
577
|
-
term_freq = self.phrase_freq(token)
|
|
574
|
+
term_freq = self.phrase_freq(token, slop=slop)
|
|
578
575
|
else:
|
|
579
576
|
term_freq = self.termfreqs(token)
|
|
580
577
|
return term_freq > 0
|
|
@@ -601,14 +598,7 @@ class SearchArray(ExtensionArray):
|
|
|
601
598
|
token = self._check_token_arg(token)
|
|
602
599
|
doc_lens = self.doclengths()
|
|
603
600
|
|
|
604
|
-
|
|
605
|
-
avg_doc_lens=self.avg_doc_length,
|
|
606
|
-
num_docs=self.corpus_size)
|
|
607
|
-
|
|
608
|
-
if not context.same_as(self.scoring_context):
|
|
609
|
-
self.scoring_context = context
|
|
610
|
-
|
|
611
|
-
scores = similarity(tfs, all_dfs, self.scoring_context)
|
|
601
|
+
scores = similarity(tfs, all_dfs, doc_lens, self.avg_doc_length, self.corpus_size)
|
|
612
602
|
return scores
|
|
613
603
|
|
|
614
604
|
def positions(self, token: str, key=None) -> List[np.ndarray]:
|
|
@@ -633,24 +623,22 @@ class SearchArray(ExtensionArray):
|
|
|
633
623
|
return mask
|
|
634
624
|
|
|
635
625
|
def phrase_freq(self, tokens: List[str],
|
|
636
|
-
slop=
|
|
626
|
+
slop=0,
|
|
637
627
|
min_posn: Optional[int] = None,
|
|
638
628
|
max_posn: Optional[int] = None) -> np.ndarray:
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
else:
|
|
651
|
-
return self.phrase_freq_every_diff(tokens, slop=slop)
|
|
629
|
+
phrase_freqs = np.zeros(len(self))
|
|
630
|
+
try:
|
|
631
|
+
doc_ids = self.term_mat.rows
|
|
632
|
+
term_ids = [self.term_dict.get_term_id(token) for token in tokens]
|
|
633
|
+
return self.posns.phrase_freqs(term_ids, doc_ids=doc_ids,
|
|
634
|
+
phrase_freqs=phrase_freqs,
|
|
635
|
+
slop=slop,
|
|
636
|
+
min_posn=min_posn,
|
|
637
|
+
max_posn=max_posn)
|
|
638
|
+
except TermMissingError:
|
|
639
|
+
return phrase_freqs
|
|
652
640
|
|
|
653
|
-
def phrase_freq_scan(self, tokens: List[str], mask=None, slop=
|
|
641
|
+
def phrase_freq_scan(self, tokens: List[str], mask=None, slop=0) -> np.ndarray:
|
|
654
642
|
if mask is None:
|
|
655
643
|
mask = self.and_query(tokens)
|
|
656
644
|
|
|
@@ -661,10 +649,10 @@ class SearchArray(ExtensionArray):
|
|
|
661
649
|
posns = [self.positions(token, mask) for token in tokens]
|
|
662
650
|
phrase_freqs = np.zeros(len(self))
|
|
663
651
|
|
|
664
|
-
phrase_freqs[mask] = scan_merge_ins(posns, phrase_freqs[mask], slop=slop)
|
|
652
|
+
phrase_freqs[mask] = scan_merge_ins(posns, phrase_freqs[mask], slop=slop + 1)
|
|
665
653
|
return phrase_freqs
|
|
666
654
|
|
|
667
|
-
def phrase_freq_every_diff(self, tokens: List[str], slop=
|
|
655
|
+
def phrase_freq_every_diff(self, tokens: List[str], slop=0) -> np.ndarray:
|
|
668
656
|
phrase_freqs = -np.ones(len(self))
|
|
669
657
|
|
|
670
658
|
mask = self.and_query(tokens)
|
|
@@ -676,7 +664,7 @@ class SearchArray(ExtensionArray):
|
|
|
676
664
|
for width in [10, 20, 30, 40]:
|
|
677
665
|
phrase_freqs[mask] = compute_phrase_freqs(term_posns,
|
|
678
666
|
phrase_freqs[mask],
|
|
679
|
-
slop=slop,
|
|
667
|
+
slop=slop + 1,
|
|
680
668
|
width=width)
|
|
681
669
|
|
|
682
670
|
remaining_mask = phrase_freqs == -1
|
|
@@ -1,3 +1,4 @@
|
|
|
1
1
|
from searcharray.roaringish.roaringish import RoaringishEncoder, convert_keys # noqa
|
|
2
2
|
from searcharray.roaringish.snp_ops import popcount64 # noqa
|
|
3
3
|
from searcharray.roaringish.snp_ops import merge, intersect, unique, adjacent # noqa
|
|
4
|
+
from searcharray.roaringish.spans import span_search # noqa
|