searcharray 0.0.45__tar.gz → 0.0.47__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {searcharray-0.0.45 → searcharray-0.0.47}/PKG-INFO +1 -1
- {searcharray-0.0.45 → searcharray-0.0.47}/searcharray/indexing.py +1 -1
- {searcharray-0.0.45 → searcharray-0.0.47}/searcharray/phrase/bigram_freqs.py +2 -2
- {searcharray-0.0.45 → searcharray-0.0.47}/searcharray/phrase/middle_out.py +7 -1
- {searcharray-0.0.45 → searcharray-0.0.47}/searcharray/phrase/scan_merge.py +1 -1
- searcharray-0.0.47/searcharray/phrase/spans.py +132 -0
- {searcharray-0.0.45 → searcharray-0.0.47}/searcharray/postings.py +35 -29
- {searcharray-0.0.45 → searcharray-0.0.47}/searcharray/roaringish/__init__.py +1 -0
- {searcharray-0.0.45 → searcharray-0.0.47}/searcharray/roaringish/roaringish_ops.c +895 -281
- {searcharray-0.0.45 → searcharray-0.0.47}/searcharray/roaringish/roaringish_ops.pyx +26 -2
- {searcharray-0.0.45 → searcharray-0.0.47}/searcharray/roaringish/snp_ops.c +192 -162
- {searcharray-0.0.45 → searcharray-0.0.47}/searcharray/roaringish/snp_ops.pxd +1 -0
- {searcharray-0.0.45 → searcharray-0.0.47}/searcharray/roaringish/snp_ops.pyx +2 -2
- searcharray-0.0.47/searcharray/roaringish/spans.c +30312 -0
- searcharray-0.0.47/searcharray/roaringish/spans.pyx +78 -0
- searcharray-0.0.47/searcharray/similarity.py +110 -0
- {searcharray-0.0.45 → searcharray-0.0.47}/searcharray.egg-info/PKG-INFO +1 -1
- {searcharray-0.0.45 → searcharray-0.0.47}/searcharray.egg-info/SOURCES.txt +5 -0
- {searcharray-0.0.45 → searcharray-0.0.47}/setup.py +1 -1
- {searcharray-0.0.45 → searcharray-0.0.47}/test/test_msmarco.py +7 -4
- {searcharray-0.0.45 → searcharray-0.0.47}/test/test_search.py +17 -0
- searcharray-0.0.47/test/test_similarity.py +44 -0
- searcharray-0.0.47/test/test_slop_matches.py +52 -0
- {searcharray-0.0.45 → searcharray-0.0.47}/test/test_snp_ops.py +25 -7
- {searcharray-0.0.45 → searcharray-0.0.47}/test/test_solr.py +3 -2
- {searcharray-0.0.45 → searcharray-0.0.47}/test/test_tmdb.py +4 -1
- searcharray-0.0.45/searcharray/similarity.py +0 -82
- {searcharray-0.0.45 → searcharray-0.0.47}/README.md +0 -0
- {searcharray-0.0.45 → searcharray-0.0.47}/pyproject.toml +0 -0
- {searcharray-0.0.45 → searcharray-0.0.47}/searcharray/__init__.py +0 -0
- {searcharray-0.0.45 → searcharray-0.0.47}/searcharray/phrase/__init__.py +0 -0
- {searcharray-0.0.45 → searcharray-0.0.47}/searcharray/phrase/posn_diffs.py +0 -0
- {searcharray-0.0.45 → searcharray-0.0.47}/searcharray/roaringish/roaringish.py +0 -0
- {searcharray-0.0.45 → searcharray-0.0.47}/searcharray/solr.py +0 -0
- {searcharray-0.0.45 → searcharray-0.0.47}/searcharray/term_dict.py +0 -0
- {searcharray-0.0.45 → searcharray-0.0.47}/searcharray/utils/__init__.py +0 -0
- {searcharray-0.0.45 → searcharray-0.0.47}/searcharray/utils/bitcount.py +0 -0
- {searcharray-0.0.45 → searcharray-0.0.47}/searcharray/utils/mat_set.py +0 -0
- {searcharray-0.0.45 → searcharray-0.0.47}/searcharray/utils/row_viewable_matrix.py +0 -0
- {searcharray-0.0.45 → searcharray-0.0.47}/searcharray.egg-info/dependency_links.txt +0 -0
- {searcharray-0.0.45 → searcharray-0.0.47}/searcharray.egg-info/entry_points.txt +0 -0
- {searcharray-0.0.45 → searcharray-0.0.47}/searcharray.egg-info/requires.txt +0 -0
- {searcharray-0.0.45 → searcharray-0.0.47}/searcharray.egg-info/top_level.txt +0 -0
- {searcharray-0.0.45 → searcharray-0.0.47}/setup.cfg +0 -0
- {searcharray-0.0.45 → searcharray-0.0.47}/test/test_bitcount64.py +0 -0
- {searcharray-0.0.45 → searcharray-0.0.47}/test/test_extension_array.py +0 -0
- {searcharray-0.0.45 → searcharray-0.0.47}/test/test_minmax_posns.py +0 -0
- {searcharray-0.0.45 → searcharray-0.0.47}/test/test_phrase_matches.py +0 -0
- {searcharray-0.0.45 → searcharray-0.0.47}/test/test_utils.py +0 -0
|
@@ -59,7 +59,7 @@ def _gather_tokens(array, tokenizer, term_dict, term_doc, start_doc_id=0, trunc_
|
|
|
59
59
|
term_doc.append(np.unique(terms))
|
|
60
60
|
|
|
61
61
|
if idx % 10000 == 0 and idx > 0:
|
|
62
|
-
logger.info(f"Tokenized {doc_id} ({100.0 * (idx/len(array))}%)")
|
|
62
|
+
logger.info(f"Tokenized {doc_id} ({100.0 * (idx / len(array))}%)")
|
|
63
63
|
|
|
64
64
|
# Flatten each
|
|
65
65
|
all_terms = np.concatenate(all_terms)
|
|
@@ -58,7 +58,7 @@ def _adj_to_phrase_freq(overlap: np.ndarray, adjacents: np.ndarray) -> np.ndarra
|
|
|
58
58
|
# [foo foo] [foo foo] mal [foo foo] -> 4 adjs, phrase freqs = 3
|
|
59
59
|
consecutive_ones = encoder.payload_lsb(overlap & (overlap << _1))
|
|
60
60
|
consecutive_ones = popcount64(consecutive_ones)
|
|
61
|
-
adjacents -= -np.floor_divide(consecutive_ones, -2
|
|
61
|
+
adjacents -= -np.floor_divide(consecutive_ones, -2, dtype=np.int64)
|
|
62
62
|
return adjacents
|
|
63
63
|
|
|
64
64
|
|
|
@@ -80,7 +80,7 @@ def _inner_bigram_same_term(lhs_int: np.ndarray, rhs_int: np.ndarray,
|
|
|
80
80
|
# Count these bits...
|
|
81
81
|
overlap = lhs_int & rhs_shift
|
|
82
82
|
adj_count = encoder.payload_lsb(overlap)
|
|
83
|
-
adjacents = popcount64(adj_count).
|
|
83
|
+
adjacents = popcount64(adj_count).view(np.int64)
|
|
84
84
|
|
|
85
85
|
phrase_freqs[lhs_doc_ids] += _adj_to_phrase_freq(overlap, adjacents)
|
|
86
86
|
# Continue with ?? ?? foo foo
|
|
@@ -10,6 +10,7 @@ from copy import deepcopy
|
|
|
10
10
|
from typing import List, Tuple, Dict, Union, cast, Optional
|
|
11
11
|
from searcharray.roaringish import RoaringishEncoder, convert_keys, merge
|
|
12
12
|
from searcharray.phrase.bigram_freqs import bigram_freqs, Continuation
|
|
13
|
+
from searcharray.phrase.spans import span_search
|
|
13
14
|
import numbers
|
|
14
15
|
import logging
|
|
15
16
|
from collections import defaultdict
|
|
@@ -337,6 +338,7 @@ class PosnBitArray:
|
|
|
337
338
|
|
|
338
339
|
def phrase_freqs(self, term_ids: List[int], phrase_freqs: np.ndarray,
|
|
339
340
|
doc_ids: np.ndarray,
|
|
341
|
+
slop: int = 0,
|
|
340
342
|
min_posn: Optional[int] = None,
|
|
341
343
|
max_posn: Optional[int] = None) -> np.ndarray:
|
|
342
344
|
if len(term_ids) < 2:
|
|
@@ -352,7 +354,11 @@ class PosnBitArray:
|
|
|
352
354
|
keys=keys,
|
|
353
355
|
min_payload=min_posn,
|
|
354
356
|
max_payload=max_posn) for term_id in term_ids]
|
|
355
|
-
|
|
357
|
+
|
|
358
|
+
if slop == 0:
|
|
359
|
+
return compute_phrase_freqs(enc_term_posns, phrase_freqs)
|
|
360
|
+
else:
|
|
361
|
+
return span_search(enc_term_posns, phrase_freqs, slop)
|
|
356
362
|
|
|
357
363
|
def positions(self, term_id: int, doc_ids) -> Union[List[np.ndarray], np.ndarray]:
|
|
358
364
|
if isinstance(doc_ids, numbers.Number):
|
|
@@ -24,7 +24,7 @@ def _self_adjs(prior_posns, next_posns):
|
|
|
24
24
|
|
|
25
25
|
|
|
26
26
|
def scan_merge_ins(term_posns: List[List[np.ndarray]],
|
|
27
|
-
phrase_freqs: np.ndarray, slop=
|
|
27
|
+
phrase_freqs: np.ndarray, slop=0) -> np.ndarray:
|
|
28
28
|
"""Merge bigram, by bigram, using np.searchsorted to find if insert posns match slop.
|
|
29
29
|
|
|
30
30
|
Description:
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
"""Find span matches up to PAYLOAD_LSB bits span distance."""
|
|
2
|
+
# Colab notebooks
|
|
3
|
+
# https://colab.research.google.com/drive/1H4g7eeaYaHT1dJ5dmcmZUcwKNT-14tYs?authuser=1
|
|
4
|
+
# Chat GPT convo
|
|
5
|
+
# https://chat.openai.com/share/b97c61e6-95d0-40e7-a8cc-447dc1495314
|
|
6
|
+
import numpy as np
|
|
7
|
+
from typing import List
|
|
8
|
+
from searcharray.roaringish import intersect, adjacent, merge, span_search as r_span_search
|
|
9
|
+
# from searcharary.roaringish import intersect_all
|
|
10
|
+
from searcharray.roaringish.roaringish import RoaringishEncoder
|
|
11
|
+
|
|
12
|
+
encoder = RoaringishEncoder()
|
|
13
|
+
|
|
14
|
+
# To not constantly type coerce
|
|
15
|
+
_64 = np.uint64(64)
|
|
16
|
+
_2 = np.uint64(2)
|
|
17
|
+
_1 = np.uint64(1)
|
|
18
|
+
_0 = np.uint64(0)
|
|
19
|
+
_neg1 = np.int64(-1)
|
|
20
|
+
_upper_bit = _1 << (encoder.payload_lsb_bits - _1)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
# ---
|
|
24
|
+
# How to find max spans in roaringish
|
|
25
|
+
# MSB
|
|
26
|
+
# term A |10| | 12| 13| | 18 * Intersect A+B -> 10,12
|
|
27
|
+
# term B |10| | 12| | 14 15 17 | * RHS intersect -> 13 -> 14
|
|
28
|
+
# term C | 11 14
|
|
29
|
+
# term D 10 | 15
|
|
30
|
+
# term E | 11 12
|
|
31
|
+
# ------
|
|
32
|
+
#
|
|
33
|
+
#
|
|
34
|
+
# term A |10| 11 | 12| 13| * Intersect A+B -> 10,12
|
|
35
|
+
# term B |10| 11 | 12| 13| * Intersect A+B -> 10,12
|
|
36
|
+
# term C 9 | | 11 |
|
|
37
|
+
#
|
|
38
|
+
# ----
|
|
39
|
+
#
|
|
40
|
+
# Finally: X X X X <- 4 words selected from each to find max span
|
|
41
|
+
#
|
|
42
|
+
#
|
|
43
|
+
def _intersect_all(posns_encoded: List[np.ndarray]):
|
|
44
|
+
"""Intersect all encoded positions at roaringish MSBs."""
|
|
45
|
+
if len(posns_encoded) < 2:
|
|
46
|
+
raise ValueError("Need at least two positions to intersect")
|
|
47
|
+
|
|
48
|
+
lhs_int = posns_encoded[0]
|
|
49
|
+
lhs_to_left = posns_encoded[0]
|
|
50
|
+
lhs_to_right = posns_encoded[0]
|
|
51
|
+
|
|
52
|
+
for term_idx, posns_next in enumerate(posns_encoded[1:]):
|
|
53
|
+
_, rhs_int = intersect(lhs_int, posns_next, mask=encoder.header_mask)
|
|
54
|
+
_, rhs_ls = adjacent(lhs_to_left, posns_next, mask=encoder.header_mask)
|
|
55
|
+
rhs_rs, _ = adjacent(lhs_to_right, posns_next, mask=encoder.header_mask)
|
|
56
|
+
|
|
57
|
+
# Update LHS to rhs_int + rhs_ls + rhs_rs indices
|
|
58
|
+
lhs_int = posns_next[rhs_int]
|
|
59
|
+
lhs_to_left = posns_next[rhs_ls]
|
|
60
|
+
lhs_to_right = posns_next[rhs_rs]
|
|
61
|
+
|
|
62
|
+
# Get active MSBs now
|
|
63
|
+
# Merge all the rest to grab them
|
|
64
|
+
int_header = encoder.header(lhs_int)
|
|
65
|
+
to_left_header = encoder.header(lhs_to_left)
|
|
66
|
+
to_right_header = encoder.header(lhs_to_right)
|
|
67
|
+
merged = merge(int_header, to_left_header, drop_duplicates=True)
|
|
68
|
+
merged = merge(merged, to_right_header, drop_duplicates=True)
|
|
69
|
+
|
|
70
|
+
# Slice only matches at MSBs
|
|
71
|
+
for i in range(len(posns_encoded)):
|
|
72
|
+
posns_encoded[i] = encoder.slice(posns_encoded[i], merged)
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
# Picking up from intersections:
|
|
76
|
+
#
|
|
77
|
+
# Finally: X X X X X X <- 6 words selected from each to find max span
|
|
78
|
+
#
|
|
79
|
+
# Each X may be adjacent or not
|
|
80
|
+
#
|
|
81
|
+
# Looking at LSBs:
|
|
82
|
+
# MSB: 9 10 13 14 15 20
|
|
83
|
+
# Term A 01001 00000 00000 01000 00001 00100
|
|
84
|
+
# Term B 00010 00010 00010 00000 01000 00010
|
|
85
|
+
# Term C 10000 00001 00000 00001 00000 00001
|
|
86
|
+
#
|
|
87
|
+
# Limit to two adj words at a time (slop must be < LSB bits)
|
|
88
|
+
#
|
|
89
|
+
# MSB: 9 10 13 14 15 20
|
|
90
|
+
# Term A 01001 00000 00000 01000 00001 00100
|
|
91
|
+
# Term B 00010 00010 00010 00000 01000 00010
|
|
92
|
+
# Term C 10000 00001 00000 00001 00000 00001
|
|
93
|
+
# -----------
|
|
94
|
+
# (has span?)
|
|
95
|
+
# -----------
|
|
96
|
+
# (has span?)
|
|
97
|
+
# -----------
|
|
98
|
+
# (has span?)
|
|
99
|
+
# -----
|
|
100
|
+
# (has span?)
|
|
101
|
+
#
|
|
102
|
+
# Has span algo, really finding minimum sized spans
|
|
103
|
+
#
|
|
104
|
+
# Concat to single 64 bit word
|
|
105
|
+
# 0100100000
|
|
106
|
+
# 0001000010
|
|
107
|
+
# 1000000001
|
|
108
|
+
#
|
|
109
|
+
# Search w/ mask
|
|
110
|
+
# ffff000000
|
|
111
|
+
# And'd, now check if > 0:
|
|
112
|
+
# 0100000000 > 0 -> 1 0 0
|
|
113
|
+
# 0001000000 > 0 -> 1 1 0
|
|
114
|
+
# 1000000000 > 0 -> 1 1 0
|
|
115
|
+
#
|
|
116
|
+
# has_span (ANDED): 1
|
|
117
|
+
#
|
|
118
|
+
# Can we shrink the mask?
|
|
119
|
+
# If any hase leading or trailing 0s in mask, then shrink
|
|
120
|
+
def span_search(posns_encoded: List[np.ndarray],
|
|
121
|
+
phrase_freqs: np.ndarray,
|
|
122
|
+
slop: int) -> np.ndarray:
|
|
123
|
+
"""Find span matches up to PAYLOAD_LSB bits span distance."""
|
|
124
|
+
# Find inner span candidates
|
|
125
|
+
_intersect_all(posns_encoded)
|
|
126
|
+
|
|
127
|
+
# Populate phrase freqs with matches of slop
|
|
128
|
+
r_span_search(posns_encoded, phrase_freqs, slop,
|
|
129
|
+
encoder.key_mask,
|
|
130
|
+
encoder.header_mask,
|
|
131
|
+
encoder.payload_lsb_bits)
|
|
132
|
+
return phrase_freqs
|
|
@@ -18,6 +18,7 @@ from searcharray.phrase.middle_out import PosnBitArray
|
|
|
18
18
|
from searcharray.similarity import Similarity, default_bm25
|
|
19
19
|
from searcharray.indexing import build_index_from_tokenizer, build_index_from_terms_list
|
|
20
20
|
from searcharray.term_dict import TermMissingError
|
|
21
|
+
from searcharray.roaringish.roaringish_ops import as_dense
|
|
21
22
|
|
|
22
23
|
logger = logging.getLogger(__name__)
|
|
23
24
|
|
|
@@ -221,6 +222,7 @@ class SearchArray(ExtensionArray):
|
|
|
221
222
|
self.term_mat, self.posns, \
|
|
222
223
|
self.term_dict, self.avg_doc_length, \
|
|
223
224
|
self.doc_lens = build_index_from_terms_list(postings, Terms)
|
|
225
|
+
self.corpus_size = len(self.doc_lens)
|
|
224
226
|
|
|
225
227
|
@classmethod
|
|
226
228
|
def index(cls, array: Iterable,
|
|
@@ -266,6 +268,7 @@ class SearchArray(ExtensionArray):
|
|
|
266
268
|
postings.term_dict = term_dict
|
|
267
269
|
postings.avg_doc_length = avg_doc_length
|
|
268
270
|
postings.doc_lens = doc_lens
|
|
271
|
+
postings.corpus_size = len(doc_lens)
|
|
269
272
|
return postings
|
|
270
273
|
|
|
271
274
|
def warm(self):
|
|
@@ -312,13 +315,18 @@ class SearchArray(ExtensionArray):
|
|
|
312
315
|
else:
|
|
313
316
|
# Construct a sliced view of this array
|
|
314
317
|
sliced_tfs = self.term_mat.slice(key)
|
|
315
|
-
|
|
318
|
+
if not self.avoid_copies:
|
|
319
|
+
sliced_posns = self.posns.slice(sliced_tfs.rows) if not self.avoid_copies else self.posns
|
|
320
|
+
else:
|
|
321
|
+
sliced_posns = self.posns
|
|
322
|
+
|
|
316
323
|
arr = SearchArray([], tokenizer=self.tokenizer)
|
|
317
324
|
arr.term_mat = sliced_tfs
|
|
318
325
|
arr.doc_lens = self.doc_lens[key]
|
|
319
326
|
arr.posns = sliced_posns
|
|
320
327
|
arr.term_dict = self.term_dict
|
|
321
328
|
arr.avg_doc_length = self.avg_doc_length
|
|
329
|
+
arr.corpus_size = self.corpus_size
|
|
322
330
|
return arr
|
|
323
331
|
|
|
324
332
|
def __setitem__(self, key, value):
|
|
@@ -486,6 +494,7 @@ class SearchArray(ExtensionArray):
|
|
|
486
494
|
postings_arr.posns = self.posns
|
|
487
495
|
postings_arr.term_dict = self.term_dict
|
|
488
496
|
postings_arr.avg_doc_length = self.avg_doc_length
|
|
497
|
+
postings_arr.corpus_size = self.corpus_size
|
|
489
498
|
|
|
490
499
|
if not self.avoid_copies:
|
|
491
500
|
postings_arr.posns = self.posns.copy()
|
|
@@ -520,17 +529,18 @@ class SearchArray(ExtensionArray):
|
|
|
520
529
|
# Search functionality
|
|
521
530
|
# ***********************************************************
|
|
522
531
|
def termfreqs(self, token: Union[List[str], str],
|
|
532
|
+
slop: int = 0,
|
|
523
533
|
min_posn: Optional[int] = None,
|
|
524
534
|
max_posn: Optional[int] = None) -> np.ndarray:
|
|
525
535
|
token = self._check_token_arg(token)
|
|
526
536
|
if isinstance(token, list):
|
|
527
|
-
return self.phrase_freq(token, min_posn=min_posn, max_posn=max_posn)
|
|
537
|
+
return self.phrase_freq(token, slop=slop, min_posn=min_posn, max_posn=max_posn)
|
|
528
538
|
|
|
529
539
|
try:
|
|
530
540
|
term_id = self.term_dict.get_term_id(token)
|
|
531
|
-
matches = np.zeros(len(self), dtype=np.float32)
|
|
532
541
|
slice_of_rows = None
|
|
533
542
|
if self.term_mat.subset:
|
|
543
|
+
matches = np.zeros(len(self), dtype=np.float32)
|
|
534
544
|
slice_of_rows = self.term_mat.rows
|
|
535
545
|
doc_ids, termfreqs = self.posns.termfreqs(term_id,
|
|
536
546
|
doc_ids=slice_of_rows, min_posn=min_posn, max_posn=max_posn)
|
|
@@ -540,9 +550,8 @@ class SearchArray(ExtensionArray):
|
|
|
540
550
|
else:
|
|
541
551
|
doc_ids, termfreqs = self.posns.termfreqs(term_id,
|
|
542
552
|
doc_ids=slice_of_rows, min_posn=min_posn, max_posn=max_posn)
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
return matches
|
|
553
|
+
# matches[doc_ids] = termfreqs
|
|
554
|
+
return as_dense(doc_ids, termfreqs, len(self))
|
|
546
555
|
except TermMissingError:
|
|
547
556
|
return np.zeros(len(self), dtype=np.float32)
|
|
548
557
|
|
|
@@ -558,11 +567,11 @@ class SearchArray(ExtensionArray):
|
|
|
558
567
|
def doclengths(self) -> np.ndarray:
|
|
559
568
|
return self.doc_lens
|
|
560
569
|
|
|
561
|
-
def match(self, token: Union[List[str], str], slop: int =
|
|
570
|
+
def match(self, token: Union[List[str], str], slop: int = 0) -> np.ndarray:
|
|
562
571
|
"""Return a boolean numpy array indicating which elements contain the given term."""
|
|
563
572
|
token = self._check_token_arg(token)
|
|
564
573
|
if isinstance(token, list):
|
|
565
|
-
term_freq = self.phrase_freq(token)
|
|
574
|
+
term_freq = self.phrase_freq(token, slop=slop)
|
|
566
575
|
else:
|
|
567
576
|
term_freq = self.termfreqs(token)
|
|
568
577
|
return term_freq > 0
|
|
@@ -588,9 +597,8 @@ class SearchArray(ExtensionArray):
|
|
|
588
597
|
tfs = self.termfreqs(token, min_posn=min_posn, max_posn=max_posn)
|
|
589
598
|
token = self._check_token_arg(token)
|
|
590
599
|
doc_lens = self.doclengths()
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
num_docs=len(self))
|
|
600
|
+
|
|
601
|
+
scores = similarity(tfs, all_dfs, doc_lens, self.avg_doc_length, self.corpus_size)
|
|
594
602
|
return scores
|
|
595
603
|
|
|
596
604
|
def positions(self, token: str, key=None) -> List[np.ndarray]:
|
|
@@ -615,24 +623,22 @@ class SearchArray(ExtensionArray):
|
|
|
615
623
|
return mask
|
|
616
624
|
|
|
617
625
|
def phrase_freq(self, tokens: List[str],
|
|
618
|
-
slop=
|
|
626
|
+
slop=0,
|
|
619
627
|
min_posn: Optional[int] = None,
|
|
620
628
|
max_posn: Optional[int] = None) -> np.ndarray:
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
else:
|
|
633
|
-
return self.phrase_freq_every_diff(tokens, slop=slop)
|
|
629
|
+
phrase_freqs = np.zeros(len(self))
|
|
630
|
+
try:
|
|
631
|
+
doc_ids = self.term_mat.rows
|
|
632
|
+
term_ids = [self.term_dict.get_term_id(token) for token in tokens]
|
|
633
|
+
return self.posns.phrase_freqs(term_ids, doc_ids=doc_ids,
|
|
634
|
+
phrase_freqs=phrase_freqs,
|
|
635
|
+
slop=slop,
|
|
636
|
+
min_posn=min_posn,
|
|
637
|
+
max_posn=max_posn)
|
|
638
|
+
except TermMissingError:
|
|
639
|
+
return phrase_freqs
|
|
634
640
|
|
|
635
|
-
def phrase_freq_scan(self, tokens: List[str], mask=None, slop=
|
|
641
|
+
def phrase_freq_scan(self, tokens: List[str], mask=None, slop=0) -> np.ndarray:
|
|
636
642
|
if mask is None:
|
|
637
643
|
mask = self.and_query(tokens)
|
|
638
644
|
|
|
@@ -643,10 +649,10 @@ class SearchArray(ExtensionArray):
|
|
|
643
649
|
posns = [self.positions(token, mask) for token in tokens]
|
|
644
650
|
phrase_freqs = np.zeros(len(self))
|
|
645
651
|
|
|
646
|
-
phrase_freqs[mask] = scan_merge_ins(posns, phrase_freqs[mask], slop=slop)
|
|
652
|
+
phrase_freqs[mask] = scan_merge_ins(posns, phrase_freqs[mask], slop=slop + 1)
|
|
647
653
|
return phrase_freqs
|
|
648
654
|
|
|
649
|
-
def phrase_freq_every_diff(self, tokens: List[str], slop=
|
|
655
|
+
def phrase_freq_every_diff(self, tokens: List[str], slop=0) -> np.ndarray:
|
|
650
656
|
phrase_freqs = -np.ones(len(self))
|
|
651
657
|
|
|
652
658
|
mask = self.and_query(tokens)
|
|
@@ -658,7 +664,7 @@ class SearchArray(ExtensionArray):
|
|
|
658
664
|
for width in [10, 20, 30, 40]:
|
|
659
665
|
phrase_freqs[mask] = compute_phrase_freqs(term_posns,
|
|
660
666
|
phrase_freqs[mask],
|
|
661
|
-
slop=slop,
|
|
667
|
+
slop=slop + 1,
|
|
662
668
|
width=width)
|
|
663
669
|
|
|
664
670
|
remaining_mask = phrase_freqs == -1
|
|
@@ -1,3 +1,4 @@
|
|
|
1
1
|
from searcharray.roaringish.roaringish import RoaringishEncoder, convert_keys # noqa
|
|
2
2
|
from searcharray.roaringish.snp_ops import popcount64 # noqa
|
|
3
3
|
from searcharray.roaringish.snp_ops import merge, intersect, unique, adjacent # noqa
|
|
4
|
+
from searcharray.roaringish.spans import span_search # noqa
|