searcharray 0.0.61__tar.gz → 0.0.63__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. {searcharray-0.0.61 → searcharray-0.0.63}/PKG-INFO +2 -1
  2. {searcharray-0.0.61 → searcharray-0.0.63}/README.md +1 -0
  3. {searcharray-0.0.61 → searcharray-0.0.63}/searcharray/indexing.py +1 -1
  4. {searcharray-0.0.61 → searcharray-0.0.63}/searcharray/phrase/memmap_arrays.py +32 -14
  5. {searcharray-0.0.61 → searcharray-0.0.63}/searcharray/phrase/middle_out.py +2 -0
  6. {searcharray-0.0.61 → searcharray-0.0.63}/searcharray/roaringish/intersect.c +149 -149
  7. {searcharray-0.0.61 → searcharray-0.0.63}/searcharray/roaringish/merge.c +149 -149
  8. {searcharray-0.0.61 → searcharray-0.0.63}/searcharray/roaringish/popcount.c +149 -149
  9. {searcharray-0.0.61 → searcharray-0.0.63}/searcharray/roaringish/roaringish_ops.c +149 -149
  10. {searcharray-0.0.61 → searcharray-0.0.63}/searcharray/roaringish/search.c +149 -149
  11. {searcharray-0.0.61 → searcharray-0.0.63}/searcharray/roaringish/spans.c +243 -241
  12. {searcharray-0.0.61 → searcharray-0.0.63}/searcharray/roaringish/unique.c +149 -149
  13. {searcharray-0.0.61 → searcharray-0.0.63}/searcharray/similarity.py +4 -1
  14. {searcharray-0.0.61 → searcharray-0.0.63}/searcharray.egg-info/PKG-INFO +2 -1
  15. {searcharray-0.0.61 → searcharray-0.0.63}/setup.py +1 -1
  16. {searcharray-0.0.61 → searcharray-0.0.63}/test/test_search.py +11 -0
  17. {searcharray-0.0.61 → searcharray-0.0.63}/test/test_tmdb.py +53 -5
  18. {searcharray-0.0.61 → searcharray-0.0.63}/pyproject.toml +0 -0
  19. {searcharray-0.0.61 → searcharray-0.0.63}/searcharray/__init__.py +0 -0
  20. {searcharray-0.0.61 → searcharray-0.0.63}/searcharray/phrase/__init__.py +0 -0
  21. {searcharray-0.0.61 → searcharray-0.0.63}/searcharray/phrase/bigram_freqs.py +0 -0
  22. {searcharray-0.0.61 → searcharray-0.0.63}/searcharray/phrase/spans.py +0 -0
  23. {searcharray-0.0.61 → searcharray-0.0.63}/searcharray/postings.py +0 -0
  24. {searcharray-0.0.61 → searcharray-0.0.63}/searcharray/roaringish/__init__.py +0 -0
  25. {searcharray-0.0.61 → searcharray-0.0.63}/searcharray/roaringish/intersect.pyi +0 -0
  26. {searcharray-0.0.61 → searcharray-0.0.63}/searcharray/roaringish/intersect.pyx +0 -0
  27. {searcharray-0.0.61 → searcharray-0.0.63}/searcharray/roaringish/merge.pyi +0 -0
  28. {searcharray-0.0.61 → searcharray-0.0.63}/searcharray/roaringish/merge.pyx +0 -0
  29. {searcharray-0.0.61 → searcharray-0.0.63}/searcharray/roaringish/popcount.pyi +0 -0
  30. {searcharray-0.0.61 → searcharray-0.0.63}/searcharray/roaringish/popcount.pyx +0 -0
  31. {searcharray-0.0.61 → searcharray-0.0.63}/searcharray/roaringish/roaringish.py +0 -0
  32. {searcharray-0.0.61 → searcharray-0.0.63}/searcharray/roaringish/roaringish_ops.pyx +0 -0
  33. {searcharray-0.0.61 → searcharray-0.0.63}/searcharray/roaringish/search.pyi +0 -0
  34. {searcharray-0.0.61 → searcharray-0.0.63}/searcharray/roaringish/search.pyx +0 -0
  35. {searcharray-0.0.61 → searcharray-0.0.63}/searcharray/roaringish/snp_ops.pxd +0 -0
  36. {searcharray-0.0.61 → searcharray-0.0.63}/searcharray/roaringish/spans.pyx +0 -0
  37. {searcharray-0.0.61 → searcharray-0.0.63}/searcharray/roaringish/unique.pyi +0 -0
  38. {searcharray-0.0.61 → searcharray-0.0.63}/searcharray/roaringish/unique.pyx +0 -0
  39. {searcharray-0.0.61 → searcharray-0.0.63}/searcharray/solr.py +0 -0
  40. {searcharray-0.0.61 → searcharray-0.0.63}/searcharray/term_dict.py +0 -0
  41. {searcharray-0.0.61 → searcharray-0.0.63}/searcharray/utils/__init__.py +0 -0
  42. {searcharray-0.0.61 → searcharray-0.0.63}/searcharray/utils/bitcount.py +0 -0
  43. {searcharray-0.0.61 → searcharray-0.0.63}/searcharray/utils/mat_set.py +0 -0
  44. {searcharray-0.0.61 → searcharray-0.0.63}/searcharray/utils/row_viewable_matrix.py +0 -0
  45. {searcharray-0.0.61 → searcharray-0.0.63}/searcharray/utils/sort.py +0 -0
  46. {searcharray-0.0.61 → searcharray-0.0.63}/searcharray.egg-info/SOURCES.txt +0 -0
  47. {searcharray-0.0.61 → searcharray-0.0.63}/searcharray.egg-info/dependency_links.txt +0 -0
  48. {searcharray-0.0.61 → searcharray-0.0.63}/searcharray.egg-info/entry_points.txt +0 -0
  49. {searcharray-0.0.61 → searcharray-0.0.63}/searcharray.egg-info/requires.txt +0 -0
  50. {searcharray-0.0.61 → searcharray-0.0.63}/searcharray.egg-info/top_level.txt +0 -0
  51. {searcharray-0.0.61 → searcharray-0.0.63}/setup.cfg +0 -0
  52. {searcharray-0.0.61 → searcharray-0.0.63}/test/test_bitcount64.py +0 -0
  53. {searcharray-0.0.61 → searcharray-0.0.63}/test/test_extension_array.py +0 -0
  54. {searcharray-0.0.61 → searcharray-0.0.63}/test/test_minmax_posns.py +0 -0
  55. {searcharray-0.0.61 → searcharray-0.0.63}/test/test_msmarco.py +0 -0
  56. {searcharray-0.0.61 → searcharray-0.0.63}/test/test_phrase_matches.py +0 -0
  57. {searcharray-0.0.61 → searcharray-0.0.63}/test/test_similarity.py +0 -0
  58. {searcharray-0.0.61 → searcharray-0.0.63}/test/test_slop_matches.py +0 -0
  59. {searcharray-0.0.61 → searcharray-0.0.63}/test/test_snp_ops.py +0 -0
  60. {searcharray-0.0.61 → searcharray-0.0.63}/test/test_solr.py +0 -0
  61. {searcharray-0.0.61 → searcharray-0.0.63}/test/test_utils.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: searcharray
3
- Version: 0.0.61
3
+ Version: 0.0.63
4
4
  Summary: Searchable pandas text extension arrays for prototyping search
5
5
  Home-page: https://github.com/softwaredoug/searcharray
6
6
  Author: Doug Turnbull
@@ -65,6 +65,7 @@ pip install searcharray
65
65
  * Search w/ a [phrase w/ edit-distance](https://lucene.apache.org/core/9_6_0/core/org/apache/lucene/search/PhraseQuery.html) by passing slop=N.
66
66
  * Access raw stats arrays in termfreqs / docfreqs methods on the array
67
67
  * Bring your own tokenizer. Pass any (`def tokenize(value: str) -> List[str]`) when indexing.
68
+ * Memory map by passing `data_dir` to index for memory mapped index
68
69
  * Accepts any python function to compute similarity. Here's [one similarity](https://github.com/softwaredoug/searcharray/blob/main/searcharray/similarity.py#L103)
69
70
  * Scores the entire dataframe, allowing combination w/ other ranking attributes (recency, popularity, etc) or scores from other fields (ie boolean queries)
70
71
  * Implement's Solr's [edismax query parser](https://github.com/softwaredoug/searcharray/blob/main/searcharray/solr.py) for efficient prototyping
@@ -36,6 +36,7 @@ pip install searcharray
36
36
  * Search w/ a [phrase w/ edit-distance](https://lucene.apache.org/core/9_6_0/core/org/apache/lucene/search/PhraseQuery.html) by passing slop=N.
37
37
  * Access raw stats arrays in termfreqs / docfreqs methods on the array
38
38
  * Bring your own tokenizer. Pass any (`def tokenize(value: str) -> List[str]`) when indexing.
39
+ * Memory map by passing `data_dir` to index for memory mapped index
39
40
  * Accepts any python function to compute similarity. Here's [one similarity](https://github.com/softwaredoug/searcharray/blob/main/searcharray/similarity.py#L103)
40
41
  * Scores the entire dataframe, allowing combination w/ other ranking attributes (recency, popularity, etc) or scores from other fields (ie boolean queries)
41
42
  * Implement's Solr's [edismax query parser](https://github.com/softwaredoug/searcharray/blob/main/searcharray/solr.py) for efficient prototyping
@@ -46,7 +46,7 @@ def _compute_doc_lens(posns: np.ndarray, doc_ids: np.ndarray, num_docs: int) ->
46
46
  non_empty_doc_ids = doc_ids[non_empty_idxs]
47
47
  non_empty_doc_lens = non_empty_doc_lens[non_empty_idxs]
48
48
  doc_lens[non_empty_doc_ids] = non_empty_doc_lens
49
- if doc_ids[-1] not in non_empty_doc_ids:
49
+ if len(doc_ids) > 0 and doc_ids[-1] not in non_empty_doc_ids:
50
50
  doc_lens[doc_ids[-1]] = posns[-1] + 1
51
51
  return doc_lens
52
52
 
@@ -1,6 +1,6 @@
1
1
  import numpy as np
2
2
  import os
3
- from typing import Optional, List
3
+ from typing import Optional, List, Dict
4
4
 
5
5
 
6
6
  def create_filename(data_dir: str):
@@ -49,21 +49,39 @@ class ArrayDict:
49
49
 
50
50
  @staticmethod
51
51
  def concat(lhs: 'ArrayDict', rhs: 'ArrayDict', sort=True):
52
- metadata = {}
53
- offset = 0
54
- for key, value in lhs.metadata.items():
55
- metadata[key] = {'offset': offset, 'length': value['length']}
56
- offset += value['length']
57
- for key, value in rhs.metadata.items():
58
- metadata[key] = {'offset': offset, 'length': value['length']}
59
- offset += value['length']
60
- data = np.concatenate((lhs.data, rhs.data))
61
- if sort:
62
- keys = sorted(metadata.keys())
63
- metadata = {key: metadata[key] for key in keys}
64
52
  arr = ArrayDict()
65
- arr.data = data
53
+ metadata: Dict[int, Dict[str, int]] = {}
54
+ lst_of_arrays: List[np.ndarray] = []
55
+
56
+ curr_offset = 0
57
+ last_offset = 0
58
+
59
+ fetched_keys = set()
60
+ for key, value in lhs.items():
61
+ fetched_keys.add(key)
62
+ all_for_key = [value]
63
+ curr_offset += value.size
64
+ if key in rhs.metadata:
65
+ rhs_value = rhs[key]
66
+ all_for_key.append(rhs_value)
67
+ curr_offset += rhs_value.size
68
+ metadata[key] = {'offset': last_offset, 'length': curr_offset - last_offset}
69
+ last_offset = curr_offset
70
+ lst_of_arrays.append(np.concatenate(all_for_key))
71
+
72
+ for key, value in rhs.items():
73
+ if key not in fetched_keys:
74
+ curr_offset += value.size
75
+ lst_of_arrays.append(value)
76
+ metadata[key] = {'offset': last_offset, 'length': curr_offset - last_offset}
77
+ last_offset = curr_offset
78
+
66
79
  arr.metadata = metadata
80
+ for curr_arr in lst_of_arrays:
81
+ if sort:
82
+ curr_arr.sort()
83
+
84
+ arr.data = np.concatenate(lst_of_arrays)
67
85
  return arr
68
86
 
69
87
  def __getitem__(self, key):
@@ -170,6 +170,8 @@ class PosnBitArrayFromFlatBuilder:
170
170
  encoded, enc_term_boundaries = encoder.encode(keys=self.flat_array[1].view(np.uint64),
171
171
  boundaries=term_boundaries[:-1],
172
172
  payload=self.flat_array[2].view(np.uint64))
173
+ if len(encoded) == 0:
174
+ return PosnBitArray({}, self.max_doc_id)
173
175
  term_ids = self.flat_array[0][term_boundaries[:-1]]
174
176
 
175
177
  encoded_term_posns = ArrayDict.from_array_with_boundaries(encoded,