base90-rag 0.2.0__tar.gz → 0.2.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {base90_rag-0.2.0 → base90_rag-0.2.2}/PKG-INFO +1 -1
- {base90_rag-0.2.0 → base90_rag-0.2.2}/base90_rag/core.py +30 -52
- {base90_rag-0.2.0 → base90_rag-0.2.2}/base90_rag.egg-info/PKG-INFO +1 -1
- {base90_rag-0.2.0 → base90_rag-0.2.2}/pyproject.toml +1 -1
- {base90_rag-0.2.0 → base90_rag-0.2.2}/LICENSE +0 -0
- {base90_rag-0.2.0 → base90_rag-0.2.2}/README.md +0 -0
- {base90_rag-0.2.0 → base90_rag-0.2.2}/base90_rag/__init__.py +0 -0
- {base90_rag-0.2.0 → base90_rag-0.2.2}/base90_rag/default_dict.json +0 -0
- {base90_rag-0.2.0 → base90_rag-0.2.2}/base90_rag.egg-info/SOURCES.txt +0 -0
- {base90_rag-0.2.0 → base90_rag-0.2.2}/base90_rag.egg-info/dependency_links.txt +0 -0
- {base90_rag-0.2.0 → base90_rag-0.2.2}/base90_rag.egg-info/top_level.txt +0 -0
- {base90_rag-0.2.0 → base90_rag-0.2.2}/setup.cfg +0 -0
|
@@ -3,12 +3,10 @@ import os
|
|
|
3
3
|
import sqlite3
|
|
4
4
|
import threading
|
|
5
5
|
import time
|
|
6
|
-
import math
|
|
7
6
|
import re
|
|
8
7
|
import string
|
|
9
8
|
import hashlib
|
|
10
9
|
import pkgutil
|
|
11
|
-
import heapq
|
|
12
10
|
import unicodedata
|
|
13
11
|
|
|
14
12
|
|
|
@@ -199,76 +197,56 @@ class Base90Memory:
|
|
|
199
197
|
c.execute("INSERT OR IGNORE INTO inverted_index VALUES (?, ?)", (code, pointer_id))
|
|
200
198
|
return pointer_id
|
|
201
199
|
|
|
202
|
-
def search(self, query: str, limit=
|
|
200
|
+
def search(self, query: str, limit=10) -> list:
|
|
203
201
|
query_codes = set(self._extract_codes(query))
|
|
204
202
|
if not query_codes: return []
|
|
205
203
|
|
|
206
|
-
hit_map = {}
|
|
207
|
-
hit_word_counts = {}
|
|
208
|
-
query_word_count = len(query_codes)
|
|
209
|
-
MAX_POSTINGS = 2000
|
|
210
|
-
|
|
211
204
|
valid_sets = []
|
|
212
205
|
for code in query_codes:
|
|
213
206
|
if code in self.mem_inverted:
|
|
214
207
|
word = self.code_to_word.get(code, "")
|
|
215
|
-
if len(word)
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
valid_sets.append((code, self.mem_inverted[code], word))
|
|
208
|
+
if len(word) == 1 and word in "的了是我你他她它": continue
|
|
209
|
+
if len(word) == 1 and word.isascii() and word.isalpha(): continue
|
|
210
|
+
|
|
211
|
+
valid_sets.append(self.mem_inverted[code])
|
|
220
212
|
|
|
221
|
-
if not valid_sets:
|
|
222
|
-
return []
|
|
213
|
+
if not valid_sets: return []
|
|
223
214
|
|
|
224
|
-
valid_sets.sort(key=
|
|
215
|
+
valid_sets.sort(key=len)
|
|
225
216
|
|
|
226
|
-
candidate_pids = valid_sets[0]
|
|
217
|
+
candidate_pids = valid_sets[0].copy()
|
|
227
218
|
for i in range(1, len(valid_sets)):
|
|
228
|
-
candidate_pids = candidate_pids.intersection(valid_sets[i]
|
|
219
|
+
candidate_pids = candidate_pids.intersection(valid_sets[i])
|
|
229
220
|
if not candidate_pids:
|
|
230
221
|
break
|
|
231
222
|
|
|
232
|
-
if
|
|
233
|
-
return []
|
|
234
|
-
|
|
235
|
-
if len(candidate_pids) > MAX_POSTINGS:
|
|
223
|
+
if len(candidate_pids) > 2000:
|
|
236
224
|
from itertools import islice
|
|
237
|
-
candidate_pids = set(islice(candidate_pids,
|
|
225
|
+
candidate_pids = set(islice(candidate_pids, 2000))
|
|
238
226
|
|
|
239
|
-
|
|
240
|
-
for code, inv_list, word in valid_sets:
|
|
241
|
-
weight = max(1.0, len(word) * 1.5 - 0.5)
|
|
242
|
-
hit_map[pid] = hit_map.get(pid, 0) + weight
|
|
243
|
-
hit_word_counts[pid] = hit_word_counts.get(pid, 0) + 1
|
|
227
|
+
final_results = []
|
|
244
228
|
|
|
245
|
-
if
|
|
246
|
-
|
|
229
|
+
if candidate_pids:
|
|
230
|
+
for pid in candidate_pids:
|
|
231
|
+
v = self.mem_vaults.get(pid)
|
|
232
|
+
if v and query in v["text"]:
|
|
233
|
+
final_results.append(v["text"])
|
|
234
|
+
if len(final_results) >= limit:
|
|
235
|
+
return final_results
|
|
247
236
|
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
hit_map = dict(top_500_items)
|
|
251
|
-
|
|
252
|
-
scored = []
|
|
253
|
-
now = time.time()
|
|
254
|
-
for pid, hit_score in hit_map.items():
|
|
237
|
+
count = 0
|
|
238
|
+
for pid in valid_sets[0]:
|
|
255
239
|
v = self.mem_vaults.get(pid)
|
|
256
|
-
if v:
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
scored.append({"text": v["text"], "score": final_score})
|
|
265
|
-
|
|
266
|
-
scored.sort(key=lambda x: x["score"], reverse=True)
|
|
267
|
-
|
|
268
|
-
if scored and scored[0]["score"] < 1.5:
|
|
269
|
-
return []
|
|
240
|
+
if v and v["text"] not in final_results: # 去重
|
|
241
|
+
if query in v["text"]: # 硬校验
|
|
242
|
+
final_results.append(v["text"])
|
|
243
|
+
if len(final_results) >= limit:
|
|
244
|
+
break
|
|
245
|
+
count += 1
|
|
246
|
+
if count > 2000:
|
|
247
|
+
break
|
|
270
248
|
|
|
271
|
-
return
|
|
249
|
+
return final_results
|
|
272
250
|
|
|
273
251
|
def close(self):
|
|
274
252
|
"""显式关闭数据库,解决 Windows 下文件被占用报错的问题"""
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|