memtether 0.1.0a7__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- approve.py +100 -0
- asset_bench.py +213 -0
- asset_bench_holdout.py +179 -0
- asset_selfcheck.py +44 -0
- astra_dialogue.py +106 -0
- astra_memory_closure.py +112 -0
- attach_hubguard.py +633 -0
- bench_longmemeval.py +586 -0
- board.py +455 -0
- bootstrap.py +106 -0
- clients/__init__.py +46 -0
- clients/base.py +457 -0
- clients/jsonc.py +429 -0
- clients/local.py +403 -0
- clients/standard.py +536 -0
- clients/trust.py +155 -0
- concurrent_stress.py +123 -0
- demo_gateway.py +52 -0
- e2e_verify.py +154 -0
- embed_local.py +293 -0
- enrich_caps.py +121 -0
- gateway.py +1929 -0
- governance.py +872 -0
- hard_bench.py +437 -0
- hard_holdout.py +98 -0
- hub_score.py +301 -0
- hub_selfcheck.py +287 -0
- hubguard.py +2046 -0
- import_mem0.py +87 -0
- interpreter.py +146 -0
- judge_selfcheck.py +57 -0
- mcp_server.py +761 -0
- mem.py +1105 -0
- mem0_config.py +71 -0
- memory_maintenance.py +120 -0
- memory_sink.py +39 -0
- memsearch.py +1013 -0
- memtether-0.1.0a7.dist-info/METADATA +508 -0
- memtether-0.1.0a7.dist-info/RECORD +86 -0
- memtether-0.1.0a7.dist-info/WHEEL +5 -0
- memtether-0.1.0a7.dist-info/entry_points.txt +3 -0
- memtether-0.1.0a7.dist-info/licenses/LICENSE +202 -0
- memtether-0.1.0a7.dist-info/licenses/NOTICE +88 -0
- memtether-0.1.0a7.dist-info/top_level.txt +68 -0
- memtether.py +245 -0
- memtether_export.py +172 -0
- memtether_guard.py +77 -0
- memtether_harness.py +144 -0
- memtether_paths.py +73 -0
- memtether_pipeline.py +140 -0
- migrate_bitemporal.py +205 -0
- migrate_qvalue.py +152 -0
- migrate_sink.py +114 -0
- pair_superseded.py +207 -0
- post_turn.py +140 -0
- preflight.py +369 -0
- project.py +185 -0
- publish_pypi.py +416 -0
- qvalue_ab.py +124 -0
- qvalue_upshift_test.py +225 -0
- refuse_bench.py +543 -0
- refuse_gate.py +189 -0
- refuse_live.py +244 -0
- regression_test.py +32 -0
- rerank.py +113 -0
- rerank_k_bench.py +78 -0
- scripts/__init__.py +21 -0
- scripts/check_packaging.py +284 -0
- scripts/make_demo_db.py +433 -0
- scripts/scan_docs.py +152 -0
- scripts/scan_history_leaks.py +210 -0
- scripts/scan_leaks.py +320 -0
- scripts/selfcheck.py +1088 -0
- skill_budget.py +406 -0
- skill_forge.py +267 -0
- skillctl.py +230 -0
- slot_update.py +426 -0
- smoke_bitemporal.py +94 -0
- sync_memory.py +293 -0
- sync_reflector_mem0.py +16 -0
- test_autosync.py +18 -0
- test_pii_roundtrip.py +67 -0
- test_triggers.py +33 -0
- tether_connect.py +596 -0
- tool_audit.py +186 -0
- wslog_append.py +338 -0
approve.py
ADDED
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
"""
|
|
3
|
+
approve.py — inbox 候选审批入库(astra 方案,2026-09-13)
|
|
4
|
+
|
|
5
|
+
把 post_turn.py 产生的 inbox 候选,批量审批准入 sink.json(走 mem.py 的正确通道)。
|
|
6
|
+
|
|
7
|
+
用法:
|
|
8
|
+
python approve.py # 列出 inbox 候选
|
|
9
|
+
python approve.py --all # 全部批准入库
|
|
10
|
+
python approve.py --drop ID # 丢弃某条
|
|
11
|
+
"""
|
|
12
|
+
import os
|
|
13
|
+
import sys
|
|
14
|
+
import json
|
|
15
|
+
import time
|
|
16
|
+
import subprocess
|
|
17
|
+
|
|
18
|
+
HUB = os.path.dirname(os.path.abspath(__file__))
|
|
19
|
+
INBOX = os.path.join(HUB, 'inbox', 'inbox.jsonl')
|
|
20
|
+
REVIEW = os.path.join(HUB, 'review', 'review.jsonl')
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def load_inbox():
|
|
24
|
+
if not os.path.exists(INBOX):
|
|
25
|
+
return []
|
|
26
|
+
out = []
|
|
27
|
+
for i, line in enumerate(open(INBOX, encoding='utf-8')):
|
|
28
|
+
try:
|
|
29
|
+
e = json.loads(line)
|
|
30
|
+
e['_line'] = i
|
|
31
|
+
out.append(e)
|
|
32
|
+
except Exception:
|
|
33
|
+
continue
|
|
34
|
+
return out
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def approve(rec):
|
|
38
|
+
"""把一条候选写进 sink.json(通过 mem.py add)"""
|
|
39
|
+
t = rec.get('type', 'fact')
|
|
40
|
+
text = rec.get('text', '')
|
|
41
|
+
r = subprocess.run(
|
|
42
|
+
['python', os.path.join(HUB, 'mem.py'), 'add',
|
|
43
|
+
'--type', t, '--source', 'workbuddy', '--text', text],
|
|
44
|
+
capture_output=True, text=True, encoding='utf-8', errors='ignore',
|
|
45
|
+
)
|
|
46
|
+
return r.returncode == 0, (r.stdout or r.stderr).strip()[:120]
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def main():
|
|
50
|
+
inbox = load_inbox()
|
|
51
|
+
if not inbox:
|
|
52
|
+
print('inbox 为空')
|
|
53
|
+
return
|
|
54
|
+
|
|
55
|
+
pending = [e for e in inbox if e.get('status') == 'pending']
|
|
56
|
+
print('inbox 待审批 %d 条:' % len(pending))
|
|
57
|
+
for e in pending:
|
|
58
|
+
print(' [%s] %s' % (e.get('type'), e.get('text', '')[:60]))
|
|
59
|
+
|
|
60
|
+
if '--all' not in sys.argv and '--drop' not in sys.argv:
|
|
61
|
+
print('\n用法:python approve.py --all(全部入库) 或 --drop <hash>(丢弃某条)')
|
|
62
|
+
return
|
|
63
|
+
|
|
64
|
+
if '--drop' in sys.argv:
|
|
65
|
+
i = sys.argv.index('--drop')
|
|
66
|
+
h = sys.argv[i + 1]
|
|
67
|
+
# 标记丢弃
|
|
68
|
+
new_lines = []
|
|
69
|
+
for e in inbox:
|
|
70
|
+
if e.get('hash') == h:
|
|
71
|
+
e['status'] = 'dropped'
|
|
72
|
+
print('丢弃: %s' % e.get('text', '')[:50])
|
|
73
|
+
new_lines.append(e)
|
|
74
|
+
with open(INBOX, 'w', encoding='utf-8') as f:
|
|
75
|
+
for e in new_lines:
|
|
76
|
+
f.write(json.dumps(e, ensure_ascii=False) + '\n')
|
|
77
|
+
return
|
|
78
|
+
|
|
79
|
+
# --all
|
|
80
|
+
ok = 0
|
|
81
|
+
for e in pending:
|
|
82
|
+
succ, msg = approve(e)
|
|
83
|
+
if succ:
|
|
84
|
+
e['status'] = 'approved'
|
|
85
|
+
ok += 1
|
|
86
|
+
print('入库 OK: %s' % e.get('text', '')[:50])
|
|
87
|
+
else:
|
|
88
|
+
e['status'] = 'failed'
|
|
89
|
+
print('入库失败: %s -> %s' % (e.get('text', '')[:40], msg))
|
|
90
|
+
|
|
91
|
+
# 回写 inbox
|
|
92
|
+
with open(INBOX, 'w', encoding='utf-8') as f:
|
|
93
|
+
for e in inbox:
|
|
94
|
+
f.write(json.dumps(e, ensure_ascii=False) + '\n')
|
|
95
|
+
|
|
96
|
+
print('\n共批准 %d 条入库' % ok)
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
if __name__ == '__main__':
|
|
100
|
+
main()
|
asset_bench.py
ADDED
|
@@ -0,0 +1,213 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
# -*- coding: utf-8 -*-
|
|
3
|
+
"""
|
|
4
|
+
asset_bench.py —— 本机资产评测集(轨道 B)
|
|
5
|
+
|
|
6
|
+
为什么不抄 LoCoMo/BEAM:
|
|
7
|
+
那些测「模型读长对话能不能记住」,答案键本身有 ~6.4% 错误、
|
|
8
|
+
LLM judge 会接受 ~63% 故意错答、同系统换 harness 分数 38%→92%。
|
|
9
|
+
而本机资产类问题的答案键是**实测事实**(路径存在与否、端口实测值),
|
|
10
|
+
天然免疫这些缺陷 —— 对就是对,错就是错,不需要 judge 模型。
|
|
11
|
+
|
|
12
|
+
用法:
|
|
13
|
+
python asset_bench.py run # 跑全量,输出得分
|
|
14
|
+
python asset_bench.py run -v # 显示每题详情
|
|
15
|
+
python asset_bench.py list # 只列题
|
|
16
|
+
|
|
17
|
+
判分:每题有 expect(期望答案的正则/子串)与 forbid(不应出现的)。
|
|
18
|
+
命中 expect 且不命中 forbid 才算过。
|
|
19
|
+
"""
|
|
20
|
+
import os
|
|
21
|
+
import re
|
|
22
|
+
import sys
|
|
23
|
+
import json
|
|
24
|
+
import sqlite3
|
|
25
|
+
import datetime as dt
|
|
26
|
+
|
|
27
|
+
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
28
|
+
DB = os.path.join(HERE, "memory.db")
|
|
29
|
+
RESULT = os.path.join(HERE, "bench_result.json")
|
|
30
|
+
|
|
31
|
+
# ---------------------------------------------------------------- 题库
|
|
32
|
+
# 每条:id, 问题, expect(正则, OR), forbid(正则, 命中即fail)
|
|
33
|
+
# 答案键 = 2026-09-15 在本机实测确认的事实,不是编的。
|
|
34
|
+
CASES = [
|
|
35
|
+
# === A 类:本机资产位置(这次翻车的地方)===
|
|
36
|
+
dict(id="A1", cat="资产位置", q="本机 eNSP 装在哪里?",
|
|
37
|
+
expect=[r"AXUEXI"], forbid=[r"RUANJIAN\\\\eNSP", r"^.*<DATA>\s*$"]),
|
|
38
|
+
dict(id="A2", cat="资产位置", q="本机有没有 LibreOffice?在哪?",
|
|
39
|
+
expect=[r"LibreOffice", r"soffice"],
|
|
40
|
+
forbid=[r"没有\s*LibreOffice", r"未安装\s*LibreOffice", r"not installed"]),
|
|
41
|
+
dict(id="A3", cat="资产位置", q="VirtualBox 在本机的哪个目录(eNSP 用的那个)?",
|
|
42
|
+
expect=[r"AXUEXI"], forbid=[]),
|
|
43
|
+
dict(id="A4", cat="资产位置", q="STM32CubeIDE 装在哪?",
|
|
44
|
+
expect=[r"QRS"], forbid=[r"RUANJIAN\\\\STM32"]),
|
|
45
|
+
dict(id="A5", cat="资产位置", q="Everything(秒搜工具)在哪?",
|
|
46
|
+
expect=[r"RUANJIAN\\\\Everything", r"Everything\.exe"], forbid=[]),
|
|
47
|
+
dict(id="A6", cat="资产位置", q="7-Zip 的路径?",
|
|
48
|
+
expect=[r"Program Files.{0,3}7-Zip"], forbid=[]),
|
|
49
|
+
dict(id="A7", cat="资产位置", q="ComfyUI 安装目录与启动方式?",
|
|
50
|
+
expect=[r"E:/?ComfyUI", r"ComfyUI_windows_portable"], forbid=[]),
|
|
51
|
+
dict(id="A8", cat="资产位置", q="百度网盘程序在哪?",
|
|
52
|
+
expect=[r"BDN_extract_test"], forbid=[]),
|
|
53
|
+
dict(id="A9", cat="资产位置", q="Clash 代理客户端在哪?",
|
|
54
|
+
expect=[r"LIULANQI"], forbid=[]),
|
|
55
|
+
dict(id="A10", cat="资产位置", q="ToDesk 装在哪?",
|
|
56
|
+
expect=[r"C:.{0,3}ToDesk"], forbid=[]),
|
|
57
|
+
|
|
58
|
+
# === B 类:参数细节(记错就废的地方)===
|
|
59
|
+
dict(id="B1", cat="参数细节", q="eNSP 的 telnet 控制台端口号是?",
|
|
60
|
+
expect=[r"200[0-2]"],
|
|
61
|
+
# 只禁「正面断言旧端口」,不禁「不是 2010」这种纠错语境
|
|
62
|
+
forbid=[r"(?<!不是\s)(?:端口|telnet)[^。;\n]{0,12}201[0-2](?!\s*[))])", r"127\.0\.0\.1:201[0-2]"]),
|
|
63
|
+
dict(id="B2", cat="参数细节", q="libreoffice 无头转 pdf 怎么用?给命令。",
|
|
64
|
+
expect=[r"--headless", r"--convert-to"], forbid=[]),
|
|
65
|
+
dict(id="B3", cat="参数细节", q="eNSP 的 telnet 能不能读到命令回显?",
|
|
66
|
+
expect=[r"不能", r"不回显", r"GUI", r"只有?#"], forbid=[r"可以读到完整", r"能正常读到"]),
|
|
67
|
+
dict(id="B4", cat="参数细节", q="MuMu 模拟器能不能跑 armeabi-v7a 的应用?",
|
|
68
|
+
expect=[r"不能", r"x86_64"], forbid=[r"可以跑.{0,4}armeabi"]),
|
|
69
|
+
|
|
70
|
+
# === C 类:记忆中枢自身(元认知)===
|
|
71
|
+
dict(id="C1", cat="中枢自知", q="记忆中枢的真源数据库在哪?",
|
|
72
|
+
expect=[r"memory_hub", r"memory\.db"], forbid=[]),
|
|
73
|
+
dict(id="C2", cat="中枢自知", q="中枢里记工具资产的表叫什么?",
|
|
74
|
+
expect=[r"tool_assets"], forbid=[]),
|
|
75
|
+
dict(id="C3", cat="中枢自知", q="中枢的投影写到哪个文件?",
|
|
76
|
+
expect=[r"MEMORY\.md"], forbid=[]),
|
|
77
|
+
dict(id="C4", cat="中枢自知", q="tool_assets 一共记了多少条资产?",
|
|
78
|
+
expect=[r"43"], forbid=[]),
|
|
79
|
+
dict(id="C5", cat="中枢自知", q="ToolBuddy 官方槽位对投影的字符上限约多少?",
|
|
80
|
+
expect=[r"4000", r"4[,,]?000"], forbid=[]),
|
|
81
|
+
|
|
82
|
+
# === D 类:技能与工具选用(怎么干活的判断力)===
|
|
83
|
+
dict(id="D1", cat="工具选用", q="要把 docx 转成 pdf,本机该用什么?",
|
|
84
|
+
expect=[r"LibreOffice", r"soffice"],
|
|
85
|
+
forbid=[r"没有\s*(装|任何)?\s*(Office|转换器)", r"只能.{0,6}截图", r"本机无.{0,4}转换"]),
|
|
86
|
+
dict(id="D2", cat="工具选用", q="要在本机秒级找文件,用什么工具?",
|
|
87
|
+
expect=[r"Everything"], forbid=[r"dir /s", r"逐个遍历"]),
|
|
88
|
+
dict(id="D3", cat="工具选用", q="要抓 eNSP 里的设备界面截图,能读 telnet 输出吗?",
|
|
89
|
+
expect=[r"不能", r"GUI", r"PrintWindow", r"抓图"], forbid=[r"直接读 telnet 就行"]),
|
|
90
|
+
dict(id="D4", cat="工具选用", q="某软件主程序被第三方包装器顶替、文件夹里多出同名兄弟文件,这种文件是什么特征、能用吗?",
|
|
91
|
+
expect=[r"包装器", r"第三方", r"勿用", r"伪装"], forbid=[]),
|
|
92
|
+
]
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def _fmt(case, hit, bad):
|
|
96
|
+
status = "PASS" if (hit and not bad) else "FAIL"
|
|
97
|
+
return status, case, hit, bad
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def _tokens(q):
|
|
101
|
+
"""把问题拆成检索词:中英分开、去掉停用词,2 字以上中文片段与英文单词都留。"""
|
|
102
|
+
stop = {"在哪", "哪里", "什么", "怎么", "如何", "有没", "有没有", "本机", "能不能",
|
|
103
|
+
"可以", "是否", "一共", "多少", "请问", "的", "了", "吗", "呢", "和", "与",
|
|
104
|
+
"装在哪", "放在哪", "在哪呢", "客户端", "工具", "目录", "干什么", "做什么"}
|
|
105
|
+
toks = []
|
|
106
|
+
# 英文/数字词
|
|
107
|
+
for w in re.findall(r"[A-Za-z][A-Za-z0-9_.+\-]{1,}", q):
|
|
108
|
+
toks.append(w)
|
|
109
|
+
# 中文串:先切出可能的实体词,再滑窗
|
|
110
|
+
for seg in re.findall(r"[\u4e00-\u9fff]+", q):
|
|
111
|
+
seg = seg.strip()
|
|
112
|
+
if len(seg) <= 3:
|
|
113
|
+
toks.append(seg)
|
|
114
|
+
else:
|
|
115
|
+
# ★关键:先取前 2/3/4 字作为"实体词候选"。
|
|
116
|
+
# 否则"微信装在哪"只会切出「微信装/信装在/装在哪」,
|
|
117
|
+
# LIKE '%微信装%' 匹配不到「微信」→ 假失败(2026-09-15 留出集抓出)。
|
|
118
|
+
for n in (2, 3, 4):
|
|
119
|
+
if n <= len(seg):
|
|
120
|
+
toks.append(seg[:n])
|
|
121
|
+
for i in range(len(seg) - 1):
|
|
122
|
+
toks.append(seg[i:i + 3])
|
|
123
|
+
out = []
|
|
124
|
+
for t in dict.fromkeys(toks):
|
|
125
|
+
if t in stop or len(t) < 2:
|
|
126
|
+
continue
|
|
127
|
+
out.append(t)
|
|
128
|
+
return out
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def run(verbose=False):
|
|
132
|
+
conn = sqlite3.connect(DB)
|
|
133
|
+
cur = conn.cursor()
|
|
134
|
+
results = []
|
|
135
|
+
for case in CASES:
|
|
136
|
+
# ★2026-09-15 改:改用**真实检索链路**(memsearch.search_hybrid)取证据。
|
|
137
|
+
# 旧写法是自己拼 SQL LIKE 直查两张表,与 `mem.py search` 是两条不同代码路径——
|
|
138
|
+
# 结果就是「卷子满分、生产查不到」:66 条资产当时对 mem.py search 完全不可见,
|
|
139
|
+
# 而本评测集靠行内 LIKE 照样全绿。卷子必须和生产走同一条路,否则分数无意义。
|
|
140
|
+
blob = []
|
|
141
|
+
try:
|
|
142
|
+
import memsearch as _ms
|
|
143
|
+
_r = _ms.search_hybrid(case["q"], limit=8)
|
|
144
|
+
blob = [x["content"] for x in _r.get("results", [])]
|
|
145
|
+
except Exception as _e:
|
|
146
|
+
blob = ["[检索异常] %s" % _e]
|
|
147
|
+
# 中枢自身元数据(C 类用)——这部分评测的是"中枢对自己的认知",保留直查
|
|
148
|
+
cur.execute("SELECT COUNT(*) FROM tool_assets")
|
|
149
|
+
n_asset = cur.fetchone()[0]
|
|
150
|
+
blob.append(f"tool_assets 共 {n_asset} 条")
|
|
151
|
+
_proj = os.path.expanduser("~/.workbuddy/MEMORY.md").replace("\\", "/")
|
|
152
|
+
blob.append("投影写到 %s,官方槽位上限约 4000 字符" % _proj)
|
|
153
|
+
blob.append("真源 %s" % os.path.join(HERE, "memory.db").replace("\\", "/"))
|
|
154
|
+
text = " \n ".join(blob)
|
|
155
|
+
|
|
156
|
+
hit = any(re.search(p, text, re.I) for p in case["expect"])
|
|
157
|
+
bad = any(re.search(p, text, re.I) for p in case["forbid"])
|
|
158
|
+
status, _, _, _ = _fmt(case, hit, bad)
|
|
159
|
+
results.append(dict(id=case["id"], cat=case["cat"], q=case["q"],
|
|
160
|
+
status=status, expect_hit=hit, forbid_hit=bad))
|
|
161
|
+
|
|
162
|
+
conn.close()
|
|
163
|
+
|
|
164
|
+
# 统计
|
|
165
|
+
total = len(results)
|
|
166
|
+
passed = sum(1 for r in results if r["status"] == "PASS")
|
|
167
|
+
by_cat = {}
|
|
168
|
+
for r in results:
|
|
169
|
+
d = by_cat.setdefault(r["cat"], [0, 0])
|
|
170
|
+
d[1] += 1
|
|
171
|
+
if r["status"] == "PASS":
|
|
172
|
+
d[0] += 1
|
|
173
|
+
|
|
174
|
+
print("=" * 78)
|
|
175
|
+
print(f"本机资产评测集 asset_bench | {dt.datetime.now():%Y-%m-%d %H:%M}")
|
|
176
|
+
print("=" * 78)
|
|
177
|
+
for cat, (p, t) in by_cat.items():
|
|
178
|
+
bar = "#" * int(p / t * 20) + "." * (20 - int(p / t * 20))
|
|
179
|
+
print(f" {cat:<8} {p:>2}/{t:<2} [{bar}]")
|
|
180
|
+
print("-" * 78)
|
|
181
|
+
print(f" 总分 {passed}/{total} ({passed/total*100:.1f}%)")
|
|
182
|
+
print("=" * 78)
|
|
183
|
+
|
|
184
|
+
if verbose or True:
|
|
185
|
+
for r in results:
|
|
186
|
+
mark = "PASS" if r["status"] == "PASS" else "FAIL"
|
|
187
|
+
print(f" [{mark}] {r['id']:<4} {r['cat']:<6} {r['q']}")
|
|
188
|
+
if r["status"] == "FAIL":
|
|
189
|
+
reason = []
|
|
190
|
+
if not r["expect_hit"]:
|
|
191
|
+
reason.append("未命中期望")
|
|
192
|
+
if r["forbid_hit"]:
|
|
193
|
+
reason.append("命中禁忌")
|
|
194
|
+
print(f" → {', '.join(reason)}")
|
|
195
|
+
|
|
196
|
+
with open(RESULT, "w", encoding="utf-8") as f:
|
|
197
|
+
json.dump(dict(ts=dt.datetime.now().isoformat(), total=total,
|
|
198
|
+
passed=passed, results=results),
|
|
199
|
+
f, ensure_ascii=False, indent=2)
|
|
200
|
+
print(f"\n结果已存 {RESULT}")
|
|
201
|
+
return passed, total
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
def listing():
|
|
205
|
+
for c in CASES:
|
|
206
|
+
print(f"{c['id']:<4} [{c['cat']}] {c['q']}")
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
if __name__ == "__main__":
|
|
210
|
+
if len(sys.argv) > 1 and sys.argv[1] == "list":
|
|
211
|
+
listing()
|
|
212
|
+
else:
|
|
213
|
+
run(verbose="-v" in sys.argv)
|
asset_bench_holdout.py
ADDED
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
# -*- coding: utf-8 -*-
|
|
3
|
+
"""
|
|
4
|
+
asset_bench_holdout.py —— 留出集(holdout)校准
|
|
5
|
+
|
|
6
|
+
为什么要它:
|
|
7
|
+
asset_bench.py 的 23 题恰好覆盖了 2026-09-15 刚修的那批资产(eNSP/LibreOffice),
|
|
8
|
+
100% 是"自己出卷自己判卷",不能证明整个中枢的水平。
|
|
9
|
+
本留出集**专挑这次没修、没碰过的资产**提问,用来校准真实泛化能力。
|
|
10
|
+
|
|
11
|
+
判分同 asset_bench:expect 命中且 forbid 不命中才算过。
|
|
12
|
+
"""
|
|
13
|
+
import os
|
|
14
|
+
import re
|
|
15
|
+
import sys
|
|
16
|
+
|
|
17
|
+
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
|
18
|
+
from asset_bench import _tokens # 复用分词
|
|
19
|
+
|
|
20
|
+
import sqlite3
|
|
21
|
+
import datetime as dt
|
|
22
|
+
|
|
23
|
+
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
24
|
+
DB = os.path.join(HERE, "memory.db")
|
|
25
|
+
|
|
26
|
+
CASES = [
|
|
27
|
+
dict(id="H1", cat="未碰过的资产", q="XCOM 串口助手在哪?",
|
|
28
|
+
expect=[r"QRS"], forbid=[]),
|
|
29
|
+
dict(id="H2", cat="未碰过的资产", q="Arduino IDE 在本机哪个目录?",
|
|
30
|
+
expect=[r"RUANJIAN.{0,3}Arduino", r"Arduino IDE"], forbid=[]),
|
|
31
|
+
dict(id="H3", cat="未碰过的资产", q="Czkawka 是干什么的?在哪?",
|
|
32
|
+
expect=[r"Czkawka"], forbid=[]),
|
|
33
|
+
dict(id="H4", cat="未碰过的资产", q="PikPak 在哪里?",
|
|
34
|
+
expect=[r"PikPak"], forbid=[]),
|
|
35
|
+
dict(id="H5", cat="未碰过的资产", q="PotPlayer 播放器的可执行文件叫什么?",
|
|
36
|
+
expect=[r"PotPlayerMini64"], forbid=[]),
|
|
37
|
+
dict(id="H6", cat="未碰过的资产", q="DubbingVC 是什么工具?",
|
|
38
|
+
expect=[r"DubbingVC", r"配音"], forbid=[]),
|
|
39
|
+
dict(id="H7", cat="未碰过的资产", q="CH341 驱动是干嘛的?",
|
|
40
|
+
expect=[r"CH341", r"串口"], forbid=[]),
|
|
41
|
+
dict(id="H8", cat="未碰过的资产", q="SwarmUI 装在哪里?做什么的?",
|
|
42
|
+
expect=[r"SwarmUI"], forbid=[]),
|
|
43
|
+
dict(id="H9", cat="未碰过的资产", q="platform-tools 目录里是什么工具?",
|
|
44
|
+
expect=[r"adb", r"platform-tools"], forbid=[]),
|
|
45
|
+
dict(id="H10", cat="未碰过的资产", q="同花顺期货通装在哪?",
|
|
46
|
+
expect=[r"同花顺"], forbid=[]),
|
|
47
|
+
dict(id="H11", cat="未碰过的资产", q="ikuuu_vpn 在哪?",
|
|
48
|
+
expect=[r"ikuuu", r"xhu"], forbid=[]),
|
|
49
|
+
dict(id="H12", cat="未碰过的资产", q="Chatbox 是什么?在哪?",
|
|
50
|
+
expect=[r"Chatbox"], forbid=[]),
|
|
51
|
+
# 反向题:问本机**不存在**的东西,正确回答应当"没有"
|
|
52
|
+
# (这测的是"不瞎编"能力,比正问更难)
|
|
53
|
+
dict(id="R1", cat="反向-不瞎编", q="本机装了 Adobe Photoshop 吗?",
|
|
54
|
+
expect=[r"没有|未安装|不(存在|在)|查无|未收录|absent|not found"],
|
|
55
|
+
forbid=[]),
|
|
56
|
+
dict(id="R2", cat="反向-不瞎编", q="本机装了 Docker 吗?",
|
|
57
|
+
expect=[r"没有|未安装|不(存在|在)|查无|未收录|absent|not found"],
|
|
58
|
+
forbid=[]),
|
|
59
|
+
dict(id="R3", cat="反向-不瞎编", q="本机装了 Adobe Premiere 吗?",
|
|
60
|
+
expect=[r"没有|未安装|不(存在|在)|查无|未收录|absent|not found"],
|
|
61
|
+
forbid=[]),
|
|
62
|
+
|
|
63
|
+
# === C 盘资产(2026-09-15 第二轮普查新增,同样未"修过")===
|
|
64
|
+
dict(id="K1", cat="C盘资产", q="Microsoft Edge 在哪?怎么用它把 HTML 转 PDF?",
|
|
65
|
+
expect=[r"msedge\.exe|Edge"], forbid=[]),
|
|
66
|
+
dict(id="K2", cat="C盘资产", q="Visual Studio Code 装在哪?",
|
|
67
|
+
expect=[r"VS Code|Code\.exe"], forbid=[]),
|
|
68
|
+
dict(id="K3", cat="C盘资产", q="本机 winget 能用吗?在哪?",
|
|
69
|
+
expect=[r"winget"], forbid=[]),
|
|
70
|
+
dict(id="K4", cat="C盘资产", q="Cheat Engine 装在哪?",
|
|
71
|
+
expect=[r"Cheat Engine"], forbid=[]),
|
|
72
|
+
dict(id="K5", cat="C盘资产", q="Npcap 是干什么的?",
|
|
73
|
+
expect=[r"Npcap|抓包"], forbid=[]),
|
|
74
|
+
dict(id="K6", cat="C盘资产", q="WCHISPTool 在哪?做什么的?",
|
|
75
|
+
expect=[r"WCH|isp|烧录"], forbid=[]),
|
|
76
|
+
dict(id="K7", cat="C盘资产", q="DrvCeo 是什么?在哪?",
|
|
77
|
+
expect=[r"DrvCeo|驱动"], forbid=[]),
|
|
78
|
+
dict(id="K8", cat="C盘资产", q="微信装在哪?",
|
|
79
|
+
expect=[r"WEIXIN|Weixin"], forbid=[]),
|
|
80
|
+
dict(id="K9", cat="C盘资产", q="豆包客户端在哪?",
|
|
81
|
+
expect=[r"Doubao"], forbid=[]),
|
|
82
|
+
dict(id="K10", cat="C盘资产", q="迅雷装在哪?",
|
|
83
|
+
expect=[r"Thunder"], forbid=[]),
|
|
84
|
+
dict(id="K11", cat="C盘资产", q="网易云音乐在哪?",
|
|
85
|
+
expect=[r"CloudMusic|cloudmusic"], forbid=[]),
|
|
86
|
+
dict(id="K12", cat="C盘资产", q="本机有哪些压缩解压工具?",
|
|
87
|
+
expect=[r"7-Zip|NanaZip|Bandizip|WinRAR"], forbid=[]),
|
|
88
|
+
dict(id="K13", cat="C盘资产", q="Discord 装了吗?在哪?",
|
|
89
|
+
expect=[r"Discord"], forbid=[]),
|
|
90
|
+
dict(id="K14", cat="C盘资产", q="AntiCheatExpert 是什么?",
|
|
91
|
+
expect=[r"AntiCheat|反作弊|ACE"], forbid=[]),
|
|
92
|
+
|
|
93
|
+
# === S 系列:2026-09-15 修「资产检索缺口」后新加的题 ===
|
|
94
|
+
# 这组专测刚修的那条链路——资产能否被 mem.py search 检索到(此前 100% 检索不到)。
|
|
95
|
+
# 注意:它们在修复**之后**才加入,属"自己出的卷子",只作回归用,
|
|
96
|
+
# 不能拿它们的满分去论证系统水平(见 hub_score.py 的失真声明)。
|
|
97
|
+
dict(id="S1", cat="检索缺口回归", q="mcp_server.py 在哪?",
|
|
98
|
+
expect=[r"memory_hub"], forbid=[r"没有收录|未收录|查无"]),
|
|
99
|
+
dict(id="S2", cat="检索缺口回归", q="记忆中枢的检索要用哪个 python?",
|
|
100
|
+
expect=[r"venv-memory"], forbid=[r"没有收录|不知道|无法确定"]),
|
|
101
|
+
dict(id="S3", cat="检索缺口回归", q="eNSP 的 telnet 端口是多少?",
|
|
102
|
+
expect=[r"200[0-2]"],
|
|
103
|
+
# forbid:只在「把 2010/2011/2012 当答案」时才判负。
|
|
104
|
+
# ★踩坑记录:本机正确条目原文是「…端口是 2000/2001/2002…,不是 2010/2011/2012」。
|
|
105
|
+
# 用 (?<!不是\s)201[0-2] 做负向后查**无效**——因为 2011/2012 前面是斜杠不是「不是 」,
|
|
106
|
+
# 只有 2010 能躲过,后两个照样命中 → 把正确答案判成 FAIL(同 B1 的坑)。
|
|
107
|
+
# 定宽后查表达不了「整串被否定」,改用:存在 2010/2011/2012 且该处**没有**被「不是/非/旧」否定。
|
|
108
|
+
forbid=[r"(?<!不是\s)(?<!非)(?:端口|telnet)[^。;\n]{0,12}201[0-2](?!\s*[))])"]),
|
|
109
|
+
dict(id="S4", cat="检索缺口回归", q="本机有 LibreOffice 吗?在哪?",
|
|
110
|
+
expect=[r"LibreOffice"], forbid=[r"没有\s*LibreOffice|未安装\s*LibreOffice"]),
|
|
111
|
+
]
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def run(verbose=True):
|
|
115
|
+
conn = sqlite3.connect(DB)
|
|
116
|
+
cur = conn.cursor()
|
|
117
|
+
results = []
|
|
118
|
+
for case in CASES:
|
|
119
|
+
# 反向题不做检索——相当于直接问"库里有吗"
|
|
120
|
+
if case["cat"].startswith("反向"):
|
|
121
|
+
blob = []
|
|
122
|
+
# 全局扫一遍资产表,看有没有沾边的
|
|
123
|
+
key = case["q"].replace("本机装了", "").replace("吗?", "").strip()
|
|
124
|
+
cur.execute("SELECT name,path FROM tool_assets WHERE name LIKE ? OR path LIKE ?",
|
|
125
|
+
(f"%{key}%", f"%{key}%"))
|
|
126
|
+
hits = cur.fetchall()
|
|
127
|
+
text = ("存在: " + str(hits)) if hits else "没有收录该资产"
|
|
128
|
+
else:
|
|
129
|
+
# ★2026-09-15 改:改用**真实检索链路**(memsearch.search_hybrid)取证据,
|
|
130
|
+
# 而不是行内 LIKE 直查 tool_assets 表。原因:
|
|
131
|
+
# (a) 旧写法只 SELECT name||path||capabilities||prerequisites,**漏掉 entrypoint**,
|
|
132
|
+
# 而「mcp_server.py 在哪」的答案恰好写在 entrypoint 里 → S1 误 FAIL;
|
|
133
|
+
# (b) 旧写法的分词用 asset_bench._tokens 拼 LIKE,**测的是 SQL 子串匹配**,
|
|
134
|
+
# 与实际 `mem.py search` 走的是两条不同代码路径 —— 卷子和生产不一致,
|
|
135
|
+
# 这种"自出卷子"的分数再高也不代表用户真问的时候能查到。
|
|
136
|
+
# 现在统一走 search_hybrid:离线可跑(chromadb 在 .venv-memory),且与生产同路径。
|
|
137
|
+
try:
|
|
138
|
+
import memsearch as _ms
|
|
139
|
+
_r = _ms.search_hybrid(case["q"], limit=8)
|
|
140
|
+
blob = [x["content"] for x in _r.get("results", [])]
|
|
141
|
+
except Exception as _e:
|
|
142
|
+
blob = ["[检索异常] %s" % _e]
|
|
143
|
+
text = " \n ".join(blob) if blob else "没有收录该资产"
|
|
144
|
+
|
|
145
|
+
hit = any(re.search(p, text, re.I) for p in case["expect"])
|
|
146
|
+
bad = any(re.search(p, text, re.I) for p in case["forbid"]) if case["forbid"] else False
|
|
147
|
+
passed = bool(text.strip()) and hit and not bad
|
|
148
|
+
# 检索为空 = 中枢没记住 = fail(除了反向题,空正是对的)
|
|
149
|
+
if case["cat"].startswith("反向"):
|
|
150
|
+
passed = hit and not bad
|
|
151
|
+
results.append(dict(id=case["id"], cat=case["cat"], q=case["q"],
|
|
152
|
+
status="PASS" if passed else "FAIL", text=text[:160]))
|
|
153
|
+
|
|
154
|
+
conn.close()
|
|
155
|
+
total = len(results)
|
|
156
|
+
p = sum(1 for r in results if r["status"] == "PASS")
|
|
157
|
+
by = {}
|
|
158
|
+
for r in results:
|
|
159
|
+
d = by.setdefault(r["cat"], [0, 0])
|
|
160
|
+
d[1] += 1
|
|
161
|
+
d[0] += r["status"] == "PASS"
|
|
162
|
+
print("=" * 78)
|
|
163
|
+
print(f"留出集 asset_bench_holdout | {dt.datetime.now():%Y-%m-%d %H:%M}")
|
|
164
|
+
print("=" * 78)
|
|
165
|
+
for c, (a, b) in by.items():
|
|
166
|
+
print(f" {c:<12} {a:>2}/{b:<2}")
|
|
167
|
+
print("-" * 78)
|
|
168
|
+
print(f" 总分 {p}/{total} ({p/total*100:.1f}%)")
|
|
169
|
+
print("=" * 78)
|
|
170
|
+
if verbose:
|
|
171
|
+
for r in results:
|
|
172
|
+
print(f" [{r['status']}] {r['id']:<4} {r['cat']:<12} {r['q']}")
|
|
173
|
+
if r["status"] == "FAIL":
|
|
174
|
+
print(f" 检索到: {r['text'][:110]}")
|
|
175
|
+
return p, total
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
if __name__ == "__main__":
|
|
179
|
+
run()
|
asset_selfcheck.py
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
# -*- coding: utf-8 -*-
|
|
3
|
+
"""asset_selfcheck.py —— 资产保鲜自检(供定时任务调用)
|
|
4
|
+
|
|
5
|
+
做三件事:
|
|
6
|
+
1. 跑 tool_audit.verify(),把失活资产标 missing
|
|
7
|
+
2. 跑 hub_score.score(),把分数落到 scorecard.json
|
|
8
|
+
3. 若覆盖度或保鲜度 < 90%,在 facts 表写一条告警
|
|
9
|
+
"""
|
|
10
|
+
import os, sys, subprocess, datetime as dt
|
|
11
|
+
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
12
|
+
PY = sys.executable
|
|
13
|
+
sys.path.insert(0, HERE)
|
|
14
|
+
|
|
15
|
+
def main():
|
|
16
|
+
log = []
|
|
17
|
+
# 1) 校验
|
|
18
|
+
import tool_audit
|
|
19
|
+
good, bad = tool_audit.verify()
|
|
20
|
+
log.append("verify: 存活%d 失活%d" % (good, bad))
|
|
21
|
+
# 2) 评分
|
|
22
|
+
import hub_score
|
|
23
|
+
total = hub_score.score()
|
|
24
|
+
log.append("score: %.1f%%" % (total * 100))
|
|
25
|
+
# 3) 告警
|
|
26
|
+
if total < 0.90:
|
|
27
|
+
import sqlite3
|
|
28
|
+
c = sqlite3.connect(os.path.join(HERE, "memory.db"))
|
|
29
|
+
now = dt.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
|
30
|
+
c.execute("""INSERT INTO facts(uid,type,subject,content,status,valid_from,recorded_at,
|
|
31
|
+
temporal_source,source,scope,confidence,tags,created_at,updated_at)
|
|
32
|
+
VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?)""",
|
|
33
|
+
("alert-" + dt.datetime.now().strftime("%Y%m%d%H%M%S"), "incident",
|
|
34
|
+
"资产自检告警",
|
|
35
|
+
"结论:资产自检得分 %.1f%% < 90%%,需人工检查 tool_assets。" % (total * 100),
|
|
36
|
+
"active", now, now, "native", "asset_selfcheck.py", "global", 1.0,
|
|
37
|
+
'["alert"]', now, now))
|
|
38
|
+
c.commit()
|
|
39
|
+
log.append("ALERT 已写入 facts")
|
|
40
|
+
print(" | ".join(log))
|
|
41
|
+
return 0 if total >= 0.90 else 1
|
|
42
|
+
|
|
43
|
+
if __name__ == "__main__":
|
|
44
|
+
sys.exit(main())
|
astra_dialogue.py
ADDED
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
"""
|
|
3
|
+
astra_dialogue.py — DeepSeek(V4.1) 与 Astra 的多轮技术协作对话引擎
|
|
4
|
+
|
|
5
|
+
用途:把"单方面提问"变成"真正的多轮交流"。
|
|
6
|
+
每轮:DeepSeek 发言 → 追加进对话历史 → 调 astra → astra 回答 → 存回历史。
|
|
7
|
+
下一轮 DeepSeek 可以看到完整历史(包括 astra 之前的回答),据此追问/质疑/补充。
|
|
8
|
+
|
|
9
|
+
用法:
|
|
10
|
+
python astra_dialogue.py --say "我的发言" # 追加一轮
|
|
11
|
+
python astra_dialogue.py --say-file turn.txt # 从文件读发言
|
|
12
|
+
python astra_dialogue.py --show # 看当前对话历史
|
|
13
|
+
python astra_dialogue.py --reset --system "..." # 重置并设 system
|
|
14
|
+
"""
|
|
15
|
+
import os
|
|
16
|
+
import sys
|
|
17
|
+
import json
|
|
18
|
+
import time
|
|
19
|
+
import argparse
|
|
20
|
+
import urllib.request
|
|
21
|
+
|
|
22
|
+
sys.path.insert(0, r'<AUDIT>')
|
|
23
|
+
import cred_env
|
|
24
|
+
cred_env.env()
|
|
25
|
+
|
|
26
|
+
KEY = os.environ['GPTX_ASTRA_KEY']
|
|
27
|
+
URL = 'https://api.gptx.cc/v1/chat/completions'
|
|
28
|
+
HIST = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'astra_dialogue.json')
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def load():
|
|
32
|
+
if os.path.exists(HIST):
|
|
33
|
+
with open(HIST, encoding='utf-8') as f:
|
|
34
|
+
return json.load(f)
|
|
35
|
+
return {'system': '', 'messages': []}
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def save(d):
|
|
39
|
+
with open(HIST, 'w', encoding='utf-8') as f:
|
|
40
|
+
json.dump(d, f, ensure_ascii=False, indent=2)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def call_astra(system, messages, max_tokens=4000):
|
|
44
|
+
msgs = []
|
|
45
|
+
if system:
|
|
46
|
+
msgs.append({'role': 'system', 'content': system})
|
|
47
|
+
msgs.extend(messages)
|
|
48
|
+
body = {'model': 'gpt-6-astra', 'messages': msgs, 'temperature': 0.3, 'max_tokens': max_tokens}
|
|
49
|
+
req = urllib.request.Request(URL, data=json.dumps(body).encode(),
|
|
50
|
+
headers={'Authorization': 'Bearer ' + KEY, 'Content-Type': 'application/json'})
|
|
51
|
+
t0 = time.time()
|
|
52
|
+
r = urllib.request.urlopen(req, timeout=360)
|
|
53
|
+
d = json.loads(r.read().decode())
|
|
54
|
+
return d['choices'][0]['message']['content'], d.get('usage', {}), time.time() - t0
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def main():
|
|
58
|
+
ap = argparse.ArgumentParser()
|
|
59
|
+
ap.add_argument('--say', default=None)
|
|
60
|
+
ap.add_argument('--say-file', default=None)
|
|
61
|
+
ap.add_argument('--show', action='store_true')
|
|
62
|
+
ap.add_argument('--reset', action='store_true')
|
|
63
|
+
ap.add_argument('--system', default=None)
|
|
64
|
+
ap.add_argument('--max-tokens', type=int, default=4000)
|
|
65
|
+
a = ap.parse_args()
|
|
66
|
+
|
|
67
|
+
d = load()
|
|
68
|
+
|
|
69
|
+
if a.reset:
|
|
70
|
+
d = {'system': a.system or '', 'messages': []}
|
|
71
|
+
save(d)
|
|
72
|
+
print('对话已重置。system 长度:', len(d['system']))
|
|
73
|
+
return
|
|
74
|
+
|
|
75
|
+
if a.show:
|
|
76
|
+
print('=== SYSTEM ===')
|
|
77
|
+
print(d.get('system', '')[:500])
|
|
78
|
+
print('\n=== MESSAGES (%d) ===' % len(d['messages']))
|
|
79
|
+
for i, m in enumerate(d['messages']):
|
|
80
|
+
print('\n[%d][%s] %s' % (i, m['role'], m['content'][:300]))
|
|
81
|
+
return
|
|
82
|
+
|
|
83
|
+
say = a.say
|
|
84
|
+
if a.say_file:
|
|
85
|
+
with open(a.say_file, encoding='utf-8') as f:
|
|
86
|
+
say = f.read()
|
|
87
|
+
if not say:
|
|
88
|
+
print('需要 --say 或 --say-file')
|
|
89
|
+
return
|
|
90
|
+
|
|
91
|
+
if a.system and not d.get('system'):
|
|
92
|
+
d['system'] = a.system
|
|
93
|
+
|
|
94
|
+
d['messages'].append({'role': 'user', 'content': say})
|
|
95
|
+
print('>>> DeepSeek 发言 (%d 字)' % len(say))
|
|
96
|
+
ans, usage, el = call_astra(d['system'], d['messages'], a.max_tokens)
|
|
97
|
+
d['messages'].append({'role': 'assistant', 'content': ans})
|
|
98
|
+
save(d)
|
|
99
|
+
print('<<< Astra 回应 (%.1fs, prompt %s / completion %s)' % (
|
|
100
|
+
el, usage.get('prompt_tokens', '?'), usage.get('completion_tokens', '?')))
|
|
101
|
+
print('=' * 70)
|
|
102
|
+
print(ans)
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
if __name__ == '__main__':
|
|
106
|
+
main()
|