mindforge-memory 6.4.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- MindForge.py +20 -0
- mindforge/__init__.py +111 -0
- mindforge/adapters/__init__.py +18 -0
- mindforge/adapters/claude_adapter.py +84 -0
- mindforge/adapters/crewai_memory.py +76 -0
- mindforge/adapters/generic_api.py +183 -0
- mindforge/adapters/langgraph_store.py +463 -0
- mindforge/adapters/openclaw_adapter.py +91 -0
- mindforge/api/__init__.py +1 -0
- mindforge/api/server.py +1567 -0
- mindforge/cli/__init__.py +1 -0
- mindforge/cli/main.py +13607 -0
- mindforge/core/__init__.py +37 -0
- mindforge/core/embedding.py +847 -0
- mindforge/core/encryption.py +597 -0
- mindforge/core/indexer.py +363 -0
- mindforge/core/mindforge.py +6298 -0
- mindforge/core/paths.py +43 -0
- mindforge/core/query.py +501 -0
- mindforge/core/storage.py +18656 -0
- mindforge/core/types.py +314 -0
- mindforge/core/version.py +56 -0
- mindforge/daemon/__init__.py +0 -0
- mindforge/daemon/server.py +277 -0
- mindforge/mcp/__init__.py +4 -0
- mindforge/mcp/server.py +2559 -0
- mindforge/modules/__init__.py +138 -0
- mindforge/modules/categorizer.py +151 -0
- mindforge/modules/conflict_detector.py +356 -0
- mindforge/modules/connectors.py +331 -0
- mindforge/modules/context_window.py +337 -0
- mindforge/modules/event_bus.py +572 -0
- mindforge/modules/evolution.py +210 -0
- mindforge/modules/experience.py +402 -0
- mindforge/modules/federated.py +722 -0
- mindforge/modules/federated_acl.py +393 -0
- mindforge/modules/follow_up.py +315 -0
- mindforge/modules/hybrid_search.py +565 -0
- mindforge/modules/importer.py +374 -0
- mindforge/modules/integrator.py +196 -0
- mindforge/modules/intent_router.py +359 -0
- mindforge/modules/knowledge_graph.py +395 -0
- mindforge/modules/memory_compressor.py +592 -0
- mindforge/modules/memory_decay.py +474 -0
- mindforge/modules/memory_passport.py +250 -0
- mindforge/modules/multi_agent.py +534 -0
- mindforge/modules/multimodal.py +304 -0
- mindforge/modules/outcome_learning.py +186 -0
- mindforge/modules/personality.py +275 -0
- mindforge/modules/principled_forgetting.py +258 -0
- mindforge/modules/privacy.py +736 -0
- mindforge/modules/recall.py +263 -0
- mindforge/modules/session_focus.py +305 -0
- mindforge/modules/share_conflict.py +521 -0
- mindforge/modules/skill_extractor.py +340 -0
- mindforge/modules/skill_pack.py +205 -0
- mindforge/modules/timeline_viz.py +206 -0
- mindforge/modules/v6.py +931 -0
- mindforge_memory-6.4.1.dist-info/METADATA +281 -0
- mindforge_memory-6.4.1.dist-info/RECORD +64 -0
- mindforge_memory-6.4.1.dist-info/WHEEL +5 -0
- mindforge_memory-6.4.1.dist-info/entry_points.txt +3 -0
- mindforge_memory-6.4.1.dist-info/licenses/LICENSE +21 -0
- mindforge_memory-6.4.1.dist-info/top_level.txt +2 -0
|
@@ -0,0 +1,722 @@
|
|
|
1
|
+
"""
|
|
2
|
+
MindForge v5.0 联邦记忆网络
|
|
3
|
+
多 Agent 间安全共享记忆,端侧联邦学习
|
|
4
|
+
|
|
5
|
+
v5.6.5 安全加固:
|
|
6
|
+
- P0 #1:跨节点签名由对称 HMAC 升级为真正的非对称 **Ed25519** 数字签名。
|
|
7
|
+
发送方用自己的私钥签名,接收方只用发送方注册的公钥验签,公钥即便公开也
|
|
8
|
+
无法伪造签名,消除“一方泄露共享密钥即可冒充对方”的根本问题。为平滑迁移,
|
|
9
|
+
仍保留 HMAC 作为仅配置了 shared_secret 的旧节点的回退方案。
|
|
10
|
+
- P0 #2:receive_memory 增加重放攻击防护。签名消息必须携带发送方时间戳与
|
|
11
|
+
随机 nonce(且均被签名覆盖):超出时钟偏差窗口的旧消息拒收,同一节点同一
|
|
12
|
+
nonce 只接受一次(带容量上限与过期清理的去重表),截获重放无法重复入队。
|
|
13
|
+
- P1 #10:shared_memories 中已过 expires_at 的共享记录由
|
|
14
|
+
purge_expired_shared_memories() 定期清理(节流触发,线程安全),不再无限膨胀。
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
import json
|
|
18
|
+
import hashlib
|
|
19
|
+
import hmac
|
|
20
|
+
import base64
|
|
21
|
+
import binascii
|
|
22
|
+
import logging
|
|
23
|
+
import threading
|
|
24
|
+
import time
|
|
25
|
+
import uuid
|
|
26
|
+
from dataclasses import dataclass, field
|
|
27
|
+
from typing import Dict, List, Optional, Any, Tuple
|
|
28
|
+
from enum import Enum
|
|
29
|
+
|
|
30
|
+
logger = logging.getLogger(__name__)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def generate_keypair() -> Tuple[str, str]:
|
|
34
|
+
"""生成一对 Ed25519 密钥(v5.6.5 P0 #1)
|
|
35
|
+
|
|
36
|
+
Returns:
|
|
37
|
+
(private_key_b64, public_key_b64),均为 32 字节原始密钥的
|
|
38
|
+
URL 安全 base64 编码(无填充)。私钥是签名种子,必须保密;
|
|
39
|
+
公钥可安全地分发给所有联邦对端用于验签。
|
|
40
|
+
"""
|
|
41
|
+
from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey
|
|
42
|
+
priv = Ed25519PrivateKey.generate()
|
|
43
|
+
seed = priv.private_bytes_raw()
|
|
44
|
+
pub = priv.public_key().public_bytes_raw()
|
|
45
|
+
return (_b64e(seed), _b64e(pub))
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _b64e(raw: bytes) -> str:
|
|
49
|
+
return base64.urlsafe_b64encode(raw).rstrip(b"=").decode("ascii")
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _b64d(text: str) -> bytes:
|
|
53
|
+
pad = "=" * (-len(text) % 4)
|
|
54
|
+
return base64.urlsafe_b64decode(text + pad)
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
class PeerStatus(Enum):
|
|
58
|
+
"""节点状态"""
|
|
59
|
+
ONLINE = "online"
|
|
60
|
+
OFFLINE = "offline"
|
|
61
|
+
TRUSTED = "trusted"
|
|
62
|
+
UNTRUSTED = "untrusted"
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
@dataclass
|
|
66
|
+
class FederatedPeer:
|
|
67
|
+
"""联邦节点"""
|
|
68
|
+
peer_id: str
|
|
69
|
+
name: str
|
|
70
|
+
status: PeerStatus = PeerStatus.OFFLINE
|
|
71
|
+
trust_level: float = 0.5
|
|
72
|
+
public_key: str = "" # Ed25519 验签公钥(b64),公开即可,配置后优先使用非对称验签
|
|
73
|
+
shared_secret: str = "" # 旧版 HMAC 共享密钥(迁移回退用),配置 Ed25519 后不再需要
|
|
74
|
+
last_seen: float = 0.0
|
|
75
|
+
shared_categories: List[str] = field(default_factory=list)
|
|
76
|
+
metadata: Dict[str, Any] = field(default_factory=dict)
|
|
77
|
+
|
|
78
|
+
def to_dict(self) -> dict:
|
|
79
|
+
return {
|
|
80
|
+
"peer_id": self.peer_id,
|
|
81
|
+
"name": self.name,
|
|
82
|
+
"status": self.status.value,
|
|
83
|
+
"trust_level": self.trust_level,
|
|
84
|
+
# 公钥不是秘密,可随节点信息导出;shared_secret 永不导出。
|
|
85
|
+
"public_key": self.public_key,
|
|
86
|
+
"last_seen": self.last_seen,
|
|
87
|
+
"shared_categories": self.shared_categories,
|
|
88
|
+
"metadata": self.metadata,
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
@dataclass
|
|
93
|
+
class SharedMemory:
|
|
94
|
+
"""共享记忆"""
|
|
95
|
+
memory_id: str
|
|
96
|
+
original_peer: str
|
|
97
|
+
shared_with: List[str] = field(default_factory=list)
|
|
98
|
+
access_count: Dict[str, int] = field(default_factory=dict)
|
|
99
|
+
shared_at: float = 0.0
|
|
100
|
+
expires_at: Optional[float] = None
|
|
101
|
+
access_policy: str = "read_only"
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
class FederatedMemory:
|
|
105
|
+
"""联邦记忆管理器"""
|
|
106
|
+
|
|
107
|
+
def __init__(self, storage=None, local_peer_id: str = "",
|
|
108
|
+
acl=None, conflict_resolver=None,
|
|
109
|
+
config: Optional[Dict] = None):
|
|
110
|
+
self.storage = storage
|
|
111
|
+
self.local_peer_id = local_peer_id or str(uuid.uuid4())
|
|
112
|
+
self.peers: Dict[str, FederatedPeer] = {}
|
|
113
|
+
self.shared_memories: Dict[str, SharedMemory] = {}
|
|
114
|
+
self._incoming_queue: List[Dict] = []
|
|
115
|
+
self._outgoing_queue: List[Dict] = []
|
|
116
|
+
# v5.4.2 修复:队列大小上限,防止恶意 peer 发送海量消息耗尽内存
|
|
117
|
+
self._MAX_QUEUE_SIZE = 10000
|
|
118
|
+
# v5.4.2 新增:细粒度 ACL(modules/federated_acl.py)与
|
|
119
|
+
# 共享记忆冲突解析器(modules/share_conflict.py),均可选注入
|
|
120
|
+
self.acl = acl
|
|
121
|
+
self.conflict_resolver = conflict_resolver
|
|
122
|
+
# P1-004: 安全加固 - 无签名节点默认拒绝(fail-closed)
|
|
123
|
+
config = config or {}
|
|
124
|
+
self.allow_unsigned_peers = config.get("allow_unsigned_peers", False)
|
|
125
|
+
|
|
126
|
+
# v5.6.5 P0 #2:重放防护参数
|
|
127
|
+
# 允许的收发双方时钟偏差(秒)。消息 _ts 与本地时间相差超过该值即视为
|
|
128
|
+
# 过期/伪造而拒绝。
|
|
129
|
+
self.allowed_skew_seconds = float(config.get("federated_allowed_skew_seconds", 300))
|
|
130
|
+
# 每个节点保留的已用 nonce 上限,超出直接拒收新消息(防止内存被打满)。
|
|
131
|
+
self._replay_max_nonces = int(config.get("federated_replay_max_nonces", 100000))
|
|
132
|
+
# peer_id -> {nonce: 该消息时间戳},仅保留偏差窗口内的 nonce
|
|
133
|
+
self._seen_nonces: Dict[str, Dict[str, float]] = {}
|
|
134
|
+
self._fed_lock = threading.Lock()
|
|
135
|
+
|
|
136
|
+
# v5.6.5 P0 #1:本地 Ed25519 签名私钥。优先取 config 注入(持久化身份),
|
|
137
|
+
# 否则生成一次性密钥对(重启后身份变化,仅适合临时进程)。
|
|
138
|
+
seed_b64 = config.get("local_private_key", "")
|
|
139
|
+
pub_b64 = config.get("local_public_key", "")
|
|
140
|
+
if seed_b64:
|
|
141
|
+
self._local_priv_b64 = seed_b64
|
|
142
|
+
if not pub_b64:
|
|
143
|
+
try:
|
|
144
|
+
from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey
|
|
145
|
+
priv = Ed25519PrivateKey.from_private_bytes(_b64d(seed_b64))
|
|
146
|
+
pub_b64 = _b64e(priv.public_key().public_bytes_raw())
|
|
147
|
+
except (ValueError, binascii.Error, TypeError) as e:
|
|
148
|
+
logger.error("本地联邦私钥无效,已改为临时密钥: %s", e)
|
|
149
|
+
seed_b64, pub_b64 = generate_keypair()
|
|
150
|
+
self._local_priv_b64 = seed_b64
|
|
151
|
+
else:
|
|
152
|
+
seed_b64, pub_b64 = generate_keypair()
|
|
153
|
+
self._local_priv_b64 = seed_b64
|
|
154
|
+
self._local_pub_b64 = pub_b64
|
|
155
|
+
|
|
156
|
+
# v5.6.5 P1 #10:共享记录过期清理的节流时间戳
|
|
157
|
+
self._last_shared_purge = 0.0
|
|
158
|
+
self._shared_purge_interval = float(config.get("shared_purge_interval", 300))
|
|
159
|
+
|
|
160
|
+
@property
|
|
161
|
+
def local_public_key(self) -> str:
|
|
162
|
+
"""本节点对外公布的 Ed25519 验签公钥(b64)"""
|
|
163
|
+
return self._local_pub_b64
|
|
164
|
+
|
|
165
|
+
def register_peer(self, peer_id: str, name: str,
|
|
166
|
+
trust_level: float = 0.5,
|
|
167
|
+
shared_categories: Optional[List[str]] = None,
|
|
168
|
+
shared_secret: str = "",
|
|
169
|
+
public_key: str = "") -> FederatedPeer:
|
|
170
|
+
"""注册联邦节点
|
|
171
|
+
|
|
172
|
+
v5.6.5(P0 #1):推荐传入对端的 Ed25519 ``public_key``(验签公钥)。
|
|
173
|
+
配置公钥后,与该节点的消息签名/验签走非对称 Ed25519;仅配置
|
|
174
|
+
``shared_secret`` 的旧节点继续走 HMAC(迁移回退)。两者都未配置时,
|
|
175
|
+
节点仍会被注册(向后兼容),但签名相关操作 fail-closed,并记录告警。
|
|
176
|
+
"""
|
|
177
|
+
peer = FederatedPeer(
|
|
178
|
+
peer_id=peer_id,
|
|
179
|
+
name=name,
|
|
180
|
+
status=PeerStatus.OFFLINE,
|
|
181
|
+
trust_level=trust_level,
|
|
182
|
+
shared_categories=shared_categories or [],
|
|
183
|
+
public_key=(public_key or "").strip(),
|
|
184
|
+
shared_secret=shared_secret or "",
|
|
185
|
+
last_seen=time.time(),
|
|
186
|
+
)
|
|
187
|
+
if not peer.public_key and not peer.shared_secret:
|
|
188
|
+
logger.warning(
|
|
189
|
+
"联邦节点 %s 未配置 public_key/shared_secret,签名/验签将 fail-closed "
|
|
190
|
+
"拒绝(推荐 register_peer(public_key=<对端 Ed25519 公钥>))",
|
|
191
|
+
peer_id,
|
|
192
|
+
)
|
|
193
|
+
elif peer.shared_secret and not peer.public_key:
|
|
194
|
+
logger.info(
|
|
195
|
+
"联邦节点 %s 仍使用 HMAC shared_secret(对称)。建议尽快交换 "
|
|
196
|
+
"Ed25519 公钥并改用非对称签名,避免单点密钥泄露即被冒充。",
|
|
197
|
+
peer_id,
|
|
198
|
+
)
|
|
199
|
+
self.peers[peer_id] = peer
|
|
200
|
+
with self._fed_lock:
|
|
201
|
+
self._seen_nonces.setdefault(peer_id, {})
|
|
202
|
+
return peer
|
|
203
|
+
|
|
204
|
+
def remove_peer(self, peer_id: str) -> bool:
|
|
205
|
+
"""移除节点"""
|
|
206
|
+
if peer_id in self.peers:
|
|
207
|
+
del self.peers[peer_id]
|
|
208
|
+
with self._fed_lock:
|
|
209
|
+
self._seen_nonces.pop(peer_id, None)
|
|
210
|
+
return True
|
|
211
|
+
return False
|
|
212
|
+
|
|
213
|
+
def update_trust_level(self, peer_id: str, new_level: float) -> bool:
|
|
214
|
+
"""更新信任级别"""
|
|
215
|
+
if peer_id in self.peers:
|
|
216
|
+
self.peers[peer_id].trust_level = max(0.0, min(1.0, new_level))
|
|
217
|
+
self.peers[peer_id].last_seen = time.time()
|
|
218
|
+
return True
|
|
219
|
+
return False
|
|
220
|
+
|
|
221
|
+
def share_memory(self, memory_id: str,
|
|
222
|
+
peer_ids: List[str],
|
|
223
|
+
access_policy: str = "read_only",
|
|
224
|
+
expires_hours: Optional[float] = None) -> Optional[SharedMemory]:
|
|
225
|
+
"""共享记忆给其他节点
|
|
226
|
+
|
|
227
|
+
v5.4.2 修复:此前信任过滤循环为空操作(dead code),未注册或
|
|
228
|
+
低信任度(<0.3)节点仍会进入 shared_with。现实际过滤,并可叠加
|
|
229
|
+
细粒度 ACL(注入 self.acl 时按 read 操作逐节点评估)。
|
|
230
|
+
"""
|
|
231
|
+
self._maybe_purge_shared()
|
|
232
|
+
|
|
233
|
+
if not self._verify_memory_exists(memory_id):
|
|
234
|
+
return None
|
|
235
|
+
|
|
236
|
+
# v5.4.2 修复:真实过滤未注册 / 低信任节点
|
|
237
|
+
eligible: List[str] = []
|
|
238
|
+
skipped: Dict[str, str] = {}
|
|
239
|
+
for pid in peer_ids:
|
|
240
|
+
if pid not in self.peers:
|
|
241
|
+
skipped[pid] = "未注册的节点"
|
|
242
|
+
continue
|
|
243
|
+
if self.peers[pid].trust_level < 0.3:
|
|
244
|
+
skipped[pid] = f"信任度不足({self.peers[pid].trust_level:.2f} < 0.3)"
|
|
245
|
+
continue
|
|
246
|
+
eligible.append(pid)
|
|
247
|
+
|
|
248
|
+
# v5.4.2 新增:细粒度 ACL 过滤(按 read 操作评估)
|
|
249
|
+
if self.acl is not None and eligible:
|
|
250
|
+
memory_category = None
|
|
251
|
+
memory_tags: List[str] = []
|
|
252
|
+
if self.storage:
|
|
253
|
+
try:
|
|
254
|
+
entry = self.storage.get_memory(memory_id)
|
|
255
|
+
if entry is not None:
|
|
256
|
+
memory_category = entry.category
|
|
257
|
+
memory_tags = list(entry.tags or [])
|
|
258
|
+
except Exception:
|
|
259
|
+
pass
|
|
260
|
+
trust_map = {pid: self.peers[pid].trust_level for pid in eligible}
|
|
261
|
+
verdict = self.acl.filter_peers(
|
|
262
|
+
memory_id=memory_id, peer_ids=eligible, operation="read",
|
|
263
|
+
trust_map=trust_map, memory_category=memory_category,
|
|
264
|
+
memory_tags=memory_tags)
|
|
265
|
+
for pid, reason in verdict["denied"].items():
|
|
266
|
+
skipped[pid] = f"ACL 拒绝: {reason}"
|
|
267
|
+
eligible = verdict["allowed"]
|
|
268
|
+
|
|
269
|
+
self.last_share_skipped = skipped
|
|
270
|
+
|
|
271
|
+
if not eligible:
|
|
272
|
+
return None
|
|
273
|
+
|
|
274
|
+
shared = SharedMemory(
|
|
275
|
+
memory_id=memory_id,
|
|
276
|
+
original_peer=self.local_peer_id,
|
|
277
|
+
shared_with=eligible,
|
|
278
|
+
shared_at=time.time(),
|
|
279
|
+
expires_at=time.time() + expires_hours * 3600 if expires_hours else None,
|
|
280
|
+
access_policy=access_policy,
|
|
281
|
+
)
|
|
282
|
+
|
|
283
|
+
# v5.6.6 P3:与 purge/get/revoke 在同一把锁内变更共享字典
|
|
284
|
+
with self._fed_lock:
|
|
285
|
+
self.shared_memories[memory_id] = shared
|
|
286
|
+
|
|
287
|
+
for pid in eligible:
|
|
288
|
+
if len(self._outgoing_queue) < self._MAX_QUEUE_SIZE:
|
|
289
|
+
self._outgoing_queue.append({
|
|
290
|
+
"type": "memory_share",
|
|
291
|
+
"from": self.local_peer_id,
|
|
292
|
+
"to": pid,
|
|
293
|
+
"memory_id": memory_id,
|
|
294
|
+
"access_policy": access_policy,
|
|
295
|
+
"timestamp": time.time(),
|
|
296
|
+
})
|
|
297
|
+
|
|
298
|
+
return shared
|
|
299
|
+
|
|
300
|
+
def revoke_share(self, memory_id: str, peer_ids: Optional[List[str]] = None) -> bool:
|
|
301
|
+
"""撤销共享"""
|
|
302
|
+
# v5.6.6 P3:检查与删除需在同一临界区原子完成,避免与 purge 竞态
|
|
303
|
+
with self._fed_lock:
|
|
304
|
+
if memory_id not in self.shared_memories:
|
|
305
|
+
return False
|
|
306
|
+
|
|
307
|
+
shared = self.shared_memories[memory_id]
|
|
308
|
+
|
|
309
|
+
if peer_ids:
|
|
310
|
+
shared.shared_with = [pid for pid in shared.shared_with if pid not in peer_ids]
|
|
311
|
+
if not shared.shared_with:
|
|
312
|
+
del self.shared_memories[memory_id]
|
|
313
|
+
else:
|
|
314
|
+
del self.shared_memories[memory_id]
|
|
315
|
+
|
|
316
|
+
return True
|
|
317
|
+
|
|
318
|
+
def sign_payload(self, data: Dict[str, Any], peer_id: str
|
|
319
|
+
) -> Tuple[Dict[str, Any], str]:
|
|
320
|
+
"""构造带防重放信封的消息并签名(v5.6.5 P0 #2)
|
|
321
|
+
|
|
322
|
+
在业务数据中注入 ``_ts``(发送时间戳)与 ``_mid``(随机 nonce),
|
|
323
|
+
两者随后被签名覆盖,攻击者无法在不破坏签名的前提下篡改或重放。
|
|
324
|
+
|
|
325
|
+
Returns:
|
|
326
|
+
(envelope, signature),调用方应把两者原样传给对端 receive_memory。
|
|
327
|
+
"""
|
|
328
|
+
envelope = dict(data)
|
|
329
|
+
envelope["_ts"] = time.time()
|
|
330
|
+
envelope["_mid"] = uuid.uuid4().hex
|
|
331
|
+
return envelope, self._compute_signature(envelope, peer_id)
|
|
332
|
+
|
|
333
|
+
def receive_memory(self, from_peer: str,
|
|
334
|
+
memory_data: Dict,
|
|
335
|
+
signature: str = "") -> bool:
|
|
336
|
+
"""接收来自其他节点的记忆
|
|
337
|
+
|
|
338
|
+
v5.6.5 P0 #2:对**已签名**消息强制重放防护——必须含被签名覆盖的
|
|
339
|
+
``_ts``/``_mid``,时间戳超窗或 nonce 重复一律拒绝。未签名(显式
|
|
340
|
+
allow_unsigned_peers)的旧链路保持兼容:携带信封则同样校验,缺失则
|
|
341
|
+
按旧行为放行。
|
|
342
|
+
"""
|
|
343
|
+
if from_peer not in self.peers:
|
|
344
|
+
return False
|
|
345
|
+
|
|
346
|
+
peer = self.peers[from_peer]
|
|
347
|
+
# v5.4.4 修复 #3:统一信任阈值为 0.3,与 share_memory 保持一致
|
|
348
|
+
if peer.trust_level < 0.3:
|
|
349
|
+
return False
|
|
350
|
+
|
|
351
|
+
signed = bool(signature)
|
|
352
|
+
if not self._verify_signature(memory_data, signature, from_peer):
|
|
353
|
+
return False
|
|
354
|
+
|
|
355
|
+
# 重放防护:签名通道强制;未签名旧链路按内容哈希去重(v5.8.3 P2-2)
|
|
356
|
+
if signed:
|
|
357
|
+
if not self._replay_check(from_peer, memory_data, strict=True):
|
|
358
|
+
return False
|
|
359
|
+
elif not self._replay_check(from_peer, memory_data, strict=False):
|
|
360
|
+
return False
|
|
361
|
+
|
|
362
|
+
# v5.4.2 修复:队列大小上限,超限拒绝新消息
|
|
363
|
+
# P3-05 修复:容量检查与入队必须原子,否则并发 peer 可越过上限。
|
|
364
|
+
with self._fed_lock:
|
|
365
|
+
if len(self._incoming_queue) >= self._MAX_QUEUE_SIZE:
|
|
366
|
+
return False
|
|
367
|
+
self._incoming_queue.append({
|
|
368
|
+
"from": from_peer,
|
|
369
|
+
"data": memory_data,
|
|
370
|
+
"received_at": time.time(),
|
|
371
|
+
})
|
|
372
|
+
|
|
373
|
+
return True
|
|
374
|
+
|
|
375
|
+
def _replay_check(self, peer_id: str, data: Dict, strict: bool) -> bool:
|
|
376
|
+
"""校验时间戳新鲜度与 nonce 唯一性(v5.6.5 P0 #2)
|
|
377
|
+
|
|
378
|
+
strict=True(已签名通道):缺少 _ts/_mid、时间戳超窗、nonce 重复均拒绝。
|
|
379
|
+
strict=False(未签名旧链路):仅当字段可解析时校验,缺失不拦。
|
|
380
|
+
"""
|
|
381
|
+
if not isinstance(data, dict):
|
|
382
|
+
return not strict
|
|
383
|
+
ts = data.get("_ts")
|
|
384
|
+
mid = data.get("_mid")
|
|
385
|
+
now = time.time()
|
|
386
|
+
if ts is None or mid is None:
|
|
387
|
+
if strict:
|
|
388
|
+
return False
|
|
389
|
+
# v5.8.3 P2-2:未签名旧链路(无 _ts/_mid)不再直接放行——
|
|
390
|
+
# 以内容哈希为伪 mid 做去重,同一消息无法无限重放入队。
|
|
391
|
+
try:
|
|
392
|
+
content_digest = hashlib.sha256(
|
|
393
|
+
json.dumps(data, sort_keys=True, ensure_ascii=False,
|
|
394
|
+
default=str).encode("utf-8")).hexdigest()
|
|
395
|
+
except (TypeError, ValueError):
|
|
396
|
+
content_digest = repr(data)
|
|
397
|
+
mid = "sha256:" + content_digest
|
|
398
|
+
ts = now
|
|
399
|
+
try:
|
|
400
|
+
ts = float(ts)
|
|
401
|
+
except (TypeError, ValueError):
|
|
402
|
+
return not strict
|
|
403
|
+
if not isinstance(mid, str) or not mid or len(mid) > 128:
|
|
404
|
+
return not strict
|
|
405
|
+
# 跨节点必须用墙上时钟(time.monotonic 各进程基准不同,无法比较)
|
|
406
|
+
if abs(now - ts) > self.allowed_skew_seconds:
|
|
407
|
+
logger.warning("联邦消息时间戳超窗,拒绝重放: peer=%s skew=%.1fs",
|
|
408
|
+
peer_id, now - ts)
|
|
409
|
+
return False
|
|
410
|
+
|
|
411
|
+
with self._fed_lock:
|
|
412
|
+
seen = self._seen_nonces.setdefault(peer_id, {})
|
|
413
|
+
# 先清理超出偏差窗口的旧 nonce(它们不可能再被合法重放)
|
|
414
|
+
stale = [n for n, t in seen.items() if now - t > self.allowed_skew_seconds]
|
|
415
|
+
for n in stale:
|
|
416
|
+
del seen[n]
|
|
417
|
+
if mid in seen:
|
|
418
|
+
logger.warning("检测到联邦重放消息,已拒绝: peer=%s mid=%s", peer_id, mid)
|
|
419
|
+
return False
|
|
420
|
+
# 容量保护:异常暴涨时拒收新消息而非无限占用内存
|
|
421
|
+
if len(seen) >= self._replay_max_nonces:
|
|
422
|
+
logger.error("节点 %s 的 nonce 去重表已满(%d),暂时拒收",
|
|
423
|
+
peer_id, self._replay_max_nonces)
|
|
424
|
+
return False
|
|
425
|
+
seen[mid] = ts
|
|
426
|
+
return True
|
|
427
|
+
|
|
428
|
+
def accept_incoming(self, memory_index: int = -1,
|
|
429
|
+
resolve_strategy: str = "manual") -> Optional[str]:
|
|
430
|
+
"""接受传入的记忆
|
|
431
|
+
|
|
432
|
+
v5.4.2 新增:注入 conflict_resolver 时,对指向本地已有记忆的
|
|
433
|
+
传入更新做冲突检测与解决:
|
|
434
|
+
- resolve_strategy="lww" 按(版本, 时间戳, peer)决胜,新者覆盖
|
|
435
|
+
- resolve_strategy="keep_both" 传入内容另存分支记忆并建立关联
|
|
436
|
+
- resolve_strategy="manual" 冲突挂起,等待人工处理(返回 None)
|
|
437
|
+
未注入 conflict_resolver 时保持原有直接入库行为。
|
|
438
|
+
"""
|
|
439
|
+
# P3-05 修复:空队列检查与出队必须原子,否则并发 accept 会竞争/越界。
|
|
440
|
+
with self._fed_lock:
|
|
441
|
+
if not self._incoming_queue:
|
|
442
|
+
return None
|
|
443
|
+
try:
|
|
444
|
+
item = self._incoming_queue.pop(memory_index)
|
|
445
|
+
except IndexError:
|
|
446
|
+
return None
|
|
447
|
+
|
|
448
|
+
# v5.4.2 新增:冲突检测与解决
|
|
449
|
+
if self.conflict_resolver is not None and self.storage:
|
|
450
|
+
incoming = dict(item.get("data") or {})
|
|
451
|
+
incoming.setdefault("from_peer", item.get("from", ""))
|
|
452
|
+
try:
|
|
453
|
+
detection = self.conflict_resolver.detect_incoming(incoming)
|
|
454
|
+
except (ValueError, KeyError, TypeError) as e:
|
|
455
|
+
# v5.4.4 修复 #2:只捕获预期异常,不吞掉数据库损坏等严重错误
|
|
456
|
+
logger.warning("detect_incoming 异常(已降级为 new): %s", e)
|
|
457
|
+
detection = {"conflict": False, "action": "new",
|
|
458
|
+
"error": str(e)}
|
|
459
|
+
if detection.get("conflict"):
|
|
460
|
+
if resolve_strategy in ("lww", "keep_both"):
|
|
461
|
+
resolution = self.conflict_resolver.resolve(
|
|
462
|
+
detection["conflict_id"], resolve_strategy,
|
|
463
|
+
actor=str(item.get("from", "")))
|
|
464
|
+
if resolution.get("success"):
|
|
465
|
+
return resolution.get("resolved_memory_id")
|
|
466
|
+
# manual 或解决失败:冲突挂起
|
|
467
|
+
return None
|
|
468
|
+
if detection.get("action") == "noop":
|
|
469
|
+
return detection.get("local_memory_id")
|
|
470
|
+
|
|
471
|
+
if self.storage:
|
|
472
|
+
data = item.get("data") or {}
|
|
473
|
+
content = data.get("content", "")
|
|
474
|
+
# P3-10 修复:空内容不再静默丢弃,显式记录后返回 None
|
|
475
|
+
if not isinstance(content, str) or not content.strip():
|
|
476
|
+
logger.warning("accept_incoming 丢弃空内容: from=%s", item.get("from", ""))
|
|
477
|
+
return None
|
|
478
|
+
raw_tags = data.get("tags", [])
|
|
479
|
+
tags = raw_tags if isinstance(raw_tags, list) else [str(raw_tags)]
|
|
480
|
+
try:
|
|
481
|
+
entry = self.storage.add_memory(
|
|
482
|
+
content=content,
|
|
483
|
+
category=data.get("category", "federated"),
|
|
484
|
+
tags=tags + [f"from:{item['from']}"],
|
|
485
|
+
source_agent=f"federated:{item['from']}",
|
|
486
|
+
metadata={
|
|
487
|
+
"federated_origin": item["from"],
|
|
488
|
+
"received_at": item["received_at"],
|
|
489
|
+
}
|
|
490
|
+
)
|
|
491
|
+
return entry.id
|
|
492
|
+
except (ValueError, TypeError) as e:
|
|
493
|
+
# P3-10 修复:入库失败记录原因,不再静默吞掉
|
|
494
|
+
logger.warning("accept_incoming 入库失败: from=%s err=%s: %s",
|
|
495
|
+
item.get("from", ""), type(e).__name__, e)
|
|
496
|
+
return None
|
|
497
|
+
|
|
498
|
+
return None
|
|
499
|
+
|
|
500
|
+
def federated_search(self, query: str,
|
|
501
|
+
peer_ids: Optional[List[str]] = None,
|
|
502
|
+
max_per_peer: int = 5) -> Dict[str, List[Dict]]:
|
|
503
|
+
"""联邦搜索(跨节点)
|
|
504
|
+
|
|
505
|
+
v5.4.2 修复:返回实际的本地搜索结果 + 将搜索请求入队等待远端响应。
|
|
506
|
+
之前仅写队列返回空 results,导致调用方永远拿不到搜索结果。
|
|
507
|
+
"""
|
|
508
|
+
results = {}
|
|
509
|
+
|
|
510
|
+
search_peers = peer_ids or list(self.peers.keys())
|
|
511
|
+
|
|
512
|
+
# 先在本地 storage 中搜索
|
|
513
|
+
local_results = []
|
|
514
|
+
if self.storage and query:
|
|
515
|
+
try:
|
|
516
|
+
# v5.4.7 修复 H-3:使用 fuzzy_search 替代不存在的 search_memories
|
|
517
|
+
local_results = self.storage.fuzzy_search(
|
|
518
|
+
query, limit=max_per_peer * len(search_peers) or max_per_peer)
|
|
519
|
+
except Exception as _local_err:
|
|
520
|
+
logger.warning("联邦搜索本地召回失败,仅返回远端结果: %s", _local_err)
|
|
521
|
+
|
|
522
|
+
for pid in search_peers:
|
|
523
|
+
if pid not in self.peers:
|
|
524
|
+
continue
|
|
525
|
+
peer = self.peers[pid]
|
|
526
|
+
if peer.status == PeerStatus.OFFLINE:
|
|
527
|
+
continue
|
|
528
|
+
if peer.trust_level < 0.3:
|
|
529
|
+
continue
|
|
530
|
+
|
|
531
|
+
# v5.4.2 修复:队列大小上限保护
|
|
532
|
+
# P3-05 修复:与 share_memory 一致,容量检查与入队需在同一锁内原子完成。
|
|
533
|
+
with self._fed_lock:
|
|
534
|
+
if len(self._outgoing_queue) < self._MAX_QUEUE_SIZE:
|
|
535
|
+
self._outgoing_queue.append({
|
|
536
|
+
"type": "search_request",
|
|
537
|
+
"from": self.local_peer_id,
|
|
538
|
+
"to": pid,
|
|
539
|
+
"query": query,
|
|
540
|
+
"max_results": max_per_peer,
|
|
541
|
+
"timestamp": time.time(),
|
|
542
|
+
})
|
|
543
|
+
|
|
544
|
+
# 将本地结果在活跃 peer 间分片分配(而非每个 peer 拿相同的副本)
|
|
545
|
+
active_peer_ids = [pid for pid in search_peers
|
|
546
|
+
if pid in self.peers
|
|
547
|
+
and self.peers[pid].status != PeerStatus.OFFLINE
|
|
548
|
+
and self.peers[pid].trust_level >= 0.3]
|
|
549
|
+
if active_peer_ids:
|
|
550
|
+
peer_idx = active_peer_ids.index(pid)
|
|
551
|
+
chunk_size = max(1, len(local_results) // len(active_peer_ids))
|
|
552
|
+
start = peer_idx * chunk_size
|
|
553
|
+
end = start + chunk_size if pid != active_peer_ids[-1] else len(local_results)
|
|
554
|
+
results[pid] = local_results[start:end]
|
|
555
|
+
else:
|
|
556
|
+
results[pid] = []
|
|
557
|
+
|
|
558
|
+
return results
|
|
559
|
+
|
|
560
|
+
def purge_expired_shared_memories(self, now: Optional[float] = None) -> int:
|
|
561
|
+
"""清理已过期的共享记忆记录(v5.6.5 P1 #10)
|
|
562
|
+
|
|
563
|
+
expires_at 已过的记录从内存字典删除,返回清理条数。线程安全。
|
|
564
|
+
"""
|
|
565
|
+
if now is None:
|
|
566
|
+
now = time.time()
|
|
567
|
+
# v5.6.6 P3:遍历 + pop 必须与 _replay_check/share/revoke/get 互斥,
|
|
568
|
+
# 否则并发迭代字典时可能 RuntimeError 或与写入相互覆盖。统一用 _fed_lock。
|
|
569
|
+
with self._fed_lock:
|
|
570
|
+
expired = [mid for mid, s in self.shared_memories.items()
|
|
571
|
+
if s.expires_at is not None and s.expires_at <= now]
|
|
572
|
+
for mid in expired:
|
|
573
|
+
self.shared_memories.pop(mid, None)
|
|
574
|
+
if expired:
|
|
575
|
+
logger.info("清理 %d 条过期联邦共享记录", len(expired))
|
|
576
|
+
return len(expired)
|
|
577
|
+
|
|
578
|
+
def _maybe_purge_shared(self) -> None:
|
|
579
|
+
"""节流触发共享记录过期清理(避免每次操作都全表扫描)"""
|
|
580
|
+
now = time.time()
|
|
581
|
+
if now - self._last_shared_purge < self._shared_purge_interval:
|
|
582
|
+
return
|
|
583
|
+
self._last_shared_purge = now
|
|
584
|
+
try:
|
|
585
|
+
self.purge_expired_shared_memories(now)
|
|
586
|
+
except Exception as e: # 清理失败不应影响正常共享
|
|
587
|
+
logger.error("purge_expired_shared_memories failed: %s", e)
|
|
588
|
+
|
|
589
|
+
def get_shared_memories(self, peer_id: Optional[str] = None) -> List[SharedMemory]:
|
|
590
|
+
"""获取共享记忆列表(自动过滤掉已过期记录)"""
|
|
591
|
+
self._maybe_purge_shared()
|
|
592
|
+
now = time.time()
|
|
593
|
+
# v5.6.6 P3:在锁内快照,避免与 purge/share/revoke 并发迭代竞态
|
|
594
|
+
with self._fed_lock:
|
|
595
|
+
if peer_id:
|
|
596
|
+
return [
|
|
597
|
+
s for s in list(self.shared_memories.values())
|
|
598
|
+
if peer_id in s.shared_with
|
|
599
|
+
and (s.expires_at is None or s.expires_at > now)
|
|
600
|
+
]
|
|
601
|
+
return [s for s in list(self.shared_memories.values())
|
|
602
|
+
if s.expires_at is None or s.expires_at > now]
|
|
603
|
+
|
|
604
|
+
def get_peers(self, status: Optional[PeerStatus] = None) -> List[FederatedPeer]:
|
|
605
|
+
"""获取节点列表"""
|
|
606
|
+
peers = list(self.peers.values())
|
|
607
|
+
if status:
|
|
608
|
+
peers = [p for p in peers if p.status == status]
|
|
609
|
+
return peers
|
|
610
|
+
|
|
611
|
+
def get_queue_sizes(self) -> Dict[str, int]:
|
|
612
|
+
"""获取队列大小"""
|
|
613
|
+
return {
|
|
614
|
+
"incoming": len(self._incoming_queue),
|
|
615
|
+
"outgoing": len(self._outgoing_queue),
|
|
616
|
+
}
|
|
617
|
+
|
|
618
|
+
def _verify_memory_exists(self, memory_id: str) -> bool:
|
|
619
|
+
"""验证记忆存在
|
|
620
|
+
|
|
621
|
+
v5.6.2 安全修复:无 storage 时 fail-closed 返回 False,
|
|
622
|
+
而非 fail-open 返回 True。
|
|
623
|
+
"""
|
|
624
|
+
if not self.storage:
|
|
625
|
+
return False # fail-closed:无法验证则视为不存在
|
|
626
|
+
try:
|
|
627
|
+
entry = self.storage.get_memory(memory_id)
|
|
628
|
+
return entry is not None
|
|
629
|
+
except Exception:
|
|
630
|
+
# v5.4.5 修复:移除顶层 import sqlite3,改为通用异常捕获
|
|
631
|
+
# OperationalError 等数据库异常在此场景下统一返回 False
|
|
632
|
+
return False
|
|
633
|
+
|
|
634
|
+
@staticmethod
|
|
635
|
+
def _canonical_message(data: Dict) -> bytes:
|
|
636
|
+
"""确定性序列化待签名内容(HMAC 与 Ed25519 共用,保持历史兼容)"""
|
|
637
|
+
return json.dumps(data, sort_keys=True, ensure_ascii=False).encode("utf-8")
|
|
638
|
+
|
|
639
|
+
def _compute_signature(self, data: Dict, peer_id: str) -> str:
|
|
640
|
+
"""计算签名(v5.6.5 P0 #1)
|
|
641
|
+
|
|
642
|
+
- 对端配置了 Ed25519 public_key:用**本节点私钥**做非对称签名(b64)。
|
|
643
|
+
- 否则对端仅有 shared_secret:回退 HMAC-SHA256(旧版对称,迁移用)。
|
|
644
|
+
- 两者皆无:返回 ""(验签端 fail-closed)。
|
|
645
|
+
"""
|
|
646
|
+
peer = self.peers.get(peer_id)
|
|
647
|
+
if not peer:
|
|
648
|
+
return ""
|
|
649
|
+
msg = self._canonical_message(data)
|
|
650
|
+
|
|
651
|
+
if peer.public_key:
|
|
652
|
+
try:
|
|
653
|
+
from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey
|
|
654
|
+
priv = Ed25519PrivateKey.from_private_bytes(_b64d(self._local_priv_b64))
|
|
655
|
+
return _b64e(priv.sign(msg))
|
|
656
|
+
except (ValueError, binascii.Error, TypeError) as e:
|
|
657
|
+
logger.error("Ed25519 签名失败: %s", e)
|
|
658
|
+
return ""
|
|
659
|
+
|
|
660
|
+
if not peer.shared_secret:
|
|
661
|
+
return ""
|
|
662
|
+
key = peer.shared_secret.encode("utf-8")
|
|
663
|
+
return hmac.new(key, msg, digestmod=hashlib.sha256).hexdigest()
|
|
664
|
+
|
|
665
|
+
def _verify_signature(self, data: Dict, signature: str, peer_id: str) -> bool:
|
|
666
|
+
"""验证签名(v5.6.5 P0 #1)
|
|
667
|
+
|
|
668
|
+
按对端注册的密钥材料选择 Ed25519 或 HMAC 验签。fail-closed:
|
|
669
|
+
- 未注册节点:拒绝
|
|
670
|
+
- 无签名:仅当 allow_unsigned_peers=True 时允许
|
|
671
|
+
- Ed25519 公钥存在:非对称验签,非法/损坏签名一律拒绝
|
|
672
|
+
- 仅 shared_secret:HMAC 恒定时间比较
|
|
673
|
+
"""
|
|
674
|
+
peer = self.peers.get(peer_id)
|
|
675
|
+
if not peer:
|
|
676
|
+
return False # fail-closed: 未注册节点直接拒绝
|
|
677
|
+
|
|
678
|
+
if not signature:
|
|
679
|
+
# 无签名时仅允许显式配置的未签名节点
|
|
680
|
+
return self.allow_unsigned_peers
|
|
681
|
+
|
|
682
|
+
msg = self._canonical_message(data)
|
|
683
|
+
|
|
684
|
+
if peer.public_key:
|
|
685
|
+
try:
|
|
686
|
+
from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey
|
|
687
|
+
from cryptography.exceptions import InvalidSignature
|
|
688
|
+
pub = Ed25519PublicKey.from_public_bytes(_b64d(peer.public_key))
|
|
689
|
+
try:
|
|
690
|
+
pub.verify(_b64d(signature), msg)
|
|
691
|
+
return True
|
|
692
|
+
except (InvalidSignature, ValueError, binascii.Error):
|
|
693
|
+
return False
|
|
694
|
+
except (ValueError, binascii.Error) as e:
|
|
695
|
+
logger.error("节点 %s 的 Ed25519 公钥无效: %s", peer_id, e)
|
|
696
|
+
return False
|
|
697
|
+
|
|
698
|
+
if not peer.shared_secret:
|
|
699
|
+
return False # fail-closed: 无任何密钥材料的节点直接拒绝
|
|
700
|
+
|
|
701
|
+
expected = self._compute_signature(data, peer_id)
|
|
702
|
+
if not expected:
|
|
703
|
+
return False
|
|
704
|
+
return hmac.compare_digest(signature, expected)
|
|
705
|
+
|
|
706
|
+
def compute_federated_stats(self) -> Dict:
|
|
707
|
+
"""计算联邦统计"""
|
|
708
|
+
self._maybe_purge_shared()
|
|
709
|
+
return {
|
|
710
|
+
"local_peer_id": self.local_peer_id,
|
|
711
|
+
"local_public_key": self._local_pub_b64,
|
|
712
|
+
"total_peers": len(self.peers),
|
|
713
|
+
"online_peers": sum(1 for p in self.peers.values() if p.status == PeerStatus.ONLINE),
|
|
714
|
+
"trusted_peers": sum(1 for p in self.peers.values() if p.trust_level >= 0.7),
|
|
715
|
+
"shared_memories": len(self.shared_memories),
|
|
716
|
+
"incoming_queue": len(self._incoming_queue),
|
|
717
|
+
"outgoing_queue": len(self._outgoing_queue),
|
|
718
|
+
"avg_trust_level": (
|
|
719
|
+
sum(p.trust_level for p in self.peers.values()) / len(self.peers)
|
|
720
|
+
if self.peers else 0
|
|
721
|
+
),
|
|
722
|
+
}
|