secagentx 3.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (182) hide show
  1. backend/__init__.py +1 -0
  2. backend/__main__.py +3 -0
  3. backend/_assets/config.yaml +195 -0
  4. backend/_assets/frontend/dist/assets/AgentsView-DFqRplI8.css +1 -0
  5. backend/_assets/frontend/dist/assets/AgentsView-DJTpfQ87.js +1 -0
  6. backend/_assets/frontend/dist/assets/AttackChainView-BVfI-SC6.js +1 -0
  7. backend/_assets/frontend/dist/assets/ChatView-Bg4nIlQ0.css +1 -0
  8. backend/_assets/frontend/dist/assets/ChatView-CRndtmEC.js +38 -0
  9. backend/_assets/frontend/dist/assets/DashboardView-BXs4AWv8.js +1 -0
  10. backend/_assets/frontend/dist/assets/DashboardView-WnHWiqBW.css +1 -0
  11. backend/_assets/frontend/dist/assets/EventDetailView-DQk6Qp7-.js +1 -0
  12. backend/_assets/frontend/dist/assets/EventDetailView-Dj-a-zWr.css +1 -0
  13. backend/_assets/frontend/dist/assets/EventsView-8xh8uKO0.js +1 -0
  14. backend/_assets/frontend/dist/assets/EventsView-BHfFn66K.css +1 -0
  15. backend/_assets/frontend/dist/assets/KnowledgeView-C2N3v9bF.css +1 -0
  16. backend/_assets/frontend/dist/assets/KnowledgeView-D54ODoC9.js +11 -0
  17. backend/_assets/frontend/dist/assets/LoginView-CjvFmo7C.js +1 -0
  18. backend/_assets/frontend/dist/assets/LoginView-D14FzcsP.css +1 -0
  19. backend/_assets/frontend/dist/assets/SettingsView-JOJuoABi.js +1 -0
  20. backend/_assets/frontend/dist/assets/UsersView-DhMXZhir.js +1 -0
  21. backend/_assets/frontend/dist/assets/_plugin-vue_export-helper-BDNMzG2s.js +1 -0
  22. backend/_assets/frontend/dist/assets/index-D9Su3fYY.css +1 -0
  23. backend/_assets/frontend/dist/assets/index-DBEjbyJR.js +4 -0
  24. backend/_assets/frontend/dist/assets/rolldown-runtime-DK3Fl9T5.js +1 -0
  25. backend/_assets/frontend/dist/assets/vendor-charts-Dh1nChzO.js +39 -0
  26. backend/_assets/frontend/dist/assets/vendor-icons-BTNgi7vU.js +1 -0
  27. backend/_assets/frontend/dist/assets/vendor-ui-Cdd65KTN.js +7367 -0
  28. backend/_assets/frontend/dist/assets/vendor-vue-CXKcZAmz.js +1 -0
  29. backend/_assets/frontend/dist/index.html +51 -0
  30. backend/_assets/frontend/index.html +46 -0
  31. backend/_assets/frontend/package-lock.json +2983 -0
  32. backend/_assets/frontend/package.json +37 -0
  33. backend/_assets/frontend/src/App.vue +231 -0
  34. backend/_assets/frontend/src/main.js +12 -0
  35. backend/_assets/frontend/src/router/index.js +117 -0
  36. backend/_assets/frontend/src/stores/chat.js +671 -0
  37. backend/_assets/frontend/src/styles/tokens.css +282 -0
  38. backend/_assets/frontend/src/utils/http.js +191 -0
  39. backend/_assets/frontend/src/views/AgentsView.vue +227 -0
  40. backend/_assets/frontend/src/views/AttackChainView.vue +210 -0
  41. backend/_assets/frontend/src/views/ChatView.vue +1107 -0
  42. backend/_assets/frontend/src/views/DashboardView.vue +514 -0
  43. backend/_assets/frontend/src/views/EventDetailView.vue +387 -0
  44. backend/_assets/frontend/src/views/EventsView.vue +253 -0
  45. backend/_assets/frontend/src/views/KnowledgeView.vue +883 -0
  46. backend/_assets/frontend/src/views/LoginView.vue +314 -0
  47. backend/_assets/frontend/src/views/SettingsView.vue +226 -0
  48. backend/_assets/frontend/src/views/UsersView.vue +235 -0
  49. backend/_assets/frontend/vite.config.js +55 -0
  50. backend/_assets/knowledge_data/compliance/regulations.json +383 -0
  51. backend/_assets/knowledge_data/cve/vulnerabilities.json +70720 -0
  52. backend/_assets/knowledge_data/mitre_attack/techniques.json +14097 -0
  53. backend/_assets/knowledge_data/remediation/remediation.json +574 -0
  54. backend/_assets/knowledge_data/threat_intel/actors.json +1602 -0
  55. backend/_assets/knowledge_data/threat_intel/malware.json +971 -0
  56. backend/agents/__init__.py +1 -0
  57. backend/agents/alert_filter/__init__.py +1 -0
  58. backend/agents/alert_filter/alert_filter_agent.py +88 -0
  59. backend/agents/analyst.py +331 -0
  60. backend/agents/base.py +553 -0
  61. backend/agents/classifier_agent.py +134 -0
  62. backend/agents/cot_engine.py +578 -0
  63. backend/agents/intel.py +46 -0
  64. backend/agents/knowledge/__init__.py +2 -0
  65. backend/agents/knowledge/agentic_rag.py +928 -0
  66. backend/agents/knowledge/knowledge_agent.py +47 -0
  67. backend/agents/registry.py +148 -0
  68. backend/agents/responder.py +51 -0
  69. backend/agents/summary_agent.py +219 -0
  70. backend/auto_ingestor.py +948 -0
  71. backend/auto_patrol.py +452 -0
  72. backend/cli_app.py +523 -0
  73. backend/config/__init__.py +20 -0
  74. backend/config/provider_profiles.py +416 -0
  75. backend/config/runtime_settings.py +84 -0
  76. backend/decision_fusion/__init__.py +32 -0
  77. backend/decision_fusion/base.py +127 -0
  78. backend/decision_fusion/dempster_shafer.py +419 -0
  79. backend/decision_fusion/weighted_average.py +141 -0
  80. backend/email_ingestor.py +588 -0
  81. backend/errors.py +243 -0
  82. backend/escalation.py +453 -0
  83. backend/evaluation/__init__.py +7 -0
  84. backend/evaluation/evaluator.py +197 -0
  85. backend/evaluation/metrics.py +210 -0
  86. backend/federation/__init__.py +22 -0
  87. backend/federation/core.py +743 -0
  88. backend/interface/__init__.py +1 -0
  89. backend/interface/api_server.py +1830 -0
  90. backend/interface/cli.py +671 -0
  91. backend/knowledge/__init__.py +11 -0
  92. backend/knowledge/compliance.py +101 -0
  93. backend/knowledge/cve_db.py +100 -0
  94. backend/knowledge/mitre_attack.py +457 -0
  95. backend/knowledge/remediation.py +84 -0
  96. backend/knowledge/threat_intel_kb.py +246 -0
  97. backend/llm/__init__.py +1 -0
  98. backend/llm/anthropic_compatible.py +191 -0
  99. backend/llm/base.py +81 -0
  100. backend/llm/deepseek.py +221 -0
  101. backend/llm/fallback.py +268 -0
  102. backend/llm/mock.py +204 -0
  103. backend/llm/openai_compatible.py +51 -0
  104. backend/llm/provider.py +217 -0
  105. backend/llm/qwen.py +218 -0
  106. backend/main.py +421 -0
  107. backend/memory/__init__.py +1 -0
  108. backend/memory/episodic_memory.py +132 -0
  109. backend/memory/manager.py +204 -0
  110. backend/memory/semantic_memory.py +156 -0
  111. backend/memory/session_memory.py +102 -0
  112. backend/ml_model/__init__.py +23 -0
  113. backend/ml_model/pipeline.py +191 -0
  114. backend/ml_model/trainer.py +627 -0
  115. backend/models/__init__.py +1 -0
  116. backend/models/event.py +62 -0
  117. backend/models/message.py +58 -0
  118. backend/models/output.py +1216 -0
  119. backend/monitoring/metrics.py +184 -0
  120. backend/orchestrator/__init__.py +17 -0
  121. backend/orchestrator/agent_router.py +145 -0
  122. backend/orchestrator/core.py +159 -0
  123. backend/orchestrator/dynamic_weights.py +287 -0
  124. backend/orchestrator/react_loop.py +1895 -0
  125. backend/orchestrator/risk_model.py +494 -0
  126. backend/orchestrator/risk_scorer.py +356 -0
  127. backend/reasoner/__init__.py +19 -0
  128. backend/reasoner/reasoner.py +316 -0
  129. backend/runtime_assets.py +45 -0
  130. backend/security/__init__.py +24 -0
  131. backend/security/adversarial/__init__.py +13 -0
  132. backend/security/adversarial/guard.py +121 -0
  133. backend/security/adversarial/redteam.py +121 -0
  134. backend/security/adversarial/scanner.py +208 -0
  135. backend/security/audit.py +118 -0
  136. backend/security/auth.py +283 -0
  137. backend/security/circuit_breaker.py +212 -0
  138. backend/security/dispatch.py +18 -0
  139. backend/security/ratelimit.py +237 -0
  140. backend/security/rbac.py +431 -0
  141. backend/security/refresh_sessions.py +193 -0
  142. backend/security/sanitizer.py +32 -0
  143. backend/storage/__init__.py +1 -0
  144. backend/storage/chroma_store.py +79 -0
  145. backend/storage/database.py +296 -0
  146. backend/storage/migrations/env.py +65 -0
  147. backend/storage/migrations/versions/0001_initial_schema.py +101 -0
  148. backend/storage/migrations/versions/0002_add_agent_logs_experiences.py +53 -0
  149. backend/storage/migrations/versions/0003_add_conversation_owner.py +30 -0
  150. backend/storage/models.py +194 -0
  151. backend/storage/repositories/__init__.py +1 -0
  152. backend/storage/repositories/asset_repo.py +86 -0
  153. backend/storage/repositories/conversation_repo.py +216 -0
  154. backend/storage/repositories/event_repo.py +76 -0
  155. backend/storage/repositories/trajectory_repo.py +324 -0
  156. backend/storage/retention.py +117 -0
  157. backend/tools/__init__.py +16 -0
  158. backend/tools/alert_filter.py +208 -0
  159. backend/tools/aliyun_firewall.py +237 -0
  160. backend/tools/aws_firewall.py +369 -0
  161. backend/tools/base.py +445 -0
  162. backend/tools/calling.py +207 -0
  163. backend/tools/cve_search.py +77 -0
  164. backend/tools/execution_engine.py +202 -0
  165. backend/tools/firewall.py +387 -0
  166. backend/tools/geoip.py +168 -0
  167. backend/tools/log_analyzer.py +185 -0
  168. backend/tools/ml_detector.py +226 -0
  169. backend/tools/registry.py +38 -0
  170. backend/tools/tencent_firewall.py +300 -0
  171. backend/tools/threat_intel.py +732 -0
  172. backend/tools/trust_checker.py +76 -0
  173. backend/utils/__init__.py +5 -0
  174. backend/utils/config_loader.py +164 -0
  175. backend/utils/env.py +75 -0
  176. backend/utils/logger.py +150 -0
  177. backend/utils/text.py +32 -0
  178. secagentx-3.0.0.dist-info/METADATA +325 -0
  179. secagentx-3.0.0.dist-info/RECORD +182 -0
  180. secagentx-3.0.0.dist-info/WHEEL +5 -0
  181. secagentx-3.0.0.dist-info/entry_points.txt +2 -0
  182. secagentx-3.0.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,204 @@
1
+ """
2
+ MemoryManager — 记忆管理器(v2.4 M4)
3
+
4
+ 统一读写三层记忆:
5
+ 1. SessionMemory — 短期工作记忆(当前会话关键事实)
6
+ 2. EpisodicMemory — 情景记忆(历史研判案例,experiences 表)
7
+ 3. SemanticMemory — 语义记忆(高置信度事实向量沉淀)
8
+
9
+ 核心职责:
10
+ - build_context(text): 召回三层记忆,构造注入 system prompt 的记忆上下文
11
+ - remember_from_result(...): 从一次研判结果提取关键事实并分层写入
12
+ - 记忆注入带置信度 + 时间衰减,低置信度不注入(避免误导 LLM)
13
+
14
+ 设计纪律(延续项目纪律):
15
+ - 所有记忆带置信度(来自确定性裁决),非 LLM 自拟
16
+ - 语义记忆仅沉淀高置信度(≥0.7)事实,防止噪声污染
17
+ - 记忆不可用(DB/Chroma 故障)→ 全部旁路,主链路永不阻断
18
+ """
19
+ import logging
20
+ from typing import Optional
21
+
22
+ logger = logging.getLogger("secagentx.memory")
23
+
24
+ # 记忆注入模板:插入 system prompt 的记忆上下文
25
+ MEMORY_CONTEXT_TEMPLATE = """
26
+ ## 历史记忆(带置信度,仅供参考,不作为唯一依据)
27
+ {items}
28
+
29
+ > 记忆标注: 每条记忆带置信度;旧记忆可能已衰减。若记忆与当前证据冲突,以当前证据为准。
30
+ """
31
+
32
+
33
+ class MemoryManager:
34
+ """三层记忆管理器。"""
35
+
36
+ def __init__(self, session_memory: Optional[dict] = None,
37
+ episodic: Optional[object] = None,
38
+ semantic: Optional[object] = None):
39
+ """
40
+ 参数:
41
+ session_memory: dict {session_id: SessionMemory} 会话记忆注册表
42
+ episodic: EpisodicMemory 实例(可空)
43
+ semantic: SemanticMemory 实例(可空)
44
+ """
45
+ self._sessions = session_memory if session_memory is not None else {}
46
+ self.episodic = episodic
47
+ self.semantic = semantic
48
+
49
+ # ═══════════════ 会话记忆 ═══════════════
50
+ def get_session(self, session_id: str):
51
+ """获取/创建会话记忆。"""
52
+ if session_id not in self._sessions:
53
+ from .session_memory import SessionMemory
54
+ self._sessions[session_id] = SessionMemory(session_id=session_id)
55
+ return self._sessions[session_id]
56
+
57
+ def remember_session(self, session_id: str, content: str,
58
+ category: str = "general", confidence: float = 0.5,
59
+ metadata: Optional[dict] = None):
60
+ try:
61
+ self.get_session(session_id).remember(
62
+ content, category=category, confidence=confidence, metadata=metadata,
63
+ )
64
+ except Exception as e:
65
+ logger.debug("会话记忆写入失败(旁路): %s", e)
66
+
67
+ # ═══════════════ 记忆上下文构建 ═══════════════
68
+ def build_context(self, text: str, session_id: str = "",
69
+ top_k: int = 6) -> str:
70
+ """召回三层记忆,构造注入 system prompt 的记忆上下文。
71
+
72
+ 返回 markdown 段落(空 → 不注入)。
73
+ """
74
+ items = []
75
+
76
+ # 1. 会话记忆(短期,当前会话)
77
+ if session_id:
78
+ try:
79
+ for f in self.get_session(session_id).to_context(limit=top_k):
80
+ items.append(
81
+ f"- [会话·{f['category']}] {f['content']}(置信度 {f['confidence']:.0%})"
82
+ )
83
+ except Exception:
84
+ pass
85
+
86
+ # 2. 情景记忆(历史案例)
87
+ if self.episodic is not None:
88
+ try:
89
+ import asyncio
90
+ episodes = asyncio.get_event_loop().run_until_complete(
91
+ self.episodic.recall_similar(text, limit=3),
92
+ )
93
+ for ep in episodes:
94
+ items.append(
95
+ f"- [案例·{ep['scenario'][:40]}] 处置: {ep.get('outcome', '')[:80]}"
96
+ )
97
+ except Exception:
98
+ pass
99
+
100
+ # 3. 语义记忆(长期向量)
101
+ if self.semantic is not None:
102
+ try:
103
+ for m in self.semantic.recall(text, k=top_k):
104
+ items.append(
105
+ f"- [语义·{m['category']}] {m['content']}(置信度 {m['confidence']:.0%})"
106
+ )
107
+ except Exception:
108
+ pass
109
+
110
+ if not items:
111
+ return ""
112
+ return MEMORY_CONTEXT_TEMPLATE.format(items="\n".join(items[:top_k]))
113
+
114
+ # ═══════════════ 研判结果沉淀 ═══════════════
115
+ def remember_from_result(self, session_id: str, text: str,
116
+ agent_results: list = None,
117
+ risk_scorecard: dict = None,
118
+ verdict: dict = None) -> int:
119
+ """从一次研判结果提取关键事实并分层写入。返回写入条数。"""
120
+ count = 0
121
+ try:
122
+ # 1. 裁决级事实(高置信度 -> 沉淀)
123
+ if verdict:
124
+ v = verdict.get("verdict", "unknown")
125
+ conf = float(verdict.get("confidence", 0.0) or 0.0)
126
+ if v in ("malicious", "benign") and conf > 0:
127
+ self.remember_session(
128
+ session_id,
129
+ f"分析 '{text[:50]}' 裁决为 {v}(置信度 {conf:.0%})",
130
+ category="verdict", confidence=conf,
131
+ metadata={"verdict": v, "source_text": text[:200]},
132
+ )
133
+ count += 1
134
+ # 高置信度恶意 -> 沉淀语义记忆(长期)
135
+ if self.semantic is not None and conf >= 0.7:
136
+ mid = self.semantic.add(
137
+ f"确认 {v}: '{text[:80]}'(置信度 {conf:.0%})",
138
+ category="verdict", confidence=conf,
139
+ metadata={"verdict": v},
140
+ )
141
+ if mid:
142
+ count += 1
143
+
144
+ # 2. Agent 裁决级事实
145
+ for r in (agent_results or []):
146
+ aid = r.get("agent_id", "")
147
+ rv = r.get("verdict")
148
+ rc = float(r.get("confidence") or 0.0)
149
+ if rv in ("malicious", "benign") and rc >= 0.5:
150
+ self.remember_session(
151
+ session_id,
152
+ f"{aid} 裁决 {rv}(置信度 {rc:.0%})",
153
+ category="agent_verdict", confidence=rc,
154
+ metadata={"agent_id": aid, "verdict": rv},
155
+ )
156
+ count += 1
157
+
158
+ # 3. 风险评分级事实
159
+ if risk_scorecard:
160
+ score = risk_scorecard.get("risk_score")
161
+ level = risk_scorecard.get("risk_level")
162
+ if score is not None:
163
+ self.remember_session(
164
+ session_id,
165
+ f"风险评分 {score}({level})",
166
+ category="risk", confidence=0.8,
167
+ metadata={"score": score, "level": level},
168
+ )
169
+ count += 1
170
+ except Exception as e:
171
+ logger.debug("记忆沉淀失败(旁路): %s", e)
172
+ return count
173
+
174
+ # ═══════════════ 查询/管理 ═══════════════
175
+ def list_session_memory(self, session_id: str) -> list[dict]:
176
+ return self.get_session(session_id).facts
177
+
178
+ def clear_session(self, session_id: str):
179
+ self.get_session(session_id).clear()
180
+
181
+ async def stats(self) -> dict:
182
+ """记忆统计(async — episodic.count 为 async 方法)。"""
183
+ import asyncio
184
+ episodic_count = 0
185
+ semantic_count = 0
186
+ if self.episodic is not None:
187
+ try:
188
+ episodic_count = await self.episodic.count()
189
+ except Exception:
190
+ episodic_count = 0
191
+ if self.semantic is not None:
192
+ try:
193
+ semantic_count = self.semantic.count()
194
+ except Exception:
195
+ semantic_count = 0
196
+ return {
197
+ "sessions": len(self._sessions),
198
+ "episodic_count": episodic_count,
199
+ "semantic_count": semantic_count,
200
+ }
201
+
202
+
203
+ __all__ = ["MemoryManager", "MEMORY_CONTEXT_TEMPLATE"]
204
+
@@ -0,0 +1,156 @@
1
+ """
2
+ SemanticMemory — 语义记忆(v2.4 M4)
3
+
4
+ 长期向量记忆:将高价值事实(已确认 IOC、资产偏好、历史结论)写入 ChromaDB
5
+ 独立 collection `session_memory`,支持:
6
+ - add: 写入带置信度的语义事实
7
+ - recall: 向量相似度召回(可用本地哈希嵌入降级)
8
+ - forget: 按 ID 删除
9
+ - count: 记忆条数
10
+
11
+ 设计纪律:
12
+ - 写入时必须带置信度阈值(低于阈值不沉淀,避免噪声污染长期记忆)
13
+ - 召回结果带 distance 分数,低相似度(高距离)不注入
14
+ """
15
+ import hashlib
16
+ import time
17
+ import logging
18
+ from typing import Optional
19
+
20
+ logger = logging.getLogger("secagentx.memory")
21
+
22
+ # 默认 collection 名
23
+ COLLECTION_NAME = "session_memory"
24
+ # 写入置信度阈值
25
+ MIN_CONFIDENCE_TO_STORE = 0.7
26
+ # 归一化距离阈值:超过则视为"不相关",不注入
27
+ MAX_DISTANCE = 1.2
28
+
29
+ try:
30
+ import numpy as np
31
+ HAS_NUMPY = True
32
+ except ImportError:
33
+ HAS_NUMPY = False
34
+
35
+ # 嵌入维度(本地哈希降级方案)
36
+ EMBEDDING_DIM = 64
37
+
38
+
39
+ def _hash_embed(text: str) -> list[float]:
40
+ """本地哈希嵌入降级方案(无模型时使用):
41
+ 对文本提取 2~5-gram,MD5 哈希后映射到固定维度,按词频归一化。
42
+ """
43
+ vec = [0.0] * EMBEDDING_DIM
44
+ ngrams = set()
45
+ for n in range(2, 6):
46
+ for i in range(len(text) - n + 1):
47
+ gram = text[i:i + n]
48
+ h = int(hashlib.md5(gram.encode('utf-8')).hexdigest(), 16)
49
+ ngrams.add(h % EMBEDDING_DIM)
50
+ for idx in ngrams:
51
+ vec[idx] += 1.0
52
+ norm = sum(x * x for x in vec) ** 0.5
53
+ if norm > 0:
54
+ vec = [x / norm for x in vec]
55
+ return vec
56
+
57
+
58
+ class SemanticMemory:
59
+ """语义记忆 — 长期向量记忆(ChromaDB + 本地哈希降级)。"""
60
+
61
+ def __init__(self, vector_store=None, min_confidence: float = MIN_CONFIDENCE_TO_STORE):
62
+ self.vector_store = vector_store
63
+ self.min_confidence = min_confidence
64
+
65
+ def _store_ready(self) -> bool:
66
+ return self.vector_store is not None
67
+
68
+ def add(self, content: str, category: str = "general",
69
+ confidence: float = 0.5, metadata: dict = None) -> Optional[str]:
70
+ """写入一条语义记忆(低于置信度阈值不沉淀)。"""
71
+ if confidence < self.min_confidence:
72
+ logger.debug("语义记忆跳过低置信度事实: %.2f < %.2f", confidence, self.min_confidence)
73
+ return None
74
+ if not self._store_ready():
75
+ return None
76
+ mid = f"mem-{int(time.time() * 1000)}-{hashlib.md5(content.encode('utf-8')).hexdigest()[:6]}"
77
+ meta = dict(metadata or {})
78
+ meta["category"] = category
79
+ meta["confidence"] = confidence
80
+ meta["created_at"] = time.time()
81
+ try:
82
+ emb = _hash_embed(content)
83
+ self.vector_store.add_documents(
84
+ COLLECTION_NAME,
85
+ ids=[mid],
86
+ documents=[content],
87
+ metadatas=[meta],
88
+ embeddings=[emb],
89
+ )
90
+ return mid
91
+ except Exception as e:
92
+ logger.debug("语义记忆写入失败(旁路): %s", e)
93
+ return None
94
+
95
+ def recall(self, query: str, k: int = 5, category: str = None) -> list[dict]:
96
+ """向量召回相似记忆,过滤距离过远 + 可选按类别过滤。"""
97
+ if not self._store_ready():
98
+ return []
99
+ try:
100
+ emb = _hash_embed(query)
101
+ items = self.vector_store.similarity_search(
102
+ COLLECTION_NAME, k=k * 3, query_embeddings=emb,
103
+ )
104
+ out = []
105
+ for it in items:
106
+ if it.get("distance", 0) > MAX_DISTANCE:
107
+ continue
108
+ meta = it.get("metadata") or {}
109
+ if category and meta.get("category") != category:
110
+ continue
111
+ out.append({
112
+ "id": it.get("id"),
113
+ "content": it.get("document", ""),
114
+ "category": meta.get("category", "general"),
115
+ "confidence": meta.get("confidence", 0.5),
116
+ "distance": round(it.get("distance", 0), 4),
117
+ "created_at": meta.get("created_at", 0),
118
+ })
119
+ return out[:k]
120
+ except Exception as e:
121
+ logger.debug("语义记忆召回失败(旁路): %s", e)
122
+ return []
123
+
124
+ def count(self) -> int:
125
+ if not self._store_ready():
126
+ return 0
127
+ try:
128
+ return self.vector_store.count(COLLECTION_NAME)
129
+ except Exception:
130
+ return 0
131
+
132
+ def forget(self, mem_id: str) -> bool:
133
+ """删除单条记忆(按 ID)。"""
134
+ if not self._store_ready():
135
+ return False
136
+ try:
137
+ coll = self.vector_store.get_or_create_collection(COLLECTION_NAME)
138
+ coll.delete(ids=[mem_id])
139
+ return True
140
+ except Exception as e:
141
+ logger.debug("语义记忆删除失败(旁路): %s", e)
142
+ return False
143
+
144
+ def clear(self) -> bool:
145
+ """清空全部语义记忆。"""
146
+ if not self._store_ready():
147
+ return False
148
+ try:
149
+ self.vector_store.delete_collection(COLLECTION_NAME)
150
+ return True
151
+ except Exception:
152
+ return False
153
+
154
+
155
+ __all__ = ["SemanticMemory", "_hash_embed", "COLLECTION_NAME"]
156
+
@@ -0,0 +1,102 @@
1
+ """
2
+ SessionMemory — 短期会话记忆(v2.4 M4)
3
+
4
+ 维护当前会话的关键事实列表,支持:
5
+ - remember: 写入结构化事实(IP/CVE/MITRE/裁决/置信度等)
6
+ - recall: 按类型/关键词召回
7
+ - 容量限制: 超限自动压缩(丢最旧)
8
+ - 时间衰减: 旧记忆置信度打折(权重随时间衰减,防止误导)
9
+
10
+ 设计纪律:
11
+ - 记忆条目必须带置信度(来自确定性裁决,非 LLM 自拟)
12
+ - 过期/衰减的记忆在召回时标注,注入 system prompt 时降低优先级
13
+ """
14
+ import time
15
+ from typing import Optional
16
+
17
+ # 默认单会话记忆容量
18
+ DEFAULT_MAX_FACTS = 50
19
+ # 记忆半衰期(秒)— 超过后置信度衰减
20
+ DEFAULT_HALF_LIFE = 3600 # 1 小时
21
+
22
+
23
+ class SessionMemory:
24
+ """会话工作记忆 — 线程内单实例,每个会话独立。"""
25
+
26
+ def __init__(self, session_id: str = "", max_facts: int = DEFAULT_MAX_FACTS,
27
+ half_life: float = DEFAULT_HALF_LIFE):
28
+ self.session_id = session_id
29
+ self.max_facts = max_facts
30
+ self.half_life = half_life
31
+ self.facts: list[dict] = [] # 按时间升序
32
+
33
+ def remember(self, content: str, category: str = "general",
34
+ confidence: float = 0.5, metadata: Optional[dict] = None) -> dict:
35
+ """写入一条结构化记忆。category 建议: ip/ioc/cve/mitre/verdict/context。"""
36
+ now = time.time()
37
+ fact = {
38
+ "id": f"{self.session_id}-{now:.0f}-{len(self.facts)}",
39
+ "content": content,
40
+ "category": category,
41
+ "confidence": float(confidence),
42
+ "created_at": now,
43
+ "metadata": metadata or {},
44
+ }
45
+ # 去重:同一 category + 高相似内容(简单用内容前缀)-> 更新而非追加
46
+ for existing in self.facts:
47
+ if existing["category"] == category and existing["content"][:50] == content[:50]:
48
+ existing["content"] = content
49
+ existing["confidence"] = float(confidence)
50
+ existing["created_at"] = now
51
+ existing["metadata"] = metadata or {}
52
+ return existing
53
+ self.facts.append(fact)
54
+ # 容量限制:超限丢最旧
55
+ if len(self.facts) > self.max_facts:
56
+ self.facts = self.facts[-self.max_facts:]
57
+ return fact
58
+
59
+ def recall(self, category: Optional[str] = None, keyword: str = "",
60
+ limit: int = 10, with_decay: bool = True) -> list[dict]:
61
+ """召回记忆。with_decay=True 时旧记忆置信度按半衰期衰减。"""
62
+ now = time.time()
63
+ results = []
64
+ for f in self.facts:
65
+ if category and f["category"] != category:
66
+ continue
67
+ if keyword and keyword.lower() not in f["content"].lower():
68
+ continue
69
+ item = dict(f)
70
+ if with_decay:
71
+ age = now - f["created_at"]
72
+ decay = 0.5 ** (age / self.half_life) if self.half_life > 0 else 1.0
73
+ item["effective_confidence"] = round(f["confidence"] * decay, 4)
74
+ else:
75
+ item["effective_confidence"] = f["confidence"]
76
+ results.append(item)
77
+ # 按有效置信度降序
78
+ results.sort(key=lambda x: x["effective_confidence"], reverse=True)
79
+ return results[:limit]
80
+
81
+ def clear(self):
82
+ self.facts = []
83
+
84
+ def count(self) -> int:
85
+ return len(self.facts)
86
+
87
+ def to_context(self, limit: int = 8) -> list[dict]:
88
+ """生成注入 LLM system prompt 的记忆上下文(仅高置信度 + 未衰减)。"""
89
+ recalled = self.recall(with_decay=True, limit=limit)
90
+ return [
91
+ {
92
+ "category": r["category"],
93
+ "content": r["content"],
94
+ "confidence": r["effective_confidence"],
95
+ }
96
+ for r in recalled
97
+ if r["effective_confidence"] >= 0.3
98
+ ]
99
+
100
+
101
+ __all__ = ["SessionMemory", "DEFAULT_MAX_FACTS", "DEFAULT_HALF_LIFE"]
102
+
@@ -0,0 +1,23 @@
1
+ """
2
+ SecAgentX ML 模型模块 — 基于机器学习的威胁检测引擎
3
+
4
+ 功能:
5
+ - 多算法支持: XGBoost / LightGBM / RandomForest / Ensemble
6
+ - SMOTE 过采样处理类别不平衡
7
+ - 贝叶斯超参数优化
8
+ - 决策阈值自动调优
9
+ - 模型持久化与版本管理
10
+ - 集成到 Agent 工具系统
11
+
12
+ 典型准确率 (NSL-KDD 测试集):
13
+ - XGBoost: ~87%
14
+ - LightGBM: ~86%
15
+ - RF + SMOTE: ~85%
16
+ - Ensemble: ~88%
17
+ """
18
+
19
+ from .trainer import MLTrainer
20
+ from .pipeline import MLPipeline
21
+
22
+ __all__ = ["MLTrainer", "MLPipeline"]
23
+
@@ -0,0 +1,191 @@
1
+ """
2
+ 数据流水线 — 从原始数据到训练就绪的特征矩阵
3
+
4
+ 支持:
5
+ - KDD Cup 99 / NSL-KDD 数据集(基于 DARPA 真实网络流量)
6
+ - 来源: https://www.unb.ca/cic/datasets/nsl.html
7
+ - 数据: 121,938 条网络流记录, 39 种攻击类型
8
+ - 特征: 41 个网络流特征 (协议/服务/标志/流量统计等)
9
+ - SMOTE 过采样
10
+ - 特征工程与标准化
11
+ - 混合编码 (数值标准化 + 类别独热)
12
+
13
+ 注意:
14
+ NSL-KDD 是 KDD Cup 99 的改进版本,解决了数据冗余和重复问题。
15
+ 虽然数据集较老(2009年发布),但它是网络安全 ML 领域最广泛使用的
16
+ 基准数据集之一,适合验证模型架构。生产环境建议使用更近期的数据集
17
+ (如 CIC-IDS2017、UNSW-NB15)重新训练。
18
+ """
19
+ import numpy as np
20
+ import pandas as pd
21
+ import os
22
+ import logging
23
+ from typing import Optional, Tuple
24
+ from sklearn.model_selection import train_test_split
25
+ from sklearn.preprocessing import StandardScaler, LabelEncoder
26
+ from imblearn.over_sampling import SMOTE
27
+
28
+ logger = logging.getLogger("secagentx.ml")
29
+
30
+ # ─── KDD Cup 99 特征定义 ───
31
+ KDD_FEATURE_NAMES = [
32
+ 'duration', 'protocol_type', 'service', 'flag',
33
+ 'src_bytes', 'dst_bytes', 'land', 'wrong_fragment', 'urgent',
34
+ 'hot', 'num_failed_logins', 'logged_in', 'num_compromised',
35
+ 'root_shell', 'su_attempted', 'num_root', 'num_file_creations',
36
+ 'num_shells', 'num_access_files', 'num_outbound_cmds',
37
+ 'is_host_login', 'is_guest_login', 'count', 'srv_count',
38
+ 'serror_rate', 'srv_serror_rate', 'rerror_rate', 'srv_rerror_rate',
39
+ 'same_srv_rate', 'diff_srv_rate', 'srv_diff_host_rate',
40
+ 'dst_host_count', 'dst_host_srv_count', 'dst_host_same_srv_rate',
41
+ 'dst_host_diff_srv_rate', 'dst_host_same_src_port_rate',
42
+ 'dst_host_srv_diff_host_rate', 'dst_host_serror_rate',
43
+ 'dst_host_srv_serror_rate', 'dst_host_rerror_rate',
44
+ 'dst_host_srv_rerror_rate', 'label'
45
+ ]
46
+
47
+ CATEGORICAL_FEATURES = ['protocol_type', 'service', 'flag']
48
+ NUMERIC_FEATURES = [c for c in KDD_FEATURE_NAMES if c not in CATEGORICAL_FEATURES + ['label']]
49
+
50
+
51
+ class MLPipeline:
52
+ """端到端数据流水线:加载 → 预处理 → 特征工程 → 划分 → 标准化 → SMOTE"""
53
+
54
+ def __init__(self, use_smote: bool = True, test_size: float = 0.25,
55
+ random_state: int = 42, scale_data: bool = True):
56
+ self.use_smote = use_smote
57
+ self.test_size = test_size
58
+ self.random_state = random_state
59
+ self.scale_data = scale_data
60
+ self.scaler: Optional[StandardScaler] = None
61
+ self.feature_names: list = []
62
+ self.label_encoder: Optional[LabelEncoder] = None
63
+ self.is_multiclass = False
64
+
65
+ # ─── 真实数据加载 ───
66
+ def load_nsl_kdd(self, train_path: str = 'dataset/KDDTrain.csv',
67
+ test_path: str = 'dataset/KDDTest.csv') -> pd.DataFrame:
68
+ """加载 NSL-KDD 真实数据集。"""
69
+ project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..'))
70
+ train_path = os.path.join(project_root, train_path) if not os.path.isabs(train_path) else train_path
71
+ test_path = os.path.join(project_root, test_path) if not os.path.isabs(test_path) else test_path
72
+
73
+ cols = KDD_FEATURE_NAMES[:41] + ['label', 'difficulty']
74
+ dfs = []
75
+ for path in [train_path, test_path]:
76
+ if os.path.exists(path):
77
+ df = pd.read_csv(path, header=None, names=cols, on_bad_lines='skip')
78
+ dfs.append(df)
79
+ logger.info(f"加载 NSL-KDD: {path} ({len(df)} 条)")
80
+
81
+ if not dfs:
82
+ raise FileNotFoundError(f"NSL-KDD 数据集未找到: {train_path} 或 {test_path}")
83
+
84
+ combined = pd.concat(dfs, ignore_index=True)
85
+ combined['label'] = combined['label'].astype(str).str.strip().str.lower().str.rstrip('.')
86
+ combined = combined.drop(columns=['difficulty'], errors='ignore')
87
+
88
+ attack_counts = combined[combined['label'] != 'normal']['label'].value_counts()
89
+ logger.info(f"NSL-KDD 加载完成: {len(combined)} 条, "
90
+ f"攻击类型: {len(attack_counts)} 种")
91
+ return combined
92
+
93
+ # ─── 预处理 ───
94
+ def preprocess(self, df: pd.DataFrame, binary: bool = True
95
+ ) -> Tuple[np.ndarray, np.ndarray]:
96
+ """完整预处理:编码 + 清洗。"""
97
+ if 'label' not in df.columns:
98
+ raise ValueError("数据缺少 'label' 列")
99
+
100
+ df = df.copy()
101
+
102
+ # 标签处理
103
+ self.is_multiclass = not binary
104
+ if binary:
105
+ y = df['label'].apply(lambda x: 0 if str(x).strip().lower().rstrip('.') == 'normal' else 1).values.astype(int)
106
+ else:
107
+ le = LabelEncoder()
108
+ y = le.fit_transform(df['label'].astype(str).str.strip().str.lower().str.rstrip('.'))
109
+ self.label_encoder = le
110
+
111
+ X_df = df.drop(columns=['label'])
112
+
113
+ # 类别特征独热编码
114
+ present_cats = [c for c in CATEGORICAL_FEATURES if c in X_df.columns]
115
+ X_df = pd.get_dummies(X_df, columns=present_cats, drop_first=False)
116
+
117
+ # 全部转为数值
118
+ for col in X_df.columns:
119
+ X_df[col] = pd.to_numeric(X_df[col], errors='coerce').fillna(0)
120
+
121
+ self.feature_names = X_df.columns.tolist()
122
+ X = X_df.values.astype(np.float64)
123
+
124
+ logger.info(f"预处理完成: {len(X)} 样本, {X.shape[1]} 特征, "
125
+ f"{'二分类' if binary else f'多分类({len(np.unique(y))}类)'}")
126
+ return X, y
127
+
128
+ def split_and_scale(self, X: np.ndarray, y: np.ndarray
129
+ ) -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]:
130
+ """划分训练/测试集 + 标准化 + SMOTE。"""
131
+ # 划分
132
+ stratify = y if len(np.unique(y)) > 1 else None
133
+ X_train, X_test, y_train, y_test = train_test_split(
134
+ X, y, test_size=self.test_size, random_state=self.random_state,
135
+ stratify=stratify
136
+ )
137
+
138
+ # 标准化
139
+ if self.scale_data:
140
+ self.scaler = StandardScaler()
141
+ X_train = self.scaler.fit_transform(X_train)
142
+ X_test = self.scaler.transform(X_test)
143
+
144
+ # SMOTE — 仅对二分类使用
145
+ if self.use_smote and not self.is_multiclass:
146
+ unique_train, counts_train = np.unique(y_train, return_counts=True)
147
+ if len(unique_train) > 1:
148
+ min_class_count = counts_train.min()
149
+ max_class_count = counts_train.max()
150
+ if min_class_count < max_class_count * 0.5:
151
+ k_neighbors = min(5, min_class_count - 1) if min_class_count > 1 else 1
152
+ if k_neighbors >= 1:
153
+ try:
154
+ smote = SMOTE(random_state=self.random_state, k_neighbors=k_neighbors)
155
+ X_train, y_train = smote.fit_resample(X_train, y_train)
156
+ logger.info(f"SMOTE 过采样后训练集: {X_train.shape[0]} 样本")
157
+ except Exception as e:
158
+ logger.warning(f"SMOTE 跳过: {e}")
159
+
160
+ return X_train, X_test, y_train, y_test
161
+
162
+ def run_pipeline(self, df: Optional[pd.DataFrame] = None,
163
+ n_samples: int = 10000, binary: bool = True
164
+ ) -> dict:
165
+ """一键执行完整流水线。"""
166
+ # 加载数据
167
+ if df is None:
168
+ try:
169
+ df = self.load_nsl_kdd()
170
+ logger.info("✅ 使用 NSL-KDD 真实数据集")
171
+ except FileNotFoundError as exc:
172
+ raise FileNotFoundError(
173
+ "未找到 NSL-KDD 真实数据集;已禁止自动生成合成训练数据。"
174
+ "请提供 dataset/KDDTrain.csv 和 dataset/KDDTest.csv,或显式传入真实 DataFrame。"
175
+ ) from exc
176
+
177
+ X, y = self.preprocess(df, binary=binary)
178
+ X_train, X_test, y_train, y_test = self.split_and_scale(X, y)
179
+
180
+ info = f"真实标注数据: {len(df)} 条"
181
+
182
+ return {
183
+ 'X_train': X_train, 'X_test': X_test,
184
+ 'y_train': y_train, 'y_test': y_test,
185
+ 'feature_names': self.feature_names,
186
+ 'scaler': self.scaler,
187
+ 'is_multiclass': self.is_multiclass,
188
+ 'label_encoder': self.label_encoder,
189
+ 'label_map': dict(enumerate(self.label_encoder.classes_)) if self.label_encoder else {},
190
+ 'info': info,
191
+ }