powermem 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- powermem/__init__.py +103 -0
- powermem/agent/__init__.py +35 -0
- powermem/agent/abstract/__init__.py +22 -0
- powermem/agent/abstract/collaboration.py +259 -0
- powermem/agent/abstract/context.py +187 -0
- powermem/agent/abstract/manager.py +232 -0
- powermem/agent/abstract/permission.py +217 -0
- powermem/agent/abstract/privacy.py +267 -0
- powermem/agent/abstract/scope.py +199 -0
- powermem/agent/agent.py +791 -0
- powermem/agent/components/__init__.py +18 -0
- powermem/agent/components/collaboration_coordinator.py +645 -0
- powermem/agent/components/permission_controller.py +586 -0
- powermem/agent/components/privacy_protector.py +767 -0
- powermem/agent/components/scope_controller.py +685 -0
- powermem/agent/factories/__init__.py +16 -0
- powermem/agent/factories/agent_factory.py +266 -0
- powermem/agent/factories/config_factory.py +308 -0
- powermem/agent/factories/memory_factory.py +229 -0
- powermem/agent/implementations/__init__.py +16 -0
- powermem/agent/implementations/hybrid.py +728 -0
- powermem/agent/implementations/multi_agent.py +1040 -0
- powermem/agent/implementations/multi_user.py +1020 -0
- powermem/agent/types.py +53 -0
- powermem/agent/wrappers/__init__.py +14 -0
- powermem/agent/wrappers/agent_memory_wrapper.py +427 -0
- powermem/agent/wrappers/compatibility_wrapper.py +520 -0
- powermem/config_loader.py +318 -0
- powermem/configs.py +249 -0
- powermem/core/__init__.py +19 -0
- powermem/core/async_memory.py +1493 -0
- powermem/core/audit.py +258 -0
- powermem/core/base.py +165 -0
- powermem/core/memory.py +1567 -0
- powermem/core/setup.py +162 -0
- powermem/core/telemetry.py +215 -0
- powermem/integrations/__init__.py +17 -0
- powermem/integrations/embeddings/__init__.py +13 -0
- powermem/integrations/embeddings/aws_bedrock.py +100 -0
- powermem/integrations/embeddings/azure_openai.py +55 -0
- powermem/integrations/embeddings/base.py +31 -0
- powermem/integrations/embeddings/config/base.py +132 -0
- powermem/integrations/embeddings/configs.py +31 -0
- powermem/integrations/embeddings/factory.py +48 -0
- powermem/integrations/embeddings/gemini.py +39 -0
- powermem/integrations/embeddings/huggingface.py +41 -0
- powermem/integrations/embeddings/langchain.py +35 -0
- powermem/integrations/embeddings/lmstudio.py +29 -0
- powermem/integrations/embeddings/mock.py +11 -0
- powermem/integrations/embeddings/ollama.py +53 -0
- powermem/integrations/embeddings/openai.py +49 -0
- powermem/integrations/embeddings/qwen.py +102 -0
- powermem/integrations/embeddings/together.py +31 -0
- powermem/integrations/embeddings/vertexai.py +54 -0
- powermem/integrations/llm/__init__.py +18 -0
- powermem/integrations/llm/anthropic.py +87 -0
- powermem/integrations/llm/base.py +132 -0
- powermem/integrations/llm/config/anthropic.py +56 -0
- powermem/integrations/llm/config/azure.py +56 -0
- powermem/integrations/llm/config/base.py +62 -0
- powermem/integrations/llm/config/deepseek.py +56 -0
- powermem/integrations/llm/config/ollama.py +56 -0
- powermem/integrations/llm/config/openai.py +79 -0
- powermem/integrations/llm/config/qwen.py +68 -0
- powermem/integrations/llm/config/qwen_asr.py +46 -0
- powermem/integrations/llm/config/vllm.py +56 -0
- powermem/integrations/llm/configs.py +26 -0
- powermem/integrations/llm/deepseek.py +106 -0
- powermem/integrations/llm/factory.py +118 -0
- powermem/integrations/llm/gemini.py +201 -0
- powermem/integrations/llm/langchain.py +65 -0
- powermem/integrations/llm/ollama.py +106 -0
- powermem/integrations/llm/openai.py +166 -0
- powermem/integrations/llm/openai_structured.py +80 -0
- powermem/integrations/llm/qwen.py +207 -0
- powermem/integrations/llm/qwen_asr.py +171 -0
- powermem/integrations/llm/vllm.py +106 -0
- powermem/integrations/rerank/__init__.py +20 -0
- powermem/integrations/rerank/base.py +43 -0
- powermem/integrations/rerank/config/__init__.py +7 -0
- powermem/integrations/rerank/config/base.py +27 -0
- powermem/integrations/rerank/configs.py +23 -0
- powermem/integrations/rerank/factory.py +68 -0
- powermem/integrations/rerank/qwen.py +159 -0
- powermem/intelligence/__init__.py +17 -0
- powermem/intelligence/ebbinghaus_algorithm.py +354 -0
- powermem/intelligence/importance_evaluator.py +361 -0
- powermem/intelligence/intelligent_memory_manager.py +284 -0
- powermem/intelligence/manager.py +148 -0
- powermem/intelligence/plugin.py +229 -0
- powermem/prompts/__init__.py +29 -0
- powermem/prompts/graph/graph_prompts.py +217 -0
- powermem/prompts/graph/graph_tools_prompts.py +469 -0
- powermem/prompts/importance_evaluation.py +246 -0
- powermem/prompts/intelligent_memory_prompts.py +163 -0
- powermem/prompts/templates.py +193 -0
- powermem/storage/__init__.py +14 -0
- powermem/storage/adapter.py +896 -0
- powermem/storage/base.py +109 -0
- powermem/storage/config/base.py +13 -0
- powermem/storage/config/oceanbase.py +58 -0
- powermem/storage/config/pgvector.py +52 -0
- powermem/storage/config/sqlite.py +27 -0
- powermem/storage/configs.py +159 -0
- powermem/storage/factory.py +59 -0
- powermem/storage/migration_manager.py +438 -0
- powermem/storage/oceanbase/__init__.py +8 -0
- powermem/storage/oceanbase/constants.py +162 -0
- powermem/storage/oceanbase/oceanbase.py +1384 -0
- powermem/storage/oceanbase/oceanbase_graph.py +1441 -0
- powermem/storage/pgvector/__init__.py +7 -0
- powermem/storage/pgvector/pgvector.py +420 -0
- powermem/storage/sqlite/__init__.py +0 -0
- powermem/storage/sqlite/sqlite.py +218 -0
- powermem/storage/sqlite/sqlite_vector_store.py +311 -0
- powermem/utils/__init__.py +35 -0
- powermem/utils/utils.py +605 -0
- powermem/version.py +23 -0
- powermem-0.1.0.dist-info/METADATA +187 -0
- powermem-0.1.0.dist-info/RECORD +123 -0
- powermem-0.1.0.dist-info/WHEEL +5 -0
- powermem-0.1.0.dist-info/licenses/LICENSE +206 -0
- powermem-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,1493 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Asynchronous memory management implementation
|
|
3
|
+
|
|
4
|
+
This module provides the asynchronous memory management interface.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
import asyncio
|
|
8
|
+
import logging
|
|
9
|
+
import hashlib
|
|
10
|
+
import json
|
|
11
|
+
from typing import Any, Dict, List, Optional, Union
|
|
12
|
+
from datetime import datetime
|
|
13
|
+
from copy import deepcopy
|
|
14
|
+
|
|
15
|
+
from .base import MemoryBase
|
|
16
|
+
from ..configs import MemoryConfig
|
|
17
|
+
from ..storage.factory import VectorStoreFactory, GraphStoreFactory
|
|
18
|
+
from ..storage.adapter import StorageAdapter, SubStorageAdapter
|
|
19
|
+
from ..intelligence.manager import IntelligenceManager
|
|
20
|
+
from ..integrations.llm.factory import LLMFactory
|
|
21
|
+
from ..integrations.embeddings.factory import EmbedderFactory
|
|
22
|
+
from .telemetry import TelemetryManager
|
|
23
|
+
from .audit import AuditLogger
|
|
24
|
+
from ..intelligence.plugin import IntelligentMemoryPlugin, EbbinghausIntelligencePlugin
|
|
25
|
+
from ..utils.utils import remove_code_blocks, convert_config_object_to_dict, parse_vision_messages
|
|
26
|
+
from ..prompts.intelligent_memory_prompts import (
|
|
27
|
+
FACT_RETRIEVAL_PROMPT,
|
|
28
|
+
FACT_EXTRACTION_PROMPT,
|
|
29
|
+
get_memory_update_prompt,
|
|
30
|
+
parse_messages_for_facts
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
logger = logging.getLogger(__name__)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _auto_convert_config(config: Dict[str, Any]) -> Dict[str, Any]:
|
|
37
|
+
"""
|
|
38
|
+
Convert legacy powermem config to format for compatibility.
|
|
39
|
+
|
|
40
|
+
Now powermem uses field names directly.
|
|
41
|
+
|
|
42
|
+
Args:
|
|
43
|
+
config: Configuration dictionary (legacy format)
|
|
44
|
+
|
|
45
|
+
Returns:
|
|
46
|
+
configuration dictionary
|
|
47
|
+
"""
|
|
48
|
+
if not config:
|
|
49
|
+
return config
|
|
50
|
+
|
|
51
|
+
# First, convert any ConfigObject instances to dicts
|
|
52
|
+
config = convert_config_object_to_dict(config)
|
|
53
|
+
|
|
54
|
+
# Check if legacy powermem format (has database or embedding)
|
|
55
|
+
if "database" in config or ("llm" in config and "embedding" in config):
|
|
56
|
+
converted = config.copy()
|
|
57
|
+
|
|
58
|
+
# Convert llm
|
|
59
|
+
if "llm" in config:
|
|
60
|
+
converted["llm"] = config["llm"]
|
|
61
|
+
|
|
62
|
+
# Convert embedding to embedder
|
|
63
|
+
if "embedding" in config:
|
|
64
|
+
converted["embedder"] = config["embedding"]
|
|
65
|
+
converted.pop("embedding", None)
|
|
66
|
+
|
|
67
|
+
# Convert database to vector_store
|
|
68
|
+
if "database" in config:
|
|
69
|
+
db_config = config["database"]
|
|
70
|
+
converted["vector_store"] = {
|
|
71
|
+
"provider": db_config.get("provider", "oceanbase"),
|
|
72
|
+
"config": db_config.get("config", {})
|
|
73
|
+
}
|
|
74
|
+
converted.pop("database", None)
|
|
75
|
+
elif "vector_store" not in converted:
|
|
76
|
+
converted["vector_store"] = {
|
|
77
|
+
"provider": "oceanbase",
|
|
78
|
+
"config": {}
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
logger.info("Converted legacy powermem config format")
|
|
82
|
+
return converted
|
|
83
|
+
|
|
84
|
+
return config
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
class AsyncMemory(MemoryBase):
|
|
88
|
+
"""
|
|
89
|
+
Asynchronous memory management implementation.
|
|
90
|
+
|
|
91
|
+
This class provides the main interface for asynchronous memory operations.
|
|
92
|
+
"""
|
|
93
|
+
|
|
94
|
+
def __init__(
|
|
95
|
+
self,
|
|
96
|
+
config: Optional[Dict[str, Any] | MemoryConfig] = None,
|
|
97
|
+
storage_type: Optional[str] = None,
|
|
98
|
+
llm_provider: Optional[str] = None,
|
|
99
|
+
embedding_provider: Optional[str] = None,
|
|
100
|
+
agent_id: Optional[str] = None,
|
|
101
|
+
):
|
|
102
|
+
"""
|
|
103
|
+
Initialize the async memory manager.
|
|
104
|
+
|
|
105
|
+
Compatible with both dict config and MemoryConfig object.
|
|
106
|
+
|
|
107
|
+
Args:
|
|
108
|
+
config: Configuration dictionary or MemoryConfig object containing all settings.
|
|
109
|
+
Dict format supports style (llm, embedder, vector_store)
|
|
110
|
+
and powermem style (database, llm, embedding)
|
|
111
|
+
storage_type: Type of storage backend to use (overrides config)
|
|
112
|
+
llm_provider: LLM provider to use (overrides config)
|
|
113
|
+
embedding_provider: Embedding provider to use (overrides config)
|
|
114
|
+
agent_id: Agent identifier for multi-agent scenarios
|
|
115
|
+
"""
|
|
116
|
+
# Handle MemoryConfig object or dict
|
|
117
|
+
if isinstance(config, MemoryConfig):
|
|
118
|
+
# Use MemoryConfig object directly
|
|
119
|
+
self.memory_config = config
|
|
120
|
+
# For backward compatibility, also store as dict
|
|
121
|
+
self.config = config.model_dump()
|
|
122
|
+
else:
|
|
123
|
+
# Convert dict config
|
|
124
|
+
dict_config = config or {}
|
|
125
|
+
dict_config = _auto_convert_config(dict_config)
|
|
126
|
+
self.config = dict_config
|
|
127
|
+
# Try to create MemoryConfig from dict, fallback to dict if fails
|
|
128
|
+
try:
|
|
129
|
+
self.memory_config = MemoryConfig(**dict_config)
|
|
130
|
+
except Exception as e:
|
|
131
|
+
logger.warning(f"Could not parse config as MemoryConfig: {e}, using dict mode")
|
|
132
|
+
self.memory_config = None
|
|
133
|
+
|
|
134
|
+
self.agent_id = agent_id
|
|
135
|
+
|
|
136
|
+
# Extract providers from config with fallbacks
|
|
137
|
+
self.storage_type = storage_type or self._get_provider('vector_store', 'oceanbase')
|
|
138
|
+
self.llm_provider = llm_provider or self._get_provider('llm', 'mock')
|
|
139
|
+
self.embedding_provider = embedding_provider or self._get_provider('embedder', 'mock')
|
|
140
|
+
|
|
141
|
+
# Initialize components
|
|
142
|
+
vector_store_config = self._get_component_config('vector_store')
|
|
143
|
+
vector_store = VectorStoreFactory.create(self.storage_type, vector_store_config)
|
|
144
|
+
|
|
145
|
+
# Extract graph_store config
|
|
146
|
+
self.enable_graph = self._get_graph_enabled()
|
|
147
|
+
self.graph_store = None
|
|
148
|
+
if self.enable_graph:
|
|
149
|
+
logger.debug("Graph store enabled")
|
|
150
|
+
graph_store_config = self.config.get("graph_store", {})
|
|
151
|
+
if graph_store_config:
|
|
152
|
+
provider = graph_store_config.get("provider", "oceanbase")
|
|
153
|
+
config_to_pass = self.memory_config if self.memory_config else self.config
|
|
154
|
+
self.graph_store = GraphStoreFactory.create(provider, config_to_pass)
|
|
155
|
+
|
|
156
|
+
# Extract LLM config
|
|
157
|
+
llm_config = self._get_component_config('llm')
|
|
158
|
+
self.llm = LLMFactory.create(self.llm_provider, llm_config)
|
|
159
|
+
|
|
160
|
+
# Extract embedder config
|
|
161
|
+
embedder_config = self._get_component_config('embedder')
|
|
162
|
+
self.embedding = EmbedderFactory.create(self.embedding_provider, embedder_config, None)
|
|
163
|
+
|
|
164
|
+
# Initialize storage adapter with embedding service
|
|
165
|
+
# Automatically select adapter based on sub_stores configuration
|
|
166
|
+
sub_stores_list = self.config.get('sub_stores', [])
|
|
167
|
+
if sub_stores_list and self.storage_type.lower() == 'oceanbase':
|
|
168
|
+
# Use SubStorageAdapter if sub stores are configured and using OceanBase
|
|
169
|
+
self.storage = SubStorageAdapter(vector_store, self.embedding)
|
|
170
|
+
logger.info("Using SubStorageAdapter with sub-store support")
|
|
171
|
+
else:
|
|
172
|
+
# Use basic StorageAdapter for single store operations
|
|
173
|
+
self.storage = StorageAdapter(vector_store, self.embedding)
|
|
174
|
+
logger.info("Using basic StorageAdapter")
|
|
175
|
+
|
|
176
|
+
self.intelligence = IntelligenceManager(self.config)
|
|
177
|
+
self.telemetry = TelemetryManager(self.config)
|
|
178
|
+
self.audit = AuditLogger(self.config)
|
|
179
|
+
|
|
180
|
+
# Save custom prompts from config
|
|
181
|
+
if self.memory_config:
|
|
182
|
+
self.custom_fact_extraction_prompt = self.memory_config.custom_fact_extraction_prompt
|
|
183
|
+
self.custom_update_memory_prompt = self.memory_config.custom_update_memory_prompt
|
|
184
|
+
else:
|
|
185
|
+
self.custom_fact_extraction_prompt = self.config.get('custom_fact_extraction_prompt')
|
|
186
|
+
self.custom_update_memory_prompt = self.config.get('custom_update_memory_prompt')
|
|
187
|
+
|
|
188
|
+
# Intelligent memory plugin (pluggable)
|
|
189
|
+
merged_cfg = self._get_intelligent_memory_config()
|
|
190
|
+
|
|
191
|
+
plugin_type = merged_cfg.get("plugin", "ebbinghaus")
|
|
192
|
+
self._intelligence_plugin: Optional[IntelligentMemoryPlugin] = None
|
|
193
|
+
if merged_cfg.get("enabled", False):
|
|
194
|
+
try:
|
|
195
|
+
if plugin_type == "ebbinghaus":
|
|
196
|
+
self._intelligence_plugin = EbbinghausIntelligencePlugin(merged_cfg)
|
|
197
|
+
else:
|
|
198
|
+
logger.warning(f"Unknown intelligence plugin: {plugin_type}")
|
|
199
|
+
except Exception as e:
|
|
200
|
+
logger.warning(f"Failed to initialize intelligence plugin (async): {e}")
|
|
201
|
+
self._intelligence_plugin = None
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
# Sub stores configuration (support multiple)
|
|
205
|
+
self.sub_stores_config: List[Dict] = []
|
|
206
|
+
|
|
207
|
+
# Initialize sub stores
|
|
208
|
+
self._init_sub_stores()
|
|
209
|
+
|
|
210
|
+
logger.info(f"AsyncMemory initialized with storage: {self.storage_type}, LLM: {self.llm_provider}, agent: {self.agent_id or 'default'}")
|
|
211
|
+
self.telemetry.capture_event("async_memory.init", {"storage_type": self.storage_type, "llm_provider": self.llm_provider, "agent_id": self.agent_id})
|
|
212
|
+
|
|
213
|
+
async def initialize(self):
|
|
214
|
+
"""Initialize async components."""
|
|
215
|
+
await self.storage.initialize_async()
|
|
216
|
+
|
|
217
|
+
def _get_provider(self, component: str, default: str) -> str:
|
|
218
|
+
"""
|
|
219
|
+
Helper method to get component provider uniformly.
|
|
220
|
+
|
|
221
|
+
Args:
|
|
222
|
+
component: Component name ('vector_store', 'llm', 'embedder')
|
|
223
|
+
default: Default provider name
|
|
224
|
+
|
|
225
|
+
Returns:
|
|
226
|
+
Provider name string
|
|
227
|
+
"""
|
|
228
|
+
if self.memory_config:
|
|
229
|
+
component_obj = getattr(self.memory_config, component, None)
|
|
230
|
+
return component_obj.provider if component_obj else default
|
|
231
|
+
else:
|
|
232
|
+
return self.config.get(component, {}).get('provider', default)
|
|
233
|
+
|
|
234
|
+
def _get_component_config(self, component: str) -> Dict[str, Any]:
|
|
235
|
+
"""
|
|
236
|
+
Helper method to get component configuration uniformly.
|
|
237
|
+
|
|
238
|
+
Args:
|
|
239
|
+
component: Component name ('vector_store', 'llm', 'embedder', 'graph_store')
|
|
240
|
+
|
|
241
|
+
Returns:
|
|
242
|
+
Component configuration dictionary
|
|
243
|
+
"""
|
|
244
|
+
if self.memory_config:
|
|
245
|
+
component_obj = getattr(self.memory_config, component, None)
|
|
246
|
+
return component_obj.config or {} if component_obj else {}
|
|
247
|
+
else:
|
|
248
|
+
return self.config.get(component, {}).get('config', {})
|
|
249
|
+
|
|
250
|
+
def _get_graph_enabled(self) -> bool:
|
|
251
|
+
"""
|
|
252
|
+
Helper method to get graph store enabled status.
|
|
253
|
+
|
|
254
|
+
Returns:
|
|
255
|
+
Boolean indicating whether graph store is enabled
|
|
256
|
+
"""
|
|
257
|
+
if self.memory_config:
|
|
258
|
+
return self.memory_config.graph_store.enabled if self.memory_config.graph_store else False
|
|
259
|
+
else:
|
|
260
|
+
graph_store_config = self.config.get('graph_store', {})
|
|
261
|
+
return graph_store_config.get('enabled', False) if graph_store_config else False
|
|
262
|
+
|
|
263
|
+
def _get_intelligent_memory_config(self) -> Dict[str, Any]:
|
|
264
|
+
"""
|
|
265
|
+
Helper method to get intelligent memory configuration.
|
|
266
|
+
Supports both "intelligence" and "intelligent_memory" config keys for backward compatibility.
|
|
267
|
+
|
|
268
|
+
Returns:
|
|
269
|
+
Merged intelligent memory configuration dictionary
|
|
270
|
+
"""
|
|
271
|
+
if self.memory_config and self.memory_config.intelligent_memory:
|
|
272
|
+
# Use MemoryConfig's intelligent_memory
|
|
273
|
+
cfg = self.memory_config.intelligent_memory.model_dump()
|
|
274
|
+
# Merge custom_importance_evaluation_prompt from top level if present
|
|
275
|
+
if self.memory_config.custom_importance_evaluation_prompt:
|
|
276
|
+
cfg["custom_importance_evaluation_prompt"] = self.memory_config.custom_importance_evaluation_prompt
|
|
277
|
+
return cfg
|
|
278
|
+
else:
|
|
279
|
+
# Fallback to dict access
|
|
280
|
+
intelligence_cfg = (self.config or {}).get("intelligence", {})
|
|
281
|
+
intelligent_memory_cfg = (self.config or {}).get("intelligent_memory", {})
|
|
282
|
+
merged_cfg = {**intelligence_cfg, **intelligent_memory_cfg}
|
|
283
|
+
# Merge custom_importance_evaluation_prompt from top level if present
|
|
284
|
+
if "custom_importance_evaluation_prompt" in self.config:
|
|
285
|
+
merged_cfg["custom_importance_evaluation_prompt"] = self.config["custom_importance_evaluation_prompt"]
|
|
286
|
+
return merged_cfg
|
|
287
|
+
|
|
288
|
+
async def _extract_facts(self, messages: Any) -> List[str]:
|
|
289
|
+
"""
|
|
290
|
+
Extract facts from messages using LLM asynchronously.
|
|
291
|
+
|
|
292
|
+
Args:
|
|
293
|
+
messages: Messages (list of dicts, single dict, or str)
|
|
294
|
+
|
|
295
|
+
Returns:
|
|
296
|
+
List of extracted facts
|
|
297
|
+
"""
|
|
298
|
+
try:
|
|
299
|
+
# Parse messages into conversation format
|
|
300
|
+
conversation = parse_messages_for_facts(messages)
|
|
301
|
+
|
|
302
|
+
# Use custom prompt if provided, otherwise use default
|
|
303
|
+
if self.custom_fact_extraction_prompt:
|
|
304
|
+
system_prompt = self.custom_fact_extraction_prompt
|
|
305
|
+
user_prompt = f"Input:\n{conversation}"
|
|
306
|
+
else:
|
|
307
|
+
system_prompt = FACT_RETRIEVAL_PROMPT
|
|
308
|
+
user_prompt = f"Input:\n{conversation}"
|
|
309
|
+
|
|
310
|
+
# Call LLM to extract facts asynchronously
|
|
311
|
+
try:
|
|
312
|
+
response = await asyncio.to_thread(
|
|
313
|
+
self.llm.generate_response,
|
|
314
|
+
messages=[
|
|
315
|
+
{"role": "system", "content": system_prompt},
|
|
316
|
+
{"role": "user", "content": user_prompt}
|
|
317
|
+
],
|
|
318
|
+
response_format={"type": "json_object"}
|
|
319
|
+
)
|
|
320
|
+
except Exception as e:
|
|
321
|
+
logger.error(f"Error in fact extraction: {e}")
|
|
322
|
+
response = ""
|
|
323
|
+
|
|
324
|
+
# Parse response
|
|
325
|
+
try:
|
|
326
|
+
# Remove code blocks if present (LLM sometimes wraps JSON in code blocks)
|
|
327
|
+
response = remove_code_blocks(response)
|
|
328
|
+
facts_data = json.loads(response)
|
|
329
|
+
facts = facts_data.get("facts", [])
|
|
330
|
+
logger.debug(f"Extracted {len(facts)} facts: {facts}")
|
|
331
|
+
return facts
|
|
332
|
+
except Exception as e:
|
|
333
|
+
logger.error(f"Error in new_retrieved_facts: {e}")
|
|
334
|
+
return []
|
|
335
|
+
|
|
336
|
+
except Exception as e:
|
|
337
|
+
logger.error(f"Error extracting facts: {e}")
|
|
338
|
+
return []
|
|
339
|
+
|
|
340
|
+
async def _decide_memory_actions(
|
|
341
|
+
self,
|
|
342
|
+
new_facts: List[str],
|
|
343
|
+
existing_memories: List[Dict[str, Any]],
|
|
344
|
+
user_id: Optional[str] = None,
|
|
345
|
+
agent_id: Optional[str] = None,
|
|
346
|
+
) -> List[Dict[str, Any]]:
|
|
347
|
+
"""
|
|
348
|
+
Use LLM to decide memory actions (ADD/UPDATE/DELETE/NONE) asynchronously.
|
|
349
|
+
|
|
350
|
+
Args:
|
|
351
|
+
new_facts: List of newly extracted facts
|
|
352
|
+
existing_memories: List of existing memories with 'id' and 'text'
|
|
353
|
+
user_id: User identifier
|
|
354
|
+
agent_id: Agent identifier
|
|
355
|
+
|
|
356
|
+
Returns:
|
|
357
|
+
List of memory action dictionaries
|
|
358
|
+
"""
|
|
359
|
+
try:
|
|
360
|
+
if not new_facts:
|
|
361
|
+
logger.debug("No new facts to process")
|
|
362
|
+
return []
|
|
363
|
+
|
|
364
|
+
# Format existing memories for prompt
|
|
365
|
+
old_memory = []
|
|
366
|
+
for mem in existing_memories:
|
|
367
|
+
# Support both "memory" and "content" field names for compatibility
|
|
368
|
+
content = mem.get("memory", "") or mem.get("content", "")
|
|
369
|
+
old_memory.append({
|
|
370
|
+
"id": mem.get("id", "unknown"),
|
|
371
|
+
"text": content
|
|
372
|
+
})
|
|
373
|
+
|
|
374
|
+
# Generate update prompt with custom prompt if provided
|
|
375
|
+
custom_prompt = None
|
|
376
|
+
if hasattr(self, 'custom_update_memory_prompt') and self.custom_update_memory_prompt:
|
|
377
|
+
custom_prompt = self.custom_update_memory_prompt
|
|
378
|
+
update_prompt = get_memory_update_prompt(old_memory, new_facts, custom_prompt)
|
|
379
|
+
|
|
380
|
+
# Call LLM asynchronously
|
|
381
|
+
try:
|
|
382
|
+
response = await asyncio.to_thread(
|
|
383
|
+
self.llm.generate_response,
|
|
384
|
+
messages=[{"role": "user", "content": update_prompt}],
|
|
385
|
+
response_format={"type": "json_object"}
|
|
386
|
+
)
|
|
387
|
+
except Exception as e:
|
|
388
|
+
logger.error(f"Error in new memory actions response: {e}")
|
|
389
|
+
response = ""
|
|
390
|
+
|
|
391
|
+
# Parse response
|
|
392
|
+
try:
|
|
393
|
+
response = remove_code_blocks(response)
|
|
394
|
+
actions_data = json.loads(response)
|
|
395
|
+
actions = actions_data.get("memory", [])
|
|
396
|
+
return actions
|
|
397
|
+
except Exception as e:
|
|
398
|
+
logger.error(f"Invalid JSON response: {e}")
|
|
399
|
+
return []
|
|
400
|
+
|
|
401
|
+
except Exception as e:
|
|
402
|
+
logger.error(f"Error deciding memory actions: {e}")
|
|
403
|
+
return []
|
|
404
|
+
|
|
405
|
+
async def add(
|
|
406
|
+
self,
|
|
407
|
+
messages,
|
|
408
|
+
user_id: Optional[str] = None,
|
|
409
|
+
agent_id: Optional[str] = None,
|
|
410
|
+
run_id: Optional[str] = None,
|
|
411
|
+
metadata: Optional[Dict[str, Any]] = None,
|
|
412
|
+
filters: Optional[Dict[str, Any]] = None,
|
|
413
|
+
scope: Optional[str] = None,
|
|
414
|
+
memory_type: Optional[str] = None,
|
|
415
|
+
prompt: Optional[str] = None,
|
|
416
|
+
infer: bool = True,
|
|
417
|
+
) -> Dict[str, Any]:
|
|
418
|
+
"""Add a new memory asynchronously with optional intelligent processing."""
|
|
419
|
+
try:
|
|
420
|
+
# Handle messages parameter
|
|
421
|
+
if messages is None:
|
|
422
|
+
raise ValueError("messages must be provided (str, dict, or list[dict])")
|
|
423
|
+
|
|
424
|
+
# Normalize input format
|
|
425
|
+
if isinstance(messages, str):
|
|
426
|
+
messages = [{"role": "user", "content": messages}]
|
|
427
|
+
elif isinstance(messages, dict):
|
|
428
|
+
messages = [messages]
|
|
429
|
+
elif not isinstance(messages, list):
|
|
430
|
+
raise ValueError("messages must be str, dict, or list[dict]")
|
|
431
|
+
|
|
432
|
+
# Vision-aware message processing
|
|
433
|
+
llm_cfg = {}
|
|
434
|
+
try:
|
|
435
|
+
llm_cfg = (self.config or {}).get("llm", {}).get("config", {})
|
|
436
|
+
except Exception:
|
|
437
|
+
llm_cfg = {}
|
|
438
|
+
if llm_cfg.get("enable_vision"):
|
|
439
|
+
messages = parse_vision_messages(messages, self.llm, llm_cfg.get("vision_details"))
|
|
440
|
+
else:
|
|
441
|
+
messages = parse_vision_messages(messages)
|
|
442
|
+
|
|
443
|
+
# Use self.agent_id as fallback if agent_id is not provided
|
|
444
|
+
agent_id = agent_id or self.agent_id
|
|
445
|
+
|
|
446
|
+
# Check if intelligent memory should be used
|
|
447
|
+
use_infer = infer and isinstance(messages, list) and len(messages) > 0
|
|
448
|
+
|
|
449
|
+
# If not using intelligent memory, fall back to simple mode
|
|
450
|
+
if not use_infer:
|
|
451
|
+
return await self._simple_add_async(messages, user_id, agent_id, run_id, metadata, filters, scope, memory_type, prompt)
|
|
452
|
+
|
|
453
|
+
# Intelligent memory mode: extract facts, search similar memories, and consolidate
|
|
454
|
+
return await self._intelligent_add_async(messages, user_id, agent_id, run_id, metadata, filters, scope, memory_type, prompt)
|
|
455
|
+
|
|
456
|
+
except Exception as e:
|
|
457
|
+
logger.error(f"Failed to add memory: {e}")
|
|
458
|
+
self.telemetry.capture_event("memory.add.error", {"error": str(e)})
|
|
459
|
+
raise
|
|
460
|
+
|
|
461
|
+
async def _simple_add_async(
|
|
462
|
+
self,
|
|
463
|
+
messages,
|
|
464
|
+
user_id: Optional[str] = None,
|
|
465
|
+
agent_id: Optional[str] = None,
|
|
466
|
+
run_id: Optional[str] = None,
|
|
467
|
+
metadata: Optional[Dict[str, Any]] = None,
|
|
468
|
+
filters: Optional[Dict[str, Any]] = None,
|
|
469
|
+
scope: Optional[str] = None,
|
|
470
|
+
memory_type: Optional[str] = None,
|
|
471
|
+
prompt: Optional[str] = None,
|
|
472
|
+
) -> Dict[str, Any]:
|
|
473
|
+
"""Simple add mode: direct storage without intelligence."""
|
|
474
|
+
# Parse messages into content
|
|
475
|
+
if isinstance(messages, str):
|
|
476
|
+
content = messages
|
|
477
|
+
elif isinstance(messages, dict):
|
|
478
|
+
content = messages.get("content", "")
|
|
479
|
+
elif isinstance(messages, list):
|
|
480
|
+
content = "\n".join([msg.get("content", "") for msg in messages if isinstance(msg, dict) and msg.get("content")])
|
|
481
|
+
else:
|
|
482
|
+
raise ValueError("messages must be str, dict, or list[dict]")
|
|
483
|
+
|
|
484
|
+
# Validate content is not empty
|
|
485
|
+
if not content or not content.strip():
|
|
486
|
+
logger.error(f"Cannot store empty content. Messages: {messages}")
|
|
487
|
+
raise ValueError(f"Cannot create memory with empty content. Original messages: {messages}")
|
|
488
|
+
|
|
489
|
+
# Select embedding service based on metadata (for sub-store routing)
|
|
490
|
+
embedding_service = self._get_embedding_service(metadata)
|
|
491
|
+
|
|
492
|
+
# Generate embedding asynchronously
|
|
493
|
+
embedding = await asyncio.to_thread(embedding_service.embed, content, memory_action="add")
|
|
494
|
+
|
|
495
|
+
# Disabled LLM-based importance evaluation to save tokens
|
|
496
|
+
# Process with intelligence manager
|
|
497
|
+
# enhanced_metadata = await self.intelligence.process_metadata_async(content, metadata)
|
|
498
|
+
enhanced_metadata = metadata # Use original metadata without LLM evaluation
|
|
499
|
+
|
|
500
|
+
# Intelligent plugin annotations
|
|
501
|
+
extra_fields = {}
|
|
502
|
+
if self._intelligence_plugin and self._intelligence_plugin.enabled:
|
|
503
|
+
extra_fields = self._intelligence_plugin.on_add(content=content, metadata=enhanced_metadata)
|
|
504
|
+
|
|
505
|
+
|
|
506
|
+
# Generate content hash for deduplication
|
|
507
|
+
content_hash = hashlib.md5(content.encode('utf-8')).hexdigest()
|
|
508
|
+
|
|
509
|
+
# Extract category from enhanced metadata if present
|
|
510
|
+
category = ""
|
|
511
|
+
if enhanced_metadata and isinstance(enhanced_metadata, dict):
|
|
512
|
+
category = enhanced_metadata.get("category", "")
|
|
513
|
+
# Remove category from metadata to avoid duplication
|
|
514
|
+
enhanced_metadata = {k: v for k, v in enhanced_metadata.items() if k != "category"}
|
|
515
|
+
|
|
516
|
+
# Final validation before storage
|
|
517
|
+
if not content or not content.strip():
|
|
518
|
+
raise ValueError(f"Refusing to store empty content. Original messages: {messages}")
|
|
519
|
+
|
|
520
|
+
# Use self.agent_id as fallback if agent_id is not provided
|
|
521
|
+
agent_id = agent_id or self.agent_id
|
|
522
|
+
|
|
523
|
+
# Store in database asynchronously
|
|
524
|
+
memory_data = {
|
|
525
|
+
"content": content,
|
|
526
|
+
"embedding": embedding,
|
|
527
|
+
"user_id": user_id,
|
|
528
|
+
"agent_id": agent_id,
|
|
529
|
+
"run_id": run_id,
|
|
530
|
+
"hash": content_hash,
|
|
531
|
+
"category": category,
|
|
532
|
+
"metadata": enhanced_metadata or {},
|
|
533
|
+
"filters": filters or {},
|
|
534
|
+
"created_at": datetime.utcnow(),
|
|
535
|
+
"updated_at": datetime.utcnow(),
|
|
536
|
+
}
|
|
537
|
+
|
|
538
|
+
if extra_fields:
|
|
539
|
+
memory_data.update(extra_fields)
|
|
540
|
+
|
|
541
|
+
memory_id = await self.storage.add_memory_async(memory_data)
|
|
542
|
+
|
|
543
|
+
# Log audit event
|
|
544
|
+
await self.audit.log_event_async("memory.add", {
|
|
545
|
+
"memory_id": memory_id,
|
|
546
|
+
"user_id": user_id,
|
|
547
|
+
"agent_id": agent_id,
|
|
548
|
+
"content_length": len(content)
|
|
549
|
+
}, user_id=user_id, agent_id=agent_id)
|
|
550
|
+
|
|
551
|
+
# Capture telemetry
|
|
552
|
+
self.telemetry.capture_event("memory.add", {
|
|
553
|
+
"memory_id": memory_id,
|
|
554
|
+
"user_id": user_id,
|
|
555
|
+
"agent_id": agent_id
|
|
556
|
+
})
|
|
557
|
+
|
|
558
|
+
# Add to graph store and get relations (only if graph store is enabled)
|
|
559
|
+
graph_result = None
|
|
560
|
+
if self.enable_graph:
|
|
561
|
+
graph_result = await self._add_to_graph_async(messages, filters, user_id, agent_id, run_id)
|
|
562
|
+
|
|
563
|
+
result: Dict[str, Any] = {
|
|
564
|
+
"results": [{
|
|
565
|
+
"id": memory_id,
|
|
566
|
+
"memory": content,
|
|
567
|
+
"event": "ADD",
|
|
568
|
+
"user_id": user_id,
|
|
569
|
+
"agent_id": agent_id,
|
|
570
|
+
"run_id": run_id,
|
|
571
|
+
"metadata": metadata,
|
|
572
|
+
"created_at": memory_data["created_at"].isoformat() if isinstance(memory_data["created_at"], datetime) else memory_data["created_at"],
|
|
573
|
+
}]
|
|
574
|
+
}
|
|
575
|
+
if graph_result:
|
|
576
|
+
result["relations"] = graph_result
|
|
577
|
+
return result
|
|
578
|
+
|
|
579
|
+
async def _intelligent_add_async(
|
|
580
|
+
self,
|
|
581
|
+
messages,
|
|
582
|
+
user_id: Optional[str] = None,
|
|
583
|
+
agent_id: Optional[str] = None,
|
|
584
|
+
run_id: Optional[str] = None,
|
|
585
|
+
metadata: Optional[Dict[str, Any]] = None,
|
|
586
|
+
filters: Optional[Dict[str, Any]] = None,
|
|
587
|
+
scope: Optional[str] = None,
|
|
588
|
+
memory_type: Optional[str] = None,
|
|
589
|
+
prompt: Optional[str] = None,
|
|
590
|
+
) -> Dict[str, Any]:
|
|
591
|
+
"""Intelligent add mode: extract facts, consolidate with existing memories."""
|
|
592
|
+
# Use self.agent_id as fallback if agent_id is not provided
|
|
593
|
+
agent_id = agent_id or self.agent_id
|
|
594
|
+
|
|
595
|
+
# Step 1: Extract facts from messages
|
|
596
|
+
logger.info("Extracting facts from messages...")
|
|
597
|
+
facts = await self._extract_facts(messages)
|
|
598
|
+
|
|
599
|
+
if not facts:
|
|
600
|
+
logger.debug("No facts extracted, skip intelligent add")
|
|
601
|
+
return {"results": []}
|
|
602
|
+
|
|
603
|
+
logger.info(f"Extracted {len(facts)} facts: {facts}")
|
|
604
|
+
|
|
605
|
+
# Step 2: Search for similar memories for each fact
|
|
606
|
+
existing_memories = []
|
|
607
|
+
fact_embeddings = {}
|
|
608
|
+
|
|
609
|
+
# Select embedding service based on metadata (for sub-store routing)
|
|
610
|
+
embedding_service = self._get_embedding_service(metadata)
|
|
611
|
+
|
|
612
|
+
for fact in facts:
|
|
613
|
+
fact_embedding = await asyncio.to_thread(embedding_service.embed, fact, memory_action="add")
|
|
614
|
+
fact_embeddings[fact] = fact_embedding
|
|
615
|
+
|
|
616
|
+
# Merge metadata into filters for correct routing
|
|
617
|
+
search_filters = filters.copy() if filters else {}
|
|
618
|
+
if metadata:
|
|
619
|
+
# Filter metadata to only include simple values (strings, numbers, booleans, None)
|
|
620
|
+
# This prevents nested dicts like {'agent': {'agent_id': ...}} from causing issues
|
|
621
|
+
# when OceanBase's build_condition tries to parse them as operators
|
|
622
|
+
simple_metadata = {
|
|
623
|
+
k: v for k, v in metadata.items()
|
|
624
|
+
if not isinstance(v, (dict, list)) and k not in ['agent_id', 'user_id', 'run_id']
|
|
625
|
+
}
|
|
626
|
+
search_filters.update(simple_metadata)
|
|
627
|
+
|
|
628
|
+
# Search for similar memories with reduced limit to reduce noise
|
|
629
|
+
# Pass fact text to enable hybrid search for better results
|
|
630
|
+
similar = await self.storage.search_memories_async(
|
|
631
|
+
query_embedding=fact_embedding,
|
|
632
|
+
user_id=user_id,
|
|
633
|
+
agent_id=agent_id,
|
|
634
|
+
run_id=run_id,
|
|
635
|
+
filters=search_filters,
|
|
636
|
+
limit=5,
|
|
637
|
+
query=fact # Enable hybrid search
|
|
638
|
+
)
|
|
639
|
+
existing_memories.extend(similar)
|
|
640
|
+
|
|
641
|
+
# Improved deduplication: prefer memories with better similarity scores
|
|
642
|
+
unique_memories = {}
|
|
643
|
+
for mem in existing_memories:
|
|
644
|
+
mem_id = mem.get("id")
|
|
645
|
+
if mem_id and mem_id not in unique_memories:
|
|
646
|
+
unique_memories[mem_id] = mem
|
|
647
|
+
elif mem_id:
|
|
648
|
+
# If duplicate ID, keep the one with better similarity (lower distance)
|
|
649
|
+
existing = unique_memories.get(mem_id)
|
|
650
|
+
mem_distance = mem.get("distance", float('inf'))
|
|
651
|
+
existing_distance = existing.get("distance", float('inf')) if existing else float('inf')
|
|
652
|
+
if mem_distance < existing_distance:
|
|
653
|
+
unique_memories[mem_id] = mem
|
|
654
|
+
|
|
655
|
+
# Limit candidates to avoid LLM prompt overload
|
|
656
|
+
existing_memories = list(unique_memories.values())[:10] # Max 10 memories
|
|
657
|
+
|
|
658
|
+
logger.info(f"Found {len(existing_memories)} existing memories to consider (after dedup and limiting)")
|
|
659
|
+
|
|
660
|
+
# Mapping IDs with integers for handling ID hallucinations
|
|
661
|
+
# Maps temporary string indices to real Snowflake IDs (integers)
|
|
662
|
+
temp_uuid_mapping = {}
|
|
663
|
+
for idx, item in enumerate(existing_memories):
|
|
664
|
+
temp_uuid_mapping[str(idx)] = item["id"]
|
|
665
|
+
existing_memories[idx]["id"] = str(idx)
|
|
666
|
+
|
|
667
|
+
# Step 3: Let LLM decide memory actions (only if we have new facts)
|
|
668
|
+
actions = []
|
|
669
|
+
if facts:
|
|
670
|
+
actions = await self._decide_memory_actions(facts, existing_memories, user_id, agent_id)
|
|
671
|
+
logger.info(f"LLM decided on {len(actions)} memory actions")
|
|
672
|
+
else:
|
|
673
|
+
logger.debug("No new facts, skipping LLM decision step")
|
|
674
|
+
|
|
675
|
+
# Step 4: Execute actions
|
|
676
|
+
results = []
|
|
677
|
+
action_counts = {"ADD": 0, "UPDATE": 0, "DELETE": 0, "NONE": 0}
|
|
678
|
+
|
|
679
|
+
if not actions:
|
|
680
|
+
logger.warning("No actions returned from LLM, skip intelligent add")
|
|
681
|
+
return {"results": []}
|
|
682
|
+
|
|
683
|
+
for action in actions:
|
|
684
|
+
action_text = action.get("text", "") or action.get("memory", "")
|
|
685
|
+
event_type = action.get("event", "NONE")
|
|
686
|
+
action_id = action.get("id", "")
|
|
687
|
+
|
|
688
|
+
# Skip actions with empty text UNLESS it's a NONE event (duplicates may have empty text)
|
|
689
|
+
if not action_text and event_type != "NONE":
|
|
690
|
+
logger.warning(f"Skipping action with empty text: {action}")
|
|
691
|
+
continue
|
|
692
|
+
|
|
693
|
+
logger.debug(f"Processing action: {event_type} - '{action_text[:50] if action_text else 'NONE'}...' (id: {action_id})")
|
|
694
|
+
|
|
695
|
+
try:
|
|
696
|
+
if event_type == "ADD":
|
|
697
|
+
# Add new memory
|
|
698
|
+
memory_id = await self._create_memory_async(
|
|
699
|
+
content=action_text,
|
|
700
|
+
user_id=user_id,
|
|
701
|
+
agent_id=agent_id,
|
|
702
|
+
run_id=run_id,
|
|
703
|
+
metadata=metadata,
|
|
704
|
+
filters=filters,
|
|
705
|
+
existing_embeddings=fact_embeddings
|
|
706
|
+
)
|
|
707
|
+
results.append({
|
|
708
|
+
"id": memory_id,
|
|
709
|
+
"memory": action_text,
|
|
710
|
+
"event": event_type
|
|
711
|
+
})
|
|
712
|
+
action_counts["ADD"] += 1
|
|
713
|
+
|
|
714
|
+
elif event_type == "UPDATE":
|
|
715
|
+
# Use ID mapping to get the real memory ID (Snowflake ID - integer)
|
|
716
|
+
real_memory_id = temp_uuid_mapping.get(str(action_id))
|
|
717
|
+
if real_memory_id:
|
|
718
|
+
await self._update_memory_async(
|
|
719
|
+
memory_id=real_memory_id,
|
|
720
|
+
content=action_text,
|
|
721
|
+
user_id=user_id,
|
|
722
|
+
agent_id=agent_id,
|
|
723
|
+
existing_embeddings=fact_embeddings,
|
|
724
|
+
metadata=metadata
|
|
725
|
+
)
|
|
726
|
+
results.append({
|
|
727
|
+
"id": real_memory_id,
|
|
728
|
+
"memory": action_text,
|
|
729
|
+
"event": event_type,
|
|
730
|
+
"previous_memory": action.get("old_memory")
|
|
731
|
+
})
|
|
732
|
+
action_counts["UPDATE"] += 1
|
|
733
|
+
else:
|
|
734
|
+
logger.warning(f"Could not find real memory ID for action ID: {action_id}")
|
|
735
|
+
|
|
736
|
+
elif event_type == "DELETE":
|
|
737
|
+
# Use ID mapping to get the real memory ID (Snowflake ID - integer)
|
|
738
|
+
real_memory_id = temp_uuid_mapping.get(str(action_id))
|
|
739
|
+
if real_memory_id:
|
|
740
|
+
await self.delete(real_memory_id, user_id, agent_id)
|
|
741
|
+
results.append({
|
|
742
|
+
"id": real_memory_id,
|
|
743
|
+
"memory": action_text,
|
|
744
|
+
"event": event_type
|
|
745
|
+
})
|
|
746
|
+
action_counts["DELETE"] += 1
|
|
747
|
+
else:
|
|
748
|
+
logger.warning(f"Could not find real memory ID for action ID: {action_id}")
|
|
749
|
+
|
|
750
|
+
elif event_type == "NONE":
|
|
751
|
+
logger.debug("No action needed for memory (duplicate detected)")
|
|
752
|
+
action_counts["NONE"] += 1
|
|
753
|
+
|
|
754
|
+
except Exception as e:
|
|
755
|
+
logger.error(f"Error executing memory action {event_type}: {e}")
|
|
756
|
+
|
|
757
|
+
# Log audit event for intelligent add operation
|
|
758
|
+
await self.audit.log_event_async("memory.intelligent_add", {
|
|
759
|
+
"user_id": user_id,
|
|
760
|
+
"agent_id": agent_id,
|
|
761
|
+
"facts_count": len(facts),
|
|
762
|
+
"action_counts": action_counts,
|
|
763
|
+
"results_count": len(results)
|
|
764
|
+
}, user_id=user_id, agent_id=agent_id)
|
|
765
|
+
|
|
766
|
+
# Add to graph store and get relations (only if graph store is enabled)
|
|
767
|
+
graph_result = None
|
|
768
|
+
if self.enable_graph:
|
|
769
|
+
graph_result = await self._add_to_graph_async(messages, filters, user_id, agent_id, run_id)
|
|
770
|
+
|
|
771
|
+
# API format: {"results": [...]}
|
|
772
|
+
if results:
|
|
773
|
+
result: Dict[str, Any] = {"results": results}
|
|
774
|
+
if graph_result:
|
|
775
|
+
result["relations"] = graph_result
|
|
776
|
+
return result
|
|
777
|
+
# If we processed actions but they were all NONE (duplicates detected), return empty results
|
|
778
|
+
elif action_counts.get("NONE", 0) > 0:
|
|
779
|
+
logger.info(f"All actions were NONE (duplicates detected), returning empty results")
|
|
780
|
+
result: Dict[str, Any] = {"results": []}
|
|
781
|
+
if graph_result:
|
|
782
|
+
result["relations"] = graph_result
|
|
783
|
+
return result
|
|
784
|
+
# Return [] if we had no actions at all
|
|
785
|
+
else:
|
|
786
|
+
logger.warning("No actions returned from LLM, skip intelligent add")
|
|
787
|
+
return {"results": []}
|
|
788
|
+
|
|
789
|
+
async def _add_to_graph_async(
|
|
790
|
+
self,
|
|
791
|
+
messages,
|
|
792
|
+
filters: Optional[Dict[str, Any]] = None,
|
|
793
|
+
user_id: Optional[str] = None,
|
|
794
|
+
agent_id: Optional[str] = None,
|
|
795
|
+
run_id: Optional[str] = None,
|
|
796
|
+
) -> Optional[Dict[str, Any]]:
|
|
797
|
+
"""
|
|
798
|
+
Add messages to graph store and return relations asynchronously.
|
|
799
|
+
|
|
800
|
+
Returns:
|
|
801
|
+
dict with added_entities and deleted_entities, or None if graph store is disabled
|
|
802
|
+
"""
|
|
803
|
+
if not self.enable_graph:
|
|
804
|
+
return None
|
|
805
|
+
|
|
806
|
+
# Extract content from messages for graph processing
|
|
807
|
+
if isinstance(messages, str):
|
|
808
|
+
data = messages
|
|
809
|
+
elif isinstance(messages, dict):
|
|
810
|
+
data = messages.get("content", "")
|
|
811
|
+
elif isinstance(messages, list):
|
|
812
|
+
data = "\n".join([
|
|
813
|
+
msg.get("content", "")
|
|
814
|
+
for msg in messages
|
|
815
|
+
if isinstance(msg, dict) and msg.get("content") and msg.get("role") != "system"
|
|
816
|
+
])
|
|
817
|
+
else:
|
|
818
|
+
data = ""
|
|
819
|
+
|
|
820
|
+
if not data:
|
|
821
|
+
return None
|
|
822
|
+
|
|
823
|
+
graph_filters = {**(filters or {}), "user_id": user_id, "agent_id": agent_id, "run_id": run_id}
|
|
824
|
+
if graph_filters.get("user_id") is None:
|
|
825
|
+
graph_filters["user_id"] = "user"
|
|
826
|
+
|
|
827
|
+
return self.graph_store.add(data, graph_filters)
|
|
828
|
+
|
|
829
|
+
async def _create_memory_async(
|
|
830
|
+
self,
|
|
831
|
+
content: str,
|
|
832
|
+
user_id: Optional[str] = None,
|
|
833
|
+
agent_id: Optional[str] = None,
|
|
834
|
+
run_id: Optional[str] = None,
|
|
835
|
+
metadata: Optional[Dict[str, Any]] = None,
|
|
836
|
+
filters: Optional[Dict[str, Any]] = None,
|
|
837
|
+
existing_embeddings: Optional[Dict[str, Any]] = None,
|
|
838
|
+
) -> int:
|
|
839
|
+
"""Create a memory asynchronously with optional embeddings."""
|
|
840
|
+
# Validate content is not empty
|
|
841
|
+
if not content or not content.strip():
|
|
842
|
+
raise ValueError(f"Cannot create memory with empty content: '{content}'")
|
|
843
|
+
|
|
844
|
+
# Select embedding service based on metadata (for sub-store routing)
|
|
845
|
+
embedding_service = self._get_embedding_service(metadata)
|
|
846
|
+
|
|
847
|
+
# Generate or use existing embedding
|
|
848
|
+
if existing_embeddings and content in existing_embeddings:
|
|
849
|
+
embedding = existing_embeddings[content]
|
|
850
|
+
else:
|
|
851
|
+
embedding = await asyncio.to_thread(embedding_service.embed, content, memory_action="add")
|
|
852
|
+
|
|
853
|
+
# Disabled LLM-based importance evaluation to save tokens
|
|
854
|
+
# Process metadata
|
|
855
|
+
# enhanced_metadata = await self.intelligence.process_metadata_async(content, metadata)
|
|
856
|
+
enhanced_metadata = metadata # Use original metadata without LLM evaluation
|
|
857
|
+
|
|
858
|
+
# Generate content hash
|
|
859
|
+
content_hash = hashlib.md5(content.encode('utf-8')).hexdigest()
|
|
860
|
+
|
|
861
|
+
# Extract category
|
|
862
|
+
category = ""
|
|
863
|
+
if enhanced_metadata and isinstance(enhanced_metadata, dict):
|
|
864
|
+
category = enhanced_metadata.get("category", "")
|
|
865
|
+
enhanced_metadata = {k: v for k, v in enhanced_metadata.items() if k != "category"}
|
|
866
|
+
|
|
867
|
+
# Use self.agent_id as fallback if agent_id is not provided
|
|
868
|
+
agent_id = agent_id or self.agent_id
|
|
869
|
+
|
|
870
|
+
# Create memory data
|
|
871
|
+
memory_data = {
|
|
872
|
+
"content": content,
|
|
873
|
+
"embedding": embedding,
|
|
874
|
+
"user_id": user_id,
|
|
875
|
+
"agent_id": agent_id,
|
|
876
|
+
"run_id": run_id,
|
|
877
|
+
"hash": content_hash,
|
|
878
|
+
"category": category,
|
|
879
|
+
"metadata": enhanced_metadata or {},
|
|
880
|
+
"filters": filters or {},
|
|
881
|
+
"created_at": datetime.utcnow(),
|
|
882
|
+
"updated_at": datetime.utcnow(),
|
|
883
|
+
}
|
|
884
|
+
|
|
885
|
+
memory_id = await self.storage.add_memory_async(memory_data)
|
|
886
|
+
|
|
887
|
+
return memory_id
|
|
888
|
+
|
|
889
|
+
async def _update_memory_async(
|
|
890
|
+
self,
|
|
891
|
+
memory_id: int,
|
|
892
|
+
content: str,
|
|
893
|
+
user_id: Optional[str] = None,
|
|
894
|
+
agent_id: Optional[str] = None,
|
|
895
|
+
existing_embeddings: Optional[Dict[str, Any]] = None,
|
|
896
|
+
metadata: Optional[Dict[str, Any]] = None,
|
|
897
|
+
):
|
|
898
|
+
"""Update a memory asynchronously with optional embeddings."""
|
|
899
|
+
# Use self.agent_id as fallback if agent_id is not provided
|
|
900
|
+
agent_id = agent_id or self.agent_id
|
|
901
|
+
|
|
902
|
+
# Validate content is not empty
|
|
903
|
+
if not content or not content.strip():
|
|
904
|
+
raise ValueError(f"Cannot update memory with empty content: '{content}'")
|
|
905
|
+
|
|
906
|
+
# Generate or use existing embedding
|
|
907
|
+
if existing_embeddings and content in existing_embeddings:
|
|
908
|
+
embedding = existing_embeddings[content]
|
|
909
|
+
else:
|
|
910
|
+
# If no metadata provided, try to get existing memory's metadata
|
|
911
|
+
if metadata is None:
|
|
912
|
+
existing = await self.storage.get_memory_async(memory_id, user_id, agent_id)
|
|
913
|
+
if existing:
|
|
914
|
+
metadata = existing.get("metadata", {})
|
|
915
|
+
|
|
916
|
+
# Select embedding service based on metadata (for sub-store routing)
|
|
917
|
+
embedding_service = self._get_embedding_service(metadata)
|
|
918
|
+
|
|
919
|
+
embedding = await asyncio.to_thread(embedding_service.embed, content, memory_action="update")
|
|
920
|
+
|
|
921
|
+
# Generate content hash
|
|
922
|
+
content_hash = hashlib.md5(content.encode('utf-8')).hexdigest()
|
|
923
|
+
|
|
924
|
+
update_data = {
|
|
925
|
+
"content": content,
|
|
926
|
+
"embedding": embedding,
|
|
927
|
+
"hash": content_hash, # Update hash
|
|
928
|
+
"updated_at": datetime.utcnow(),
|
|
929
|
+
}
|
|
930
|
+
|
|
931
|
+
logger.debug(f"Updating memory {memory_id} with content: '{content[:50]}...'")
|
|
932
|
+
|
|
933
|
+
await self.storage.update_memory_async(memory_id, update_data, user_id, agent_id)
|
|
934
|
+
|
|
935
|
+
async def search(
|
|
936
|
+
self,
|
|
937
|
+
query: str,
|
|
938
|
+
user_id: Optional[str] = None,
|
|
939
|
+
agent_id: Optional[str] = None,
|
|
940
|
+
run_id: Optional[str] = None,
|
|
941
|
+
filters: Optional[Dict[str, Any]] = None,
|
|
942
|
+
limit: int = 30,
|
|
943
|
+
threshold: Optional[float] = None,
|
|
944
|
+
) -> Dict[str, Any]:
|
|
945
|
+
"""Search for memories asynchronously."""
|
|
946
|
+
try:
|
|
947
|
+
# Select embedding service based on filters (for sub-store routing)
|
|
948
|
+
embedding_service = self._get_embedding_service(filters)
|
|
949
|
+
|
|
950
|
+
# Generate query embedding asynchronously
|
|
951
|
+
query_embedding = await asyncio.to_thread(embedding_service.embed, query, memory_action="search")
|
|
952
|
+
|
|
953
|
+
|
|
954
|
+
# Search in storage asynchronously - pass query text to enable hybrid search
|
|
955
|
+
results = await self.storage.search_memories_async(
|
|
956
|
+
query_embedding=query_embedding,
|
|
957
|
+
user_id=user_id,
|
|
958
|
+
agent_id=agent_id,
|
|
959
|
+
run_id=run_id,
|
|
960
|
+
filters=filters,
|
|
961
|
+
limit=limit,
|
|
962
|
+
query=query # Pass query text for hybrid search (vector + full-text)
|
|
963
|
+
)
|
|
964
|
+
|
|
965
|
+
# Process results with intelligence manager (only if enabled to avoid unnecessary calls)
|
|
966
|
+
if self.intelligence.enabled:
|
|
967
|
+
processed_results = await self.intelligence.process_search_results_async(results, query)
|
|
968
|
+
else:
|
|
969
|
+
processed_results = results
|
|
970
|
+
|
|
971
|
+
# Intelligent plugin lifecycle management on search
|
|
972
|
+
if self._intelligence_plugin and self._intelligence_plugin.enabled:
|
|
973
|
+
updates, deletes = self._intelligence_plugin.on_search(processed_results)
|
|
974
|
+
for mem_id, upd in updates:
|
|
975
|
+
try:
|
|
976
|
+
await self.storage.update_memory_async(mem_id, {**upd}, user_id, agent_id)
|
|
977
|
+
except Exception:
|
|
978
|
+
continue
|
|
979
|
+
for mem_id in deletes:
|
|
980
|
+
try:
|
|
981
|
+
await self.storage.delete_memory_async(mem_id, user_id, agent_id)
|
|
982
|
+
except Exception:
|
|
983
|
+
continue
|
|
984
|
+
|
|
985
|
+
# Transform results to match benchmark expected format
|
|
986
|
+
# Benchmark expects: {"results": [{"memory": ..., "metadata": {...}, "score": ...}], "relations": [...]}
|
|
987
|
+
transformed_results = []
|
|
988
|
+
for result in processed_results:
|
|
989
|
+
score = result.get("score", 0.0)
|
|
990
|
+
# Apply threshold filtering
|
|
991
|
+
# Only include results if threshold is None or score >= threshold
|
|
992
|
+
if threshold is not None and score < threshold:
|
|
993
|
+
continue
|
|
994
|
+
|
|
995
|
+
transformed_result = {
|
|
996
|
+
"memory": result.get("memory", ""),
|
|
997
|
+
"metadata": result.get("metadata", {}), # Keep metadata as-is from storage
|
|
998
|
+
"score": score,
|
|
999
|
+
}
|
|
1000
|
+
# Preserve other fields if needed
|
|
1001
|
+
for key in ["id", "created_at", "updated_at", "user_id", "agent_id", "run_id"]:
|
|
1002
|
+
if key in result:
|
|
1003
|
+
transformed_result[key] = result[key]
|
|
1004
|
+
transformed_results.append(transformed_result)
|
|
1005
|
+
|
|
1006
|
+
# Log audit event
|
|
1007
|
+
await self.audit.log_event_async("memory.search", {
|
|
1008
|
+
"query": query,
|
|
1009
|
+
"user_id": user_id,
|
|
1010
|
+
"agent_id": agent_id,
|
|
1011
|
+
"results_count": len(transformed_results)
|
|
1012
|
+
}, user_id=user_id, agent_id=agent_id)
|
|
1013
|
+
|
|
1014
|
+
# Capture telemetry
|
|
1015
|
+
self.telemetry.capture_event("memory.search", {
|
|
1016
|
+
"user_id": user_id,
|
|
1017
|
+
"agent_id": agent_id,
|
|
1018
|
+
"results_count": len(transformed_results),
|
|
1019
|
+
"threshold": threshold
|
|
1020
|
+
})
|
|
1021
|
+
|
|
1022
|
+
# Search in graph store
|
|
1023
|
+
if self.enable_graph:
|
|
1024
|
+
filters = {**(filters or {}), "user_id": user_id, "agent_id": agent_id, "run_id": run_id}
|
|
1025
|
+
graph_results = await asyncio.to_thread(self.graph_store.search, query, filters, limit)
|
|
1026
|
+
return {"results": transformed_results, "relations": graph_results}
|
|
1027
|
+
|
|
1028
|
+
# Return in benchmark expected format
|
|
1029
|
+
return {"results": transformed_results}
|
|
1030
|
+
|
|
1031
|
+
except Exception as e:
|
|
1032
|
+
logger.error(f"Failed to search memories: {e}")
|
|
1033
|
+
self.telemetry.capture_event("memory.search.error", {"error": str(e)})
|
|
1034
|
+
raise
|
|
1035
|
+
|
|
1036
|
+
async def get(
|
|
1037
|
+
self,
|
|
1038
|
+
memory_id: int,
|
|
1039
|
+
user_id: Optional[str] = None,
|
|
1040
|
+
agent_id: Optional[str] = None,
|
|
1041
|
+
) -> Optional[Dict[str, Any]]:
|
|
1042
|
+
"""Get a specific memory by ID asynchronously."""
|
|
1043
|
+
try:
|
|
1044
|
+
result = await self.storage.get_memory_async(memory_id, user_id, agent_id)
|
|
1045
|
+
|
|
1046
|
+
if result:
|
|
1047
|
+
if self._intelligence_plugin and self._intelligence_plugin.enabled:
|
|
1048
|
+
updates, delete_flag = self._intelligence_plugin.on_get(result)
|
|
1049
|
+
try:
|
|
1050
|
+
if delete_flag:
|
|
1051
|
+
await self.storage.delete_memory_async(memory_id, user_id, agent_id)
|
|
1052
|
+
return None
|
|
1053
|
+
if updates:
|
|
1054
|
+
await self.storage.update_memory_async(memory_id, {**updates}, user_id, agent_id)
|
|
1055
|
+
except Exception:
|
|
1056
|
+
pass
|
|
1057
|
+
await self.audit.log_event_async("memory.get", {
|
|
1058
|
+
"memory_id": memory_id,
|
|
1059
|
+
"user_id": user_id,
|
|
1060
|
+
"agent_id": agent_id
|
|
1061
|
+
}, user_id=user_id, agent_id=agent_id)
|
|
1062
|
+
|
|
1063
|
+
return result
|
|
1064
|
+
|
|
1065
|
+
except Exception as e:
|
|
1066
|
+
logger.error(f"Failed to get memory {memory_id}: {e}")
|
|
1067
|
+
raise
|
|
1068
|
+
|
|
1069
|
+
async def update(
|
|
1070
|
+
self,
|
|
1071
|
+
memory_id: int,
|
|
1072
|
+
content: str,
|
|
1073
|
+
user_id: Optional[str] = None,
|
|
1074
|
+
agent_id: Optional[str] = None,
|
|
1075
|
+
metadata: Optional[Dict[str, Any]] = None,
|
|
1076
|
+
) -> Dict[str, Any]:
|
|
1077
|
+
"""Update an existing memory asynchronously."""
|
|
1078
|
+
try:
|
|
1079
|
+
# Validate content is not empty
|
|
1080
|
+
if not content or not content.strip():
|
|
1081
|
+
raise ValueError(f"Cannot update memory with empty content: '{content}'")
|
|
1082
|
+
|
|
1083
|
+
# If no metadata provided, try to get existing memory's metadata
|
|
1084
|
+
if metadata is None:
|
|
1085
|
+
existing = await self.storage.get_memory_async(memory_id, user_id, agent_id)
|
|
1086
|
+
if existing:
|
|
1087
|
+
metadata = existing.get("metadata", {})
|
|
1088
|
+
|
|
1089
|
+
# Select embedding service based on metadata (for sub-store routing)
|
|
1090
|
+
embedding_service = self._get_embedding_service(metadata)
|
|
1091
|
+
|
|
1092
|
+
# Generate new embedding asynchronously
|
|
1093
|
+
embedding = await asyncio.to_thread(embedding_service.embed, content, memory_action="update")
|
|
1094
|
+
|
|
1095
|
+
# Process metadata with intelligence manager (if enabled)
|
|
1096
|
+
# Disabled LLM-based importance evaluation to save tokens (consistent with add method)
|
|
1097
|
+
# enhanced_metadata = await self.intelligence.process_metadata_async(content, metadata)
|
|
1098
|
+
enhanced_metadata = metadata # Use original metadata without LLM evaluation
|
|
1099
|
+
|
|
1100
|
+
# Intelligent plugin annotations
|
|
1101
|
+
extra_fields = {}
|
|
1102
|
+
if self._intelligence_plugin and self._intelligence_plugin.enabled:
|
|
1103
|
+
# Get existing memory for context
|
|
1104
|
+
existing_memory = await self.get(memory_id, user_id=user_id)
|
|
1105
|
+
if existing_memory:
|
|
1106
|
+
# Plugin can process update event
|
|
1107
|
+
extra_fields = self._intelligence_plugin.on_add(content=content, metadata=enhanced_metadata)
|
|
1108
|
+
|
|
1109
|
+
# Generate content hash for deduplication
|
|
1110
|
+
content_hash = hashlib.md5(content.encode('utf-8')).hexdigest()
|
|
1111
|
+
|
|
1112
|
+
# Extract category from enhanced metadata if present
|
|
1113
|
+
category = ""
|
|
1114
|
+
if enhanced_metadata and isinstance(enhanced_metadata, dict):
|
|
1115
|
+
category = enhanced_metadata.get("category", "")
|
|
1116
|
+
# Remove category from metadata to avoid duplication
|
|
1117
|
+
enhanced_metadata = {k: v for k, v in enhanced_metadata.items() if k != "category"}
|
|
1118
|
+
|
|
1119
|
+
# Merge extra fields from intelligence plugin
|
|
1120
|
+
if extra_fields and isinstance(extra_fields, dict):
|
|
1121
|
+
enhanced_metadata = {**(enhanced_metadata or {}), **extra_fields}
|
|
1122
|
+
|
|
1123
|
+
# Update in storage asynchronously
|
|
1124
|
+
update_data = {
|
|
1125
|
+
"content": content,
|
|
1126
|
+
"embedding": embedding,
|
|
1127
|
+
"metadata": enhanced_metadata,
|
|
1128
|
+
"hash": content_hash, # Update hash
|
|
1129
|
+
"category": category,
|
|
1130
|
+
"updated_at": datetime.utcnow(),
|
|
1131
|
+
}
|
|
1132
|
+
|
|
1133
|
+
result = await self.storage.update_memory_async(memory_id, update_data, user_id, agent_id)
|
|
1134
|
+
|
|
1135
|
+
# Log audit event
|
|
1136
|
+
await self.audit.log_event_async("memory.update", {
|
|
1137
|
+
"memory_id": memory_id,
|
|
1138
|
+
"user_id": user_id,
|
|
1139
|
+
"agent_id": agent_id
|
|
1140
|
+
}, user_id=user_id, agent_id=agent_id)
|
|
1141
|
+
|
|
1142
|
+
return result
|
|
1143
|
+
|
|
1144
|
+
except Exception as e:
|
|
1145
|
+
logger.error(f"Failed to update memory {memory_id}: {e}")
|
|
1146
|
+
raise
|
|
1147
|
+
|
|
1148
|
+
async def delete(
|
|
1149
|
+
self,
|
|
1150
|
+
memory_id: int,
|
|
1151
|
+
user_id: Optional[str] = None,
|
|
1152
|
+
agent_id: Optional[str] = None,
|
|
1153
|
+
) -> bool:
|
|
1154
|
+
"""Delete a memory asynchronously."""
|
|
1155
|
+
try:
|
|
1156
|
+
result = await self.storage.delete_memory_async(memory_id, user_id, agent_id)
|
|
1157
|
+
|
|
1158
|
+
if result:
|
|
1159
|
+
await self.audit.log_event_async("memory.delete", {
|
|
1160
|
+
"memory_id": memory_id,
|
|
1161
|
+
"user_id": user_id,
|
|
1162
|
+
"agent_id": agent_id
|
|
1163
|
+
}, user_id=user_id, agent_id=agent_id)
|
|
1164
|
+
|
|
1165
|
+
return result
|
|
1166
|
+
|
|
1167
|
+
except Exception as e:
|
|
1168
|
+
logger.error(f"Failed to delete memory {memory_id}: {e}")
|
|
1169
|
+
raise
|
|
1170
|
+
|
|
1171
|
+
async def get_all(
|
|
1172
|
+
self,
|
|
1173
|
+
user_id: Optional[str] = None,
|
|
1174
|
+
agent_id: Optional[str] = None,
|
|
1175
|
+
run_id: Optional[str] = None,
|
|
1176
|
+
limit: int = 100,
|
|
1177
|
+
offset: int = 0,
|
|
1178
|
+
filters: Optional[Dict[str, Any]] = None,
|
|
1179
|
+
) -> Dict[str, List[Dict[str, Any]]]:
|
|
1180
|
+
"""Get all memories with optional filtering asynchronously."""
|
|
1181
|
+
try:
|
|
1182
|
+
results = await self.storage.get_all_memories_async(user_id, agent_id, run_id, limit, offset)
|
|
1183
|
+
|
|
1184
|
+
await self.audit.log_event_async("memory.get_all", {
|
|
1185
|
+
"user_id": user_id,
|
|
1186
|
+
"agent_id": agent_id,
|
|
1187
|
+
"run_id": run_id,
|
|
1188
|
+
"limit": limit,
|
|
1189
|
+
"offset": offset,
|
|
1190
|
+
"results_count": len(results)
|
|
1191
|
+
}, user_id=user_id, agent_id=agent_id)
|
|
1192
|
+
|
|
1193
|
+
# get from graph store
|
|
1194
|
+
if self.enable_graph:
|
|
1195
|
+
filters = {**(filters or {}), "user_id": user_id, "agent_id": agent_id, "run_id": run_id}
|
|
1196
|
+
graph_results = await asyncio.to_thread(self.graph_store.get_all, filters, limit + offset)
|
|
1197
|
+
results.extend(graph_results)
|
|
1198
|
+
return {"results": results, "relations": graph_results}
|
|
1199
|
+
|
|
1200
|
+
return {"results": results}
|
|
1201
|
+
|
|
1202
|
+
except Exception as e:
|
|
1203
|
+
logger.error(f"Failed to get all memories: {e}")
|
|
1204
|
+
raise
|
|
1205
|
+
|
|
1206
|
+
async def delete_all(
|
|
1207
|
+
self,
|
|
1208
|
+
user_id: Optional[str] = None,
|
|
1209
|
+
agent_id: Optional[str] = None,
|
|
1210
|
+
run_id: Optional[str] = None,
|
|
1211
|
+
) -> bool:
|
|
1212
|
+
"""Delete all memories for given identifiers asynchronously."""
|
|
1213
|
+
try:
|
|
1214
|
+
result = await self.storage.clear_memories_async(user_id, agent_id, run_id)
|
|
1215
|
+
|
|
1216
|
+
if result:
|
|
1217
|
+
await self.audit.log_event_async("memory.delete_all", {
|
|
1218
|
+
"user_id": user_id,
|
|
1219
|
+
"agent_id": agent_id,
|
|
1220
|
+
"run_id": run_id
|
|
1221
|
+
}, user_id=user_id, agent_id=agent_id)
|
|
1222
|
+
|
|
1223
|
+
self.telemetry.capture_event("memory.delete_all", {
|
|
1224
|
+
"user_id": user_id,
|
|
1225
|
+
"agent_id": agent_id,
|
|
1226
|
+
"run_id": run_id
|
|
1227
|
+
})
|
|
1228
|
+
|
|
1229
|
+
if self.enable_graph:
|
|
1230
|
+
filters = {"user_id": user_id, "agent_id": agent_id, "run_id": run_id}
|
|
1231
|
+
await asyncio.to_thread(self.graph_store.delete_all, filters)
|
|
1232
|
+
|
|
1233
|
+
return result
|
|
1234
|
+
|
|
1235
|
+
except Exception as e:
|
|
1236
|
+
logger.error(f"Failed to delete all memories: {e}")
|
|
1237
|
+
raise
|
|
1238
|
+
|
|
1239
|
+
async def reset(self):
|
|
1240
|
+
"""
|
|
1241
|
+
Reset the memory store asynchronously by:
|
|
1242
|
+
Deletes the vector store collection
|
|
1243
|
+
Resets the database
|
|
1244
|
+
Recreates the vector store with a new client
|
|
1245
|
+
"""
|
|
1246
|
+
logger.warning("Resetting all memories")
|
|
1247
|
+
|
|
1248
|
+
try:
|
|
1249
|
+
# Reset vector store asynchronously
|
|
1250
|
+
if hasattr(self.storage.vector_store, "reset"):
|
|
1251
|
+
await asyncio.to_thread(self.storage.vector_store.reset)
|
|
1252
|
+
else:
|
|
1253
|
+
logger.warning("Vector store does not support reset. Skipping.")
|
|
1254
|
+
await asyncio.to_thread(self.storage.vector_store.delete_col)
|
|
1255
|
+
# Recreate vector store
|
|
1256
|
+
from ..storage.factory import VectorStoreFactory
|
|
1257
|
+
vector_store_config = self._get_component_config('vector_store')
|
|
1258
|
+
self.storage.vector_store = VectorStoreFactory.create(self.storage_type, vector_store_config)
|
|
1259
|
+
# Update storage adapter
|
|
1260
|
+
self.storage = StorageAdapter(self.storage.vector_store, self.embedding)
|
|
1261
|
+
|
|
1262
|
+
# Reset graph store if enabled
|
|
1263
|
+
if self.enable_graph and hasattr(self.graph_store, "reset"):
|
|
1264
|
+
await asyncio.to_thread(self.graph_store.reset)
|
|
1265
|
+
|
|
1266
|
+
# Log telemetry event
|
|
1267
|
+
self.telemetry.capture_event("memory.reset", {"sync_type": "async"})
|
|
1268
|
+
|
|
1269
|
+
logger.info("Memory store reset completed successfully")
|
|
1270
|
+
|
|
1271
|
+
except Exception as e:
|
|
1272
|
+
logger.error(f"Failed to reset memory store: {e}")
|
|
1273
|
+
raise
|
|
1274
|
+
|
|
1275
|
+
def _init_sub_stores(self):
|
|
1276
|
+
"""Initialize multiple sub stores configuration"""
|
|
1277
|
+
if self.sub_stores_config:
|
|
1278
|
+
logger.info(f"Sub stores enabled: {len(self.sub_stores_config)} stores")
|
|
1279
|
+
|
|
1280
|
+
sub_stores_list = self.config.get('sub_stores', [])
|
|
1281
|
+
|
|
1282
|
+
if not sub_stores_list:
|
|
1283
|
+
logger.info("No sub stores configured")
|
|
1284
|
+
return
|
|
1285
|
+
|
|
1286
|
+
# Sub store feature only supports OceanBase storage
|
|
1287
|
+
if self.storage_type.lower() != 'oceanbase':
|
|
1288
|
+
logger.warning(f"Sub store feature only supports OceanBase storage, current storage: {self.storage_type}")
|
|
1289
|
+
logger.warning("Sub stores configuration will be ignored")
|
|
1290
|
+
return
|
|
1291
|
+
|
|
1292
|
+
# Get main table information
|
|
1293
|
+
main_collection_name = self.config.get('vector_store', {}).get('config', {}).get('collection_name', 'memories')
|
|
1294
|
+
main_embedding_dims = self.config.get('vector_store', {}).get('config', {}).get('embedding_model_dims', 1536)
|
|
1295
|
+
|
|
1296
|
+
# Iterate through configs and initialize each sub store
|
|
1297
|
+
for index, sub_config in enumerate(sub_stores_list):
|
|
1298
|
+
try:
|
|
1299
|
+
self._init_single_sub_store(index, sub_config, main_collection_name, main_embedding_dims)
|
|
1300
|
+
except Exception as e:
|
|
1301
|
+
logger.error(f"Failed to initialize sub store {index}: {e}")
|
|
1302
|
+
continue
|
|
1303
|
+
|
|
1304
|
+
def _init_single_sub_store(
|
|
1305
|
+
self,
|
|
1306
|
+
index: int,
|
|
1307
|
+
sub_config: Dict,
|
|
1308
|
+
main_collection_name: str,
|
|
1309
|
+
main_embedding_dims: int
|
|
1310
|
+
):
|
|
1311
|
+
"""Initialize a single sub store"""
|
|
1312
|
+
|
|
1313
|
+
# 1. Determine sub store name (default: {main_table_name}_sub_{index})
|
|
1314
|
+
sub_store_name = sub_config.get(
|
|
1315
|
+
'collection_name',
|
|
1316
|
+
f"{main_collection_name}_sub_{index}"
|
|
1317
|
+
)
|
|
1318
|
+
|
|
1319
|
+
# 2. Get routing rules (required)
|
|
1320
|
+
routing_filter = sub_config.get('routing_filter')
|
|
1321
|
+
if not routing_filter:
|
|
1322
|
+
logger.warning(f"Sub store {index} has no routing_filter, skipping")
|
|
1323
|
+
return
|
|
1324
|
+
|
|
1325
|
+
# 3. Determine vector dimension (default: same as main table)
|
|
1326
|
+
embedding_model_dims = sub_config.get('embedding_model_dims', main_embedding_dims)
|
|
1327
|
+
|
|
1328
|
+
# 4. Initialize sub store's embedding service
|
|
1329
|
+
sub_embedding_config = sub_config.get('embedding', {})
|
|
1330
|
+
|
|
1331
|
+
if sub_embedding_config:
|
|
1332
|
+
# Has independent embedding configuration
|
|
1333
|
+
sub_embedding_provider = sub_embedding_config.get('provider', self.embedding_provider)
|
|
1334
|
+
sub_embedding_params = sub_embedding_config.get('config', {})
|
|
1335
|
+
|
|
1336
|
+
# Inherit api_key and other configs from main table
|
|
1337
|
+
main_embedding_config = self.config.get('embedder', {}).get('config', {})
|
|
1338
|
+
for key in ['api_key', 'openai_base_url', 'timeout']:
|
|
1339
|
+
if key not in sub_embedding_params and key in main_embedding_config:
|
|
1340
|
+
sub_embedding_params[key] = main_embedding_config[key]
|
|
1341
|
+
|
|
1342
|
+
sub_embedding = EmbedderFactory.create(
|
|
1343
|
+
sub_embedding_provider,
|
|
1344
|
+
sub_embedding_params,
|
|
1345
|
+
None
|
|
1346
|
+
)
|
|
1347
|
+
logger.info(f"Created sub embedding service for store {index}: {sub_embedding_provider}")
|
|
1348
|
+
else:
|
|
1349
|
+
# Reuse main table's embedding service
|
|
1350
|
+
sub_embedding = self.embedding
|
|
1351
|
+
logger.info(f"Sub store {index} using main embedding service")
|
|
1352
|
+
|
|
1353
|
+
# 5. Create sub store storage instance
|
|
1354
|
+
db_config = self.config.get('vector_store', {}).get('config', {}).copy()
|
|
1355
|
+
db_config['collection_name'] = sub_store_name
|
|
1356
|
+
db_config['embedding_model_dims'] = embedding_model_dims
|
|
1357
|
+
|
|
1358
|
+
sub_vector_store = VectorStoreFactory.create(self.storage_type, db_config)
|
|
1359
|
+
|
|
1360
|
+
# 6. Register sub store in Adapter (with embedding service for migration)
|
|
1361
|
+
self.storage.register_sub_store(
|
|
1362
|
+
store_name=sub_store_name,
|
|
1363
|
+
routing_filter=routing_filter,
|
|
1364
|
+
vector_store=sub_vector_store,
|
|
1365
|
+
embedding_service=sub_embedding,
|
|
1366
|
+
)
|
|
1367
|
+
|
|
1368
|
+
# 7. Save sub store configuration
|
|
1369
|
+
self.sub_stores_config.append({
|
|
1370
|
+
'name': sub_store_name,
|
|
1371
|
+
'routing_filter': routing_filter,
|
|
1372
|
+
'embedding_service': sub_embedding,
|
|
1373
|
+
'embedding_dims': embedding_model_dims,
|
|
1374
|
+
})
|
|
1375
|
+
|
|
1376
|
+
logger.info(f"Registered sub store {index}: {sub_store_name} (dims={embedding_model_dims})")
|
|
1377
|
+
|
|
1378
|
+
def _get_embedding_service(self, filters_or_metadata: Optional[Dict] = None):
|
|
1379
|
+
"""
|
|
1380
|
+
Select appropriate embedding service based on filters or metadata
|
|
1381
|
+
|
|
1382
|
+
Args:
|
|
1383
|
+
filters_or_metadata: Query filters (for search) or memory metadata (for add)
|
|
1384
|
+
|
|
1385
|
+
Returns:
|
|
1386
|
+
Corresponding embedding service instance
|
|
1387
|
+
"""
|
|
1388
|
+
if not filters_or_metadata or not self.sub_stores_config:
|
|
1389
|
+
return self.embedding
|
|
1390
|
+
|
|
1391
|
+
# Iterate through all sub stores to find a match
|
|
1392
|
+
if isinstance(self.storage, SubStorageAdapter):
|
|
1393
|
+
for sub_config in self.sub_stores_config:
|
|
1394
|
+
# Check if sub store is ready
|
|
1395
|
+
if not self.storage.is_sub_store_ready(sub_config['name']):
|
|
1396
|
+
continue
|
|
1397
|
+
|
|
1398
|
+
# Check if filters_or_metadata matches routing rules
|
|
1399
|
+
routing_filter = sub_config['routing_filter']
|
|
1400
|
+
if all(
|
|
1401
|
+
key in filters_or_metadata and filters_or_metadata[key] == value
|
|
1402
|
+
for key, value in routing_filter.items()
|
|
1403
|
+
):
|
|
1404
|
+
logger.debug(f"Using sub embedding for store: {sub_config['name']}")
|
|
1405
|
+
return sub_config['embedding_service']
|
|
1406
|
+
|
|
1407
|
+
logger.debug("Using main embedding service")
|
|
1408
|
+
return self.embedding
|
|
1409
|
+
|
|
1410
|
+
async def migrate_to_sub_store(self, sub_store_index: int = 0, delete_source: bool = False) -> int:
|
|
1411
|
+
"""
|
|
1412
|
+
Migrate data to specified sub store
|
|
1413
|
+
|
|
1414
|
+
Args:
|
|
1415
|
+
sub_store_index: Sub store index (default 0, i.e., first sub store)
|
|
1416
|
+
delete_source: Whether to delete source data
|
|
1417
|
+
|
|
1418
|
+
Returns:
|
|
1419
|
+
Number of migrated records
|
|
1420
|
+
"""
|
|
1421
|
+
if not self.sub_stores_config:
|
|
1422
|
+
raise ValueError("No sub stores configured.")
|
|
1423
|
+
|
|
1424
|
+
if sub_store_index >= len(self.sub_stores_config):
|
|
1425
|
+
raise ValueError(f"Sub store index {sub_store_index} out of range")
|
|
1426
|
+
|
|
1427
|
+
sub_config = self.sub_stores_config[sub_store_index]
|
|
1428
|
+
|
|
1429
|
+
logger.info(f"Starting migration to sub store: {sub_config['name']}")
|
|
1430
|
+
|
|
1431
|
+
# Call adapter's migration method
|
|
1432
|
+
if isinstance(self.storage, SubStorageAdapter):
|
|
1433
|
+
migrated_count = await asyncio.to_thread(
|
|
1434
|
+
self.storage.migrate_to_sub_store,
|
|
1435
|
+
store_name=sub_config['name'],
|
|
1436
|
+
delete_source=delete_source
|
|
1437
|
+
)
|
|
1438
|
+
|
|
1439
|
+
logger.info(f"Migration completed: {migrated_count} records migrated")
|
|
1440
|
+
return migrated_count
|
|
1441
|
+
else:
|
|
1442
|
+
raise ValueError("Storage adapter does not support migration")
|
|
1443
|
+
|
|
1444
|
+
async def migrate_all_sub_stores(self, delete_source: bool = True) -> Dict[str, int]:
|
|
1445
|
+
"""
|
|
1446
|
+
Migrate all sub stores
|
|
1447
|
+
|
|
1448
|
+
Args:
|
|
1449
|
+
delete_source: Whether to delete source data
|
|
1450
|
+
|
|
1451
|
+
Returns:
|
|
1452
|
+
Migration record count for each sub store {store_name: count}
|
|
1453
|
+
"""
|
|
1454
|
+
results = {}
|
|
1455
|
+
for index, sub_config in enumerate(self.sub_stores_config):
|
|
1456
|
+
try:
|
|
1457
|
+
count = await self.migrate_to_sub_store(index, delete_source)
|
|
1458
|
+
results[sub_config['name']] = count
|
|
1459
|
+
except Exception as e:
|
|
1460
|
+
logger.error(f"Failed to migrate sub store {index}: {e}")
|
|
1461
|
+
results[sub_config['name']] = 0
|
|
1462
|
+
|
|
1463
|
+
return results
|
|
1464
|
+
|
|
1465
|
+
@classmethod
|
|
1466
|
+
async def from_config(cls, config: Optional[Dict[str, Any]] = None, **kwargs):
|
|
1467
|
+
"""
|
|
1468
|
+
Create AsyncMemory instance from configuration.
|
|
1469
|
+
|
|
1470
|
+
Args:
|
|
1471
|
+
config: Configuration dictionary
|
|
1472
|
+
**kwargs: Additional parameters
|
|
1473
|
+
|
|
1474
|
+
Returns:
|
|
1475
|
+
AsyncMemory instance
|
|
1476
|
+
|
|
1477
|
+
Example:
|
|
1478
|
+
```python
|
|
1479
|
+
memory = await AsyncMemory.from_config({
|
|
1480
|
+
"llm": {"provider": "openai", "config": {"api_key": "..."}},
|
|
1481
|
+
"embedder": {"provider": "openai", "config": {"api_key": "..."}},
|
|
1482
|
+
"vector_store": {"provider": "oceanbase", "config": {...}},
|
|
1483
|
+
})
|
|
1484
|
+
```
|
|
1485
|
+
"""
|
|
1486
|
+
if config is None:
|
|
1487
|
+
# Use auto config from environment
|
|
1488
|
+
from ..config_loader import auto_config
|
|
1489
|
+
config = auto_config()
|
|
1490
|
+
|
|
1491
|
+
converted_config = _auto_convert_config(config)
|
|
1492
|
+
|
|
1493
|
+
return cls(config=converted_config, **kwargs)
|