MemoryOS 0.2.0__py3-none-any.whl → 0.2.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of MemoryOS might be problematic. Click here for more details.
- {memoryos-0.2.0.dist-info → memoryos-0.2.2.dist-info}/METADATA +67 -26
- memoryos-0.2.2.dist-info/RECORD +169 -0
- memoryos-0.2.2.dist-info/entry_points.txt +3 -0
- memos/__init__.py +1 -1
- memos/api/config.py +562 -0
- memos/api/context/context.py +147 -0
- memos/api/context/dependencies.py +90 -0
- memos/api/exceptions.py +28 -0
- memos/api/mcp_serve.py +502 -0
- memos/api/product_api.py +35 -0
- memos/api/product_models.py +163 -0
- memos/api/routers/__init__.py +1 -0
- memos/api/routers/product_router.py +386 -0
- memos/chunkers/sentence_chunker.py +8 -2
- memos/cli.py +113 -0
- memos/configs/embedder.py +27 -0
- memos/configs/graph_db.py +132 -3
- memos/configs/internet_retriever.py +6 -0
- memos/configs/llm.py +47 -0
- memos/configs/mem_cube.py +1 -1
- memos/configs/mem_os.py +5 -0
- memos/configs/mem_reader.py +9 -0
- memos/configs/mem_scheduler.py +107 -7
- memos/configs/mem_user.py +58 -0
- memos/configs/memory.py +5 -4
- memos/dependency.py +52 -0
- memos/embedders/ark.py +92 -0
- memos/embedders/factory.py +4 -0
- memos/embedders/sentence_transformer.py +8 -2
- memos/embedders/universal_api.py +32 -0
- memos/graph_dbs/base.py +11 -3
- memos/graph_dbs/factory.py +4 -0
- memos/graph_dbs/nebular.py +1364 -0
- memos/graph_dbs/neo4j.py +333 -124
- memos/graph_dbs/neo4j_community.py +300 -0
- memos/llms/base.py +9 -0
- memos/llms/deepseek.py +54 -0
- memos/llms/factory.py +10 -1
- memos/llms/hf.py +170 -13
- memos/llms/hf_singleton.py +114 -0
- memos/llms/ollama.py +4 -0
- memos/llms/openai.py +67 -1
- memos/llms/qwen.py +63 -0
- memos/llms/vllm.py +153 -0
- memos/log.py +1 -1
- memos/mem_cube/general.py +77 -16
- memos/mem_cube/utils.py +109 -0
- memos/mem_os/core.py +251 -51
- memos/mem_os/main.py +94 -12
- memos/mem_os/product.py +1220 -43
- memos/mem_os/utils/default_config.py +352 -0
- memos/mem_os/utils/format_utils.py +1401 -0
- memos/mem_reader/simple_struct.py +18 -10
- memos/mem_scheduler/base_scheduler.py +441 -40
- memos/mem_scheduler/general_scheduler.py +249 -248
- memos/mem_scheduler/modules/base.py +14 -5
- memos/mem_scheduler/modules/dispatcher.py +67 -4
- memos/mem_scheduler/modules/misc.py +104 -0
- memos/mem_scheduler/modules/monitor.py +240 -50
- memos/mem_scheduler/modules/rabbitmq_service.py +319 -0
- memos/mem_scheduler/modules/redis_service.py +32 -22
- memos/mem_scheduler/modules/retriever.py +167 -23
- memos/mem_scheduler/modules/scheduler_logger.py +255 -0
- memos/mem_scheduler/mos_for_test_scheduler.py +140 -0
- memos/mem_scheduler/schemas/__init__.py +0 -0
- memos/mem_scheduler/schemas/general_schemas.py +43 -0
- memos/mem_scheduler/{modules/schemas.py → schemas/message_schemas.py} +63 -61
- memos/mem_scheduler/schemas/monitor_schemas.py +329 -0
- memos/mem_scheduler/utils/__init__.py +0 -0
- memos/mem_scheduler/utils/filter_utils.py +176 -0
- memos/mem_scheduler/utils/misc_utils.py +61 -0
- memos/mem_user/factory.py +94 -0
- memos/mem_user/mysql_persistent_user_manager.py +271 -0
- memos/mem_user/mysql_user_manager.py +500 -0
- memos/mem_user/persistent_factory.py +96 -0
- memos/mem_user/persistent_user_manager.py +260 -0
- memos/mem_user/user_manager.py +4 -4
- memos/memories/activation/item.py +29 -0
- memos/memories/activation/kv.py +10 -3
- memos/memories/activation/vllmkv.py +219 -0
- memos/memories/factory.py +2 -0
- memos/memories/textual/base.py +1 -1
- memos/memories/textual/general.py +43 -97
- memos/memories/textual/item.py +5 -33
- memos/memories/textual/tree.py +22 -12
- memos/memories/textual/tree_text_memory/organize/conflict.py +9 -5
- memos/memories/textual/tree_text_memory/organize/manager.py +26 -18
- memos/memories/textual/tree_text_memory/organize/redundancy.py +25 -44
- memos/memories/textual/tree_text_memory/organize/relation_reason_detector.py +50 -48
- memos/memories/textual/tree_text_memory/organize/reorganizer.py +81 -56
- memos/memories/textual/tree_text_memory/retrieve/internet_retriever.py +6 -3
- memos/memories/textual/tree_text_memory/retrieve/internet_retriever_factory.py +2 -0
- memos/memories/textual/tree_text_memory/retrieve/recall.py +0 -1
- memos/memories/textual/tree_text_memory/retrieve/reranker.py +2 -2
- memos/memories/textual/tree_text_memory/retrieve/retrieval_mid_structs.py +2 -0
- memos/memories/textual/tree_text_memory/retrieve/searcher.py +52 -28
- memos/memories/textual/tree_text_memory/retrieve/task_goal_parser.py +42 -15
- memos/memories/textual/tree_text_memory/retrieve/utils.py +11 -7
- memos/memories/textual/tree_text_memory/retrieve/xinyusearch.py +62 -58
- memos/memos_tools/dinding_report_bot.py +422 -0
- memos/memos_tools/notification_service.py +44 -0
- memos/memos_tools/notification_utils.py +96 -0
- memos/parsers/markitdown.py +8 -2
- memos/settings.py +3 -1
- memos/templates/mem_reader_prompts.py +66 -23
- memos/templates/mem_scheduler_prompts.py +126 -43
- memos/templates/mos_prompts.py +87 -0
- memos/templates/tree_reorganize_prompts.py +85 -30
- memos/vec_dbs/base.py +12 -0
- memos/vec_dbs/qdrant.py +46 -20
- memoryos-0.2.0.dist-info/RECORD +0 -128
- memos/mem_scheduler/utils.py +0 -26
- {memoryos-0.2.0.dist-info → memoryos-0.2.2.dist-info}/LICENSE +0 -0
- {memoryos-0.2.0.dist-info → memoryos-0.2.2.dist-info}/WHEEL +0 -0
|
@@ -7,11 +7,12 @@ from typing import Any
|
|
|
7
7
|
from tenacity import retry, retry_if_exception_type, stop_after_attempt
|
|
8
8
|
|
|
9
9
|
from memos.configs.memory import GeneralTextMemoryConfig
|
|
10
|
-
from memos.embedders.factory import EmbedderFactory, OllamaEmbedder
|
|
11
|
-
from memos.llms.factory import LLMFactory, OllamaLLM, OpenAILLM
|
|
10
|
+
from memos.embedders.factory import ArkEmbedder, EmbedderFactory, OllamaEmbedder
|
|
11
|
+
from memos.llms.factory import AzureLLM, LLMFactory, OllamaLLM, OpenAILLM
|
|
12
12
|
from memos.log import get_logger
|
|
13
13
|
from memos.memories.textual.base import BaseTextMemory
|
|
14
14
|
from memos.memories.textual.item import TextualMemoryItem
|
|
15
|
+
from memos.templates.mem_reader_prompts import SIMPLE_STRUCT_MEM_READER_PROMPT
|
|
15
16
|
from memos.types import MessageList
|
|
16
17
|
from memos.vec_dbs.factory import QdrantVecDB, VecDBFactory
|
|
17
18
|
from memos.vec_dbs.item import VecDBItem
|
|
@@ -26,19 +27,17 @@ class GeneralTextMemory(BaseTextMemory):
|
|
|
26
27
|
def __init__(self, config: GeneralTextMemoryConfig):
|
|
27
28
|
"""Initialize memory with the given configuration."""
|
|
28
29
|
self.config: GeneralTextMemoryConfig = config
|
|
29
|
-
self.extractor_llm: OpenAILLM | OllamaLLM = LLMFactory.from_config(
|
|
30
|
+
self.extractor_llm: OpenAILLM | OllamaLLM | AzureLLM = LLMFactory.from_config(
|
|
31
|
+
config.extractor_llm
|
|
32
|
+
)
|
|
30
33
|
self.vector_db: QdrantVecDB = VecDBFactory.from_config(config.vector_db)
|
|
31
|
-
self.embedder: OllamaEmbedder = EmbedderFactory.from_config(config.embedder)
|
|
34
|
+
self.embedder: OllamaEmbedder | ArkEmbedder = EmbedderFactory.from_config(config.embedder)
|
|
32
35
|
|
|
33
36
|
@retry(
|
|
34
37
|
stop=stop_after_attempt(3),
|
|
35
38
|
retry=retry_if_exception_type(json.JSONDecodeError),
|
|
36
39
|
before_sleep=lambda retry_state: logger.warning(
|
|
37
|
-
|
|
38
|
-
error=retry_state.outcome.exception(),
|
|
39
|
-
attempt_number=retry_state.attempt_number,
|
|
40
|
-
max_attempt_number=3,
|
|
41
|
-
)
|
|
40
|
+
f"Extracting memory failed due to JSON decode error: {retry_state.outcome.exception()}, Attempt retry: {retry_state.attempt_number} / {3}"
|
|
42
41
|
),
|
|
43
42
|
)
|
|
44
43
|
def extract(self, messages: MessageList) -> list[TextualMemoryItem]:
|
|
@@ -50,14 +49,27 @@ class GeneralTextMemory(BaseTextMemory):
|
|
|
50
49
|
Returns:
|
|
51
50
|
List of TextualMemoryItem objects representing the extracted memories.
|
|
52
51
|
"""
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
messages
|
|
52
|
+
|
|
53
|
+
str_messages = "\n".join(
|
|
54
|
+
[message["role"] + ":" + message["content"] for message in messages]
|
|
56
55
|
)
|
|
57
|
-
|
|
58
|
-
|
|
56
|
+
|
|
57
|
+
prompt = SIMPLE_STRUCT_MEM_READER_PROMPT.replace("${conversation}", str_messages)
|
|
58
|
+
messages = [{"role": "user", "content": prompt}]
|
|
59
|
+
response_text = self.extractor_llm.generate(messages)
|
|
60
|
+
response_json = self.parse_json_result(response_text)
|
|
61
|
+
|
|
59
62
|
extracted_memories = [
|
|
60
|
-
TextualMemoryItem(
|
|
63
|
+
TextualMemoryItem(
|
|
64
|
+
memory=memory_dict["value"],
|
|
65
|
+
metadata={
|
|
66
|
+
"key": memory_dict["key"],
|
|
67
|
+
"source": "conversation",
|
|
68
|
+
"tags": memory_dict["tags"],
|
|
69
|
+
"updated_at": datetime.now().isoformat(),
|
|
70
|
+
},
|
|
71
|
+
)
|
|
72
|
+
for memory_dict in response_json["memory list"]
|
|
61
73
|
]
|
|
62
74
|
|
|
63
75
|
return extracted_memories
|
|
@@ -202,85 +214,19 @@ class GeneralTextMemory(BaseTextMemory):
|
|
|
202
214
|
|
|
203
215
|
def _embed_one_sentence(self, sentence: str) -> list[float]:
|
|
204
216
|
"""Embed a single sentence."""
|
|
205
|
-
return self.embedder.embed(sentence)[0]
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
- "visibility": The accessibility scope of the memory (string), e.g., `"private"`, `"public"`, `"session"`, determining who or what contexts can access it.
|
|
222
|
-
- "updated_at": The timestamp of the last modification to the memory (string). Useful for tracking memory freshness or change history. Format: ISO 8601 or natural language.
|
|
223
|
-
* Current date and time is {datetime.now().isoformat()}.
|
|
224
|
-
* Only return the list of memories in JSON format.
|
|
225
|
-
* Do not include any explanations
|
|
226
|
-
* Do not include any extra text
|
|
227
|
-
* Do not include code blocks (```json```)
|
|
228
|
-
|
|
229
|
-
## Example
|
|
230
|
-
|
|
231
|
-
### Input
|
|
232
|
-
|
|
233
|
-
[
|
|
234
|
-
{{"role": "user", "content": "I plan to visit Paris next week."}},
|
|
235
|
-
{{"role": "assistant", "content": "Paris is a beautiful city with many attractions."}},
|
|
236
|
-
{{"role": "user", "content": "I love the Eiffel Tower."}},
|
|
237
|
-
{{"role": "assistant", "content": "The Eiffel Tower is a must-see landmark in Paris."}}
|
|
238
|
-
]
|
|
239
|
-
|
|
240
|
-
### Output
|
|
241
|
-
|
|
242
|
-
[
|
|
243
|
-
{{
|
|
244
|
-
"memory": "The user plans to visit Paris on 05-26-2025.",
|
|
245
|
-
"metadata": {{
|
|
246
|
-
"type": "event",
|
|
247
|
-
"memory_time": "2025-05-26",
|
|
248
|
-
"source": "conversation",
|
|
249
|
-
"confidence": 90.0,
|
|
250
|
-
"entities": ["Paris"],
|
|
251
|
-
"tags": ["travel", "plans"],
|
|
252
|
-
"visibility": "private",
|
|
253
|
-
"updated_at": "2025-05-19T00:00:00"
|
|
254
|
-
}}
|
|
255
|
-
}},
|
|
256
|
-
{{
|
|
257
|
-
"memory": "The user loves the Eiffel Tower.",
|
|
258
|
-
"metadata": {{
|
|
259
|
-
"type": "opinion",
|
|
260
|
-
"memory_time": "2025-05-19",
|
|
261
|
-
"source": "conversation",
|
|
262
|
-
"confidence": 100.0,
|
|
263
|
-
"entities": ["Eiffel Tower"],
|
|
264
|
-
"tags": ["opinions", "landmarks"],
|
|
265
|
-
"visibility": "session",
|
|
266
|
-
"updated_at": "2025-05-19T00:00:00"
|
|
267
|
-
}}
|
|
268
|
-
}}
|
|
269
|
-
]
|
|
270
|
-
|
|
271
|
-
"""
|
|
272
|
-
|
|
273
|
-
EXTRACTION_PROMPT_PART_2 = """
|
|
274
|
-
## Query
|
|
275
|
-
|
|
276
|
-
### Input
|
|
277
|
-
|
|
278
|
-
{messages}
|
|
279
|
-
|
|
280
|
-
### Output
|
|
281
|
-
|
|
282
|
-
"""
|
|
283
|
-
|
|
284
|
-
EXTRACTION_RETRY_LOG = """Extracting memory failed due to JSON decode error: {error},
|
|
285
|
-
Attempt retry: {attempt_number} / {max_attempt_number}
|
|
286
|
-
"""
|
|
217
|
+
return self.embedder.embed([sentence])[0]
|
|
218
|
+
|
|
219
|
+
def parse_json_result(self, response_text):
|
|
220
|
+
try:
|
|
221
|
+
json_start = response_text.find("{")
|
|
222
|
+
response_text = response_text[json_start:]
|
|
223
|
+
response_text = response_text.replace("```", "").strip()
|
|
224
|
+
if response_text[-1] != "}":
|
|
225
|
+
response_text += "}"
|
|
226
|
+
response_json = json.loads(response_text)
|
|
227
|
+
return response_json
|
|
228
|
+
except json.JSONDecodeError as e:
|
|
229
|
+
logger.warning(
|
|
230
|
+
f"Failed to parse LLM response as JSON: {e}\nRaw response:\n{response_text}"
|
|
231
|
+
)
|
|
232
|
+
return {}
|
memos/memories/textual/item.py
CHANGED
|
@@ -27,23 +27,14 @@ class TextualMemoryMetadata(BaseModel):
|
|
|
27
27
|
default="activated",
|
|
28
28
|
description="The status of the memory, e.g., 'activated', 'archived', 'deleted'.",
|
|
29
29
|
)
|
|
30
|
-
type:
|
|
31
|
-
|
|
32
|
-
)
|
|
33
|
-
memory_time: str | None = Field(
|
|
34
|
-
default=None,
|
|
35
|
-
description='The time the memory occurred or refers to. Must be in standard `YYYY-MM-DD` format. Relative expressions such as "yesterday" or "tomorrow" are not allowed.',
|
|
36
|
-
)
|
|
37
|
-
source: Literal["conversation", "retrieved", "web", "file"] | None = Field(
|
|
38
|
-
default=None, description="The origin of the memory"
|
|
39
|
-
)
|
|
30
|
+
type: str | None = Field(default=None)
|
|
31
|
+
key: str | None = Field(default=None, description="Memory key or title.")
|
|
40
32
|
confidence: float | None = Field(
|
|
41
33
|
default=None,
|
|
42
34
|
description="A numeric score (float between 0 and 100) indicating how certain you are about the accuracy or reliability of the memory.",
|
|
43
35
|
)
|
|
44
|
-
|
|
45
|
-
default=None,
|
|
46
|
-
description='A list of key entities mentioned in the memory, e.g., people, places, organizations, e.g., `["Alice", "Paris", "OpenAI"]`.',
|
|
36
|
+
source: Literal["conversation", "retrieved", "web", "file"] | None = Field(
|
|
37
|
+
default=None, description="The origin of the memory"
|
|
47
38
|
)
|
|
48
39
|
tags: list[str] | None = Field(
|
|
49
40
|
default=None,
|
|
@@ -59,23 +50,6 @@ class TextualMemoryMetadata(BaseModel):
|
|
|
59
50
|
|
|
60
51
|
model_config = ConfigDict(extra="allow")
|
|
61
52
|
|
|
62
|
-
@field_validator("memory_time")
|
|
63
|
-
@classmethod
|
|
64
|
-
def validate_memory_time(cls, v):
|
|
65
|
-
try:
|
|
66
|
-
if v:
|
|
67
|
-
datetime.strptime(v, "%Y-%m-%d")
|
|
68
|
-
except ValueError as e:
|
|
69
|
-
raise ValueError("Invalid date format. Use YYYY-MM-DD.") from e
|
|
70
|
-
return v
|
|
71
|
-
|
|
72
|
-
@field_validator("confidence")
|
|
73
|
-
@classmethod
|
|
74
|
-
def validate_confidence(cls, v):
|
|
75
|
-
if v is not None and (v < 0 or v > 100):
|
|
76
|
-
raise ValueError("Confidence must be between 0 and 100.")
|
|
77
|
-
return v
|
|
78
|
-
|
|
79
53
|
def __str__(self) -> str:
|
|
80
54
|
"""Pretty string representation of the metadata."""
|
|
81
55
|
meta = self.model_dump(exclude_none=True)
|
|
@@ -85,10 +59,9 @@ class TextualMemoryMetadata(BaseModel):
|
|
|
85
59
|
class TreeNodeTextualMemoryMetadata(TextualMemoryMetadata):
|
|
86
60
|
"""Extended metadata for structured memory, layered retrieval, and lifecycle tracking."""
|
|
87
61
|
|
|
88
|
-
memory_type: Literal["WorkingMemory", "LongTermMemory", "UserMemory"] = Field(
|
|
62
|
+
memory_type: Literal["WorkingMemory", "LongTermMemory", "UserMemory", "OuterMemory"] = Field(
|
|
89
63
|
default="WorkingMemory", description="Memory lifecycle type."
|
|
90
64
|
)
|
|
91
|
-
key: str | None = Field(default=None, description="Memory key or title.")
|
|
92
65
|
sources: list[str] | None = Field(
|
|
93
66
|
default=None, description="Multiple origins of the memory (e.g., URLs, notes)."
|
|
94
67
|
)
|
|
@@ -148,7 +121,6 @@ class TextualMemoryItem(BaseModel):
|
|
|
148
121
|
|
|
149
122
|
model_config = ConfigDict(extra="forbid")
|
|
150
123
|
|
|
151
|
-
@field_validator("id")
|
|
152
124
|
@classmethod
|
|
153
125
|
def validate_id(cls, v):
|
|
154
126
|
try:
|
memos/memories/textual/tree.py
CHANGED
|
@@ -10,7 +10,7 @@ from typing import Any
|
|
|
10
10
|
from memos.configs.memory import TreeTextMemoryConfig
|
|
11
11
|
from memos.embedders.factory import EmbedderFactory, OllamaEmbedder
|
|
12
12
|
from memos.graph_dbs.factory import GraphStoreFactory, Neo4jGraphDB
|
|
13
|
-
from memos.llms.factory import LLMFactory, OllamaLLM, OpenAILLM
|
|
13
|
+
from memos.llms.factory import AzureLLM, LLMFactory, OllamaLLM, OpenAILLM
|
|
14
14
|
from memos.log import get_logger
|
|
15
15
|
from memos.memories.textual.base import BaseTextMemory
|
|
16
16
|
from memos.memories.textual.item import TextualMemoryItem, TreeNodeTextualMemoryMetadata
|
|
@@ -31,8 +31,12 @@ class TreeTextMemory(BaseTextMemory):
|
|
|
31
31
|
def __init__(self, config: TreeTextMemoryConfig):
|
|
32
32
|
"""Initialize memory with the given configuration."""
|
|
33
33
|
self.config: TreeTextMemoryConfig = config
|
|
34
|
-
self.extractor_llm: OpenAILLM | OllamaLLM = LLMFactory.from_config(
|
|
35
|
-
|
|
34
|
+
self.extractor_llm: OpenAILLM | OllamaLLM | AzureLLM = LLMFactory.from_config(
|
|
35
|
+
config.extractor_llm
|
|
36
|
+
)
|
|
37
|
+
self.dispatcher_llm: OpenAILLM | OllamaLLM | AzureLLM = LLMFactory.from_config(
|
|
38
|
+
config.dispatcher_llm
|
|
39
|
+
)
|
|
36
40
|
self.embedder: OllamaEmbedder = EmbedderFactory.from_config(config.embedder)
|
|
37
41
|
self.graph_store: Neo4jGraphDB = GraphStoreFactory.from_config(config.graph_db)
|
|
38
42
|
self.is_reorganize = config.reorganize
|
|
@@ -53,7 +57,7 @@ class TreeTextMemory(BaseTextMemory):
|
|
|
53
57
|
else:
|
|
54
58
|
logger.info("No internet retriever configured")
|
|
55
59
|
|
|
56
|
-
def add(self, memories: list[TextualMemoryItem | dict[str, Any]]) ->
|
|
60
|
+
def add(self, memories: list[TextualMemoryItem | dict[str, Any]]) -> list[str]:
|
|
57
61
|
"""Add memories.
|
|
58
62
|
Args:
|
|
59
63
|
memories: List of TextualMemoryItem objects or dictionaries to add.
|
|
@@ -63,7 +67,7 @@ class TreeTextMemory(BaseTextMemory):
|
|
|
63
67
|
plan = plan_memory_operations(memory_items, metadata, self.graph_store)
|
|
64
68
|
execute_plan(memory_items, metadata, plan, self.graph_store)
|
|
65
69
|
"""
|
|
66
|
-
self.memory_manager.add(memories)
|
|
70
|
+
return self.memory_manager.add(memories)
|
|
67
71
|
|
|
68
72
|
def replace_working_memory(self, memories: list[TextualMemoryItem]) -> None:
|
|
69
73
|
self.memory_manager.replace_working_memory(memories)
|
|
@@ -113,13 +117,19 @@ class TreeTextMemory(BaseTextMemory):
|
|
|
113
117
|
logger.warning(
|
|
114
118
|
"Internet retriever is init by config , but this search set manual_close_internet is True and will close it"
|
|
115
119
|
)
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
120
|
+
searcher = Searcher(
|
|
121
|
+
self.dispatcher_llm,
|
|
122
|
+
self.graph_store,
|
|
123
|
+
self.embedder,
|
|
124
|
+
internet_retriever=None,
|
|
125
|
+
)
|
|
126
|
+
else:
|
|
127
|
+
searcher = Searcher(
|
|
128
|
+
self.dispatcher_llm,
|
|
129
|
+
self.graph_store,
|
|
130
|
+
self.embedder,
|
|
131
|
+
internet_retriever=self.internet_retriever,
|
|
132
|
+
)
|
|
123
133
|
return searcher.search(query, top_k, info, mode, memory_type)
|
|
124
134
|
|
|
125
135
|
def get_relevant_subgraph(
|
|
@@ -3,6 +3,8 @@ import re
|
|
|
3
3
|
|
|
4
4
|
from datetime import datetime
|
|
5
5
|
|
|
6
|
+
from dateutil import parser
|
|
7
|
+
|
|
6
8
|
from memos.embedders.base import BaseEmbedder
|
|
7
9
|
from memos.graph_dbs.neo4j import Neo4jGraphDB
|
|
8
10
|
from memos.llms.base import BaseLLM
|
|
@@ -133,8 +135,8 @@ class ConflictHandler:
|
|
|
133
135
|
"""
|
|
134
136
|
Hard update: compare updated_at, keep the newer one, overwrite the older one's metadata.
|
|
135
137
|
"""
|
|
136
|
-
time_a =
|
|
137
|
-
time_b =
|
|
138
|
+
time_a = parser.isoparse(memory_a.metadata.updated_at)
|
|
139
|
+
time_b = parser.isoparse(memory_b.metadata.updated_at)
|
|
138
140
|
|
|
139
141
|
newer_mem = memory_a if time_a >= time_b else memory_b
|
|
140
142
|
older_mem = memory_b if time_a >= time_b else memory_a
|
|
@@ -167,10 +169,12 @@ class ConflictHandler:
|
|
|
167
169
|
if not self.graph_store.edge_exists(new_from, new_to, edge["type"], direction="ANY"):
|
|
168
170
|
self.graph_store.add_edge(new_from, new_to, edge["type"])
|
|
169
171
|
|
|
170
|
-
self.graph_store.
|
|
171
|
-
self.graph_store.
|
|
172
|
+
self.graph_store.update_node(conflict_a.id, {"status": "archived"})
|
|
173
|
+
self.graph_store.update_node(conflict_b.id, {"status": "archived"})
|
|
174
|
+
self.graph_store.add_edge(conflict_a.id, merged.id, type="MERGED_TO")
|
|
175
|
+
self.graph_store.add_edge(conflict_b.id, merged.id, type="MERGED_TO")
|
|
172
176
|
logger.debug(
|
|
173
|
-
f"
|
|
177
|
+
f"Archive {conflict_a.id} and {conflict_b.id}, and inherit their edges to {merged.id}."
|
|
174
178
|
)
|
|
175
179
|
|
|
176
180
|
def _merge_metadata(
|
|
@@ -5,7 +5,7 @@ from datetime import datetime
|
|
|
5
5
|
|
|
6
6
|
from memos.embedders.factory import OllamaEmbedder
|
|
7
7
|
from memos.graph_dbs.neo4j import Neo4jGraphDB
|
|
8
|
-
from memos.llms.factory import OllamaLLM, OpenAILLM
|
|
8
|
+
from memos.llms.factory import AzureLLM, OllamaLLM, OpenAILLM
|
|
9
9
|
from memos.log import get_logger
|
|
10
10
|
from memos.memories.textual.item import TextualMemoryItem, TreeNodeTextualMemoryMetadata
|
|
11
11
|
from memos.memories.textual.tree_text_memory.organize.reorganizer import (
|
|
@@ -22,7 +22,7 @@ class MemoryManager:
|
|
|
22
22
|
self,
|
|
23
23
|
graph_store: Neo4jGraphDB,
|
|
24
24
|
embedder: OllamaEmbedder,
|
|
25
|
-
llm: OpenAILLM | OllamaLLM,
|
|
25
|
+
llm: OpenAILLM | OllamaLLM | AzureLLM,
|
|
26
26
|
memory_size: dict | None = None,
|
|
27
27
|
threshold: float | None = 0.80,
|
|
28
28
|
merged_threshold: float | None = 0.92,
|
|
@@ -49,15 +49,18 @@ class MemoryManager:
|
|
|
49
49
|
)
|
|
50
50
|
self._merged_threshold = merged_threshold
|
|
51
51
|
|
|
52
|
-
def add(self, memories: list[TextualMemoryItem]) ->
|
|
52
|
+
def add(self, memories: list[TextualMemoryItem]) -> list[str]:
|
|
53
53
|
"""
|
|
54
54
|
Add new memories in parallel to different memory types (WorkingMemory, LongTermMemory, UserMemory).
|
|
55
55
|
"""
|
|
56
|
+
added_ids: list[str] = []
|
|
57
|
+
|
|
56
58
|
with ThreadPoolExecutor(max_workers=8) as executor:
|
|
57
|
-
futures =
|
|
59
|
+
futures = {executor.submit(self._process_memory, m): m for m in memories}
|
|
58
60
|
for future in as_completed(futures):
|
|
59
61
|
try:
|
|
60
|
-
future.result()
|
|
62
|
+
ids = future.result()
|
|
63
|
+
added_ids.extend(ids)
|
|
61
64
|
except Exception as e:
|
|
62
65
|
logger.exception("Memory processing error: ", exc_info=e)
|
|
63
66
|
|
|
@@ -72,6 +75,7 @@ class MemoryManager:
|
|
|
72
75
|
)
|
|
73
76
|
|
|
74
77
|
self._refresh_memory_size()
|
|
78
|
+
return added_ids
|
|
75
79
|
|
|
76
80
|
def replace_working_memory(self, memories: list[TextualMemoryItem]) -> None:
|
|
77
81
|
"""
|
|
@@ -113,17 +117,23 @@ class MemoryManager:
|
|
|
113
117
|
Process and add memory to different memory types (WorkingMemory, LongTermMemory, UserMemory).
|
|
114
118
|
This method runs asynchronously to process each memory item.
|
|
115
119
|
"""
|
|
120
|
+
ids = []
|
|
121
|
+
|
|
116
122
|
# Add to WorkingMemory
|
|
117
|
-
self._add_memory_to_db(memory, "WorkingMemory")
|
|
123
|
+
working_id = self._add_memory_to_db(memory, "WorkingMemory")
|
|
124
|
+
ids.append(working_id)
|
|
118
125
|
|
|
119
126
|
# Add to LongTermMemory and UserMemory
|
|
120
127
|
if memory.metadata.memory_type in ["LongTermMemory", "UserMemory"]:
|
|
121
|
-
self._add_to_graph_memory(
|
|
128
|
+
added_id = self._add_to_graph_memory(
|
|
122
129
|
memory=memory,
|
|
123
130
|
memory_type=memory.metadata.memory_type,
|
|
124
131
|
)
|
|
132
|
+
ids.append(added_id)
|
|
125
133
|
|
|
126
|
-
|
|
134
|
+
return ids
|
|
135
|
+
|
|
136
|
+
def _add_memory_to_db(self, memory: TextualMemoryItem, memory_type: str) -> str:
|
|
127
137
|
"""
|
|
128
138
|
Add a single memory item to the graph store, with FIFO logic for WorkingMemory.
|
|
129
139
|
"""
|
|
@@ -135,6 +145,7 @@ class MemoryManager:
|
|
|
135
145
|
|
|
136
146
|
# Insert node into graph
|
|
137
147
|
self.graph_store.add_node(working_memory.id, working_memory.memory, metadata)
|
|
148
|
+
return working_memory.id
|
|
138
149
|
|
|
139
150
|
def _add_to_graph_memory(self, memory: TextualMemoryItem, memory_type: str):
|
|
140
151
|
"""
|
|
@@ -159,7 +170,7 @@ class MemoryManager:
|
|
|
159
170
|
)
|
|
160
171
|
|
|
161
172
|
if similar_nodes and similar_nodes[0]["score"] > self._merged_threshold:
|
|
162
|
-
self._merge(memory, similar_nodes)
|
|
173
|
+
return self._merge(memory, similar_nodes)
|
|
163
174
|
else:
|
|
164
175
|
node_id = str(uuid.uuid4())
|
|
165
176
|
# Step 2: Add new node to graph
|
|
@@ -172,8 +183,9 @@ class MemoryManager:
|
|
|
172
183
|
after_node=[node_id],
|
|
173
184
|
)
|
|
174
185
|
)
|
|
186
|
+
return node_id
|
|
175
187
|
|
|
176
|
-
def _merge(self, source_node: TextualMemoryItem, similar_nodes: list[dict]) ->
|
|
188
|
+
def _merge(self, source_node: TextualMemoryItem, similar_nodes: list[dict]) -> str:
|
|
177
189
|
"""
|
|
178
190
|
TODO: Add node traceability support by optionally preserving source nodes and linking them with MERGED_FROM edges.
|
|
179
191
|
|
|
@@ -200,7 +212,9 @@ class MemoryManager:
|
|
|
200
212
|
merged_background = f"{original_meta.background}\n⟵MERGED⟶\n{source_meta.background}"
|
|
201
213
|
merged_embedding = self.embedder.embed([merged_text])[0]
|
|
202
214
|
|
|
203
|
-
|
|
215
|
+
original_conf = original_meta.confidence or 0.0
|
|
216
|
+
source_conf = source_meta.confidence or 0.0
|
|
217
|
+
merged_confidence = float((original_conf + source_conf) / 2)
|
|
204
218
|
merged_usage = list(set((original_meta.usage or []) + (source_meta.usage or [])))
|
|
205
219
|
|
|
206
220
|
# Create new merged node
|
|
@@ -232,13 +246,6 @@ class MemoryManager:
|
|
|
232
246
|
# After creating merged node and tracing lineage
|
|
233
247
|
self._inherit_edges(original_id, merged_id)
|
|
234
248
|
|
|
235
|
-
# Relate other similar nodes to merged if needed
|
|
236
|
-
for related_node in similar_nodes[1:]:
|
|
237
|
-
if not self.graph_store.edge_exists(
|
|
238
|
-
merged_id, related_node["id"], type="ANY", direction="ANY"
|
|
239
|
-
):
|
|
240
|
-
self.graph_store.add_edge(merged_id, related_node["id"], type="RELATE")
|
|
241
|
-
|
|
242
249
|
# log to reorganizer before updating the graph
|
|
243
250
|
self.reorganizer.add_message(
|
|
244
251
|
QueueMessage(
|
|
@@ -250,6 +257,7 @@ class MemoryManager:
|
|
|
250
257
|
after_node=[merged_id],
|
|
251
258
|
)
|
|
252
259
|
)
|
|
260
|
+
return merged_id
|
|
253
261
|
|
|
254
262
|
def _inherit_edges(self, from_id: str, to_id: str) -> None:
|
|
255
263
|
"""
|
|
@@ -30,7 +30,7 @@ class RedundancyHandler:
|
|
|
30
30
|
self, memory: TextualMemoryItem, top_k: int = 5, scope: str | None = None
|
|
31
31
|
) -> list[tuple[TextualMemoryItem, TextualMemoryItem]]:
|
|
32
32
|
"""
|
|
33
|
-
Detect redundancy by finding the most similar items in the graph database based on embedding, then use LLM to judge
|
|
33
|
+
Detect redundancy by finding the most similar items in the graph database based on embedding, then use LLM to judge redundancy.
|
|
34
34
|
Args:
|
|
35
35
|
memory: The memory item (should have an embedding attribute or field).
|
|
36
36
|
top_k: Number of top similar nodes to retrieve.
|
|
@@ -49,7 +49,7 @@ class RedundancyHandler:
|
|
|
49
49
|
for info in embedding_candidates_info
|
|
50
50
|
if info["score"] >= self.EMBEDDING_THRESHOLD and info["id"] != memory.id
|
|
51
51
|
]
|
|
52
|
-
# 3. Judge
|
|
52
|
+
# 3. Judge redundancys using LLM
|
|
53
53
|
embedding_candidates = self.graph_store.get_nodes(embedding_candidates_ids)
|
|
54
54
|
redundant_pairs = []
|
|
55
55
|
for embedding_candidate in embedding_candidates:
|
|
@@ -57,7 +57,7 @@ class RedundancyHandler:
|
|
|
57
57
|
prompt = [
|
|
58
58
|
{
|
|
59
59
|
"role": "system",
|
|
60
|
-
"content": "You are a
|
|
60
|
+
"content": "You are a redundancy detector for memory items.",
|
|
61
61
|
},
|
|
62
62
|
{
|
|
63
63
|
"role": "user",
|
|
@@ -71,12 +71,12 @@ class RedundancyHandler:
|
|
|
71
71
|
if "yes" in result.lower():
|
|
72
72
|
redundant_pairs.append([memory, embedding_candidate])
|
|
73
73
|
if len(redundant_pairs):
|
|
74
|
-
|
|
74
|
+
redundant_text = "\n".join(
|
|
75
75
|
f'"{pair[0].memory!s}" <==REDUNDANCY==> "{pair[1].memory!s}"'
|
|
76
76
|
for pair in redundant_pairs
|
|
77
77
|
)
|
|
78
78
|
logger.warning(
|
|
79
|
-
f"Detected {len(redundant_pairs)} redundancies for memory {memory.id}\n {
|
|
79
|
+
f"Detected {len(redundant_pairs)} redundancies for memory {memory.id}\n {redundant_text}"
|
|
80
80
|
)
|
|
81
81
|
return redundant_pairs
|
|
82
82
|
|
|
@@ -84,12 +84,12 @@ class RedundancyHandler:
|
|
|
84
84
|
"""
|
|
85
85
|
Resolve detected redundancies between two memory items using LLM fusion.
|
|
86
86
|
Args:
|
|
87
|
-
memory_a: The first
|
|
88
|
-
memory_b: The second
|
|
87
|
+
memory_a: The first redundant memory item.
|
|
88
|
+
memory_b: The second redundant memory item.
|
|
89
89
|
Returns:
|
|
90
90
|
A fused TextualMemoryItem representing the resolved memory.
|
|
91
91
|
"""
|
|
92
|
-
|
|
92
|
+
return # waiting for implementation
|
|
93
93
|
# ———————————— 1. LLM generate fused memory ————————————
|
|
94
94
|
metadata_for_resolve = ["key", "background", "confidence", "updated_at"]
|
|
95
95
|
metadata_1 = memory_a.metadata.model_dump_json(include=metadata_for_resolve)
|
|
@@ -115,18 +115,10 @@ class RedundancyHandler:
|
|
|
115
115
|
try:
|
|
116
116
|
answer = re.search(r"<answer>(.*?)</answer>", response, re.DOTALL)
|
|
117
117
|
answer = answer.group(1).strip()
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
)
|
|
123
|
-
self._hard_update(memory_a, memory_b)
|
|
124
|
-
# —————— 2.2 Conflict resolved, update metadata and memory ————
|
|
125
|
-
else:
|
|
126
|
-
fixed_metadata = self._merge_metadata(answer, memory_a.metadata, memory_b.metadata)
|
|
127
|
-
merged_memory = TextualMemoryItem(memory=answer, metadata=fixed_metadata)
|
|
128
|
-
logger.info(f"Resolved result: {merged_memory}")
|
|
129
|
-
self._resolve_in_graph(memory_a, memory_b, merged_memory)
|
|
118
|
+
fixed_metadata = self._merge_metadata(answer, memory_a.metadata, memory_b.metadata)
|
|
119
|
+
merged_memory = TextualMemoryItem(memory=answer, metadata=fixed_metadata)
|
|
120
|
+
logger.info(f"Resolved result: {merged_memory}")
|
|
121
|
+
self._resolve_in_graph(memory_a, memory_b, merged_memory)
|
|
130
122
|
except json.decoder.JSONDecodeError:
|
|
131
123
|
logger.error(f"Failed to parse LLM response: {response}")
|
|
132
124
|
|
|
@@ -145,29 +137,14 @@ class RedundancyHandler:
|
|
|
145
137
|
)
|
|
146
138
|
logger.debug(f"Merged memory: {memory.memory}")
|
|
147
139
|
|
|
148
|
-
def _hard_update(self, memory_a: TextualMemoryItem, memory_b: TextualMemoryItem):
|
|
149
|
-
"""
|
|
150
|
-
Hard update: compare updated_at, keep the newer one, overwrite the older one's metadata.
|
|
151
|
-
"""
|
|
152
|
-
time_a = datetime.fromisoformat(memory_a.metadata.updated_at)
|
|
153
|
-
time_b = datetime.fromisoformat(memory_b.metadata.updated_at)
|
|
154
|
-
|
|
155
|
-
newer_mem = memory_a if time_a >= time_b else memory_b
|
|
156
|
-
older_mem = memory_b if time_a >= time_b else memory_a
|
|
157
|
-
|
|
158
|
-
self.graph_store.delete_node(older_mem.id)
|
|
159
|
-
logger.warning(
|
|
160
|
-
f"Delete older memory {older_mem.id}: <{older_mem.memory}> due to conflict with {newer_mem.id}: <{newer_mem.memory}>"
|
|
161
|
-
)
|
|
162
|
-
|
|
163
140
|
def _resolve_in_graph(
|
|
164
141
|
self,
|
|
165
|
-
|
|
166
|
-
|
|
142
|
+
redundant_a: TextualMemoryItem,
|
|
143
|
+
redundant_b: TextualMemoryItem,
|
|
167
144
|
merged: TextualMemoryItem,
|
|
168
145
|
):
|
|
169
|
-
edges_a = self.graph_store.get_edges(
|
|
170
|
-
edges_b = self.graph_store.get_edges(
|
|
146
|
+
edges_a = self.graph_store.get_edges(redundant_a.id, type="ANY", direction="ANY")
|
|
147
|
+
edges_b = self.graph_store.get_edges(redundant_b.id, type="ANY", direction="ANY")
|
|
171
148
|
all_edges = edges_a + edges_b
|
|
172
149
|
|
|
173
150
|
self.graph_store.add_node(
|
|
@@ -175,18 +152,22 @@ class RedundancyHandler:
|
|
|
175
152
|
)
|
|
176
153
|
|
|
177
154
|
for edge in all_edges:
|
|
178
|
-
new_from =
|
|
179
|
-
|
|
155
|
+
new_from = (
|
|
156
|
+
merged.id if edge["from"] in (redundant_a.id, redundant_b.id) else edge["from"]
|
|
157
|
+
)
|
|
158
|
+
new_to = merged.id if edge["to"] in (redundant_a.id, redundant_b.id) else edge["to"]
|
|
180
159
|
if new_from == new_to:
|
|
181
160
|
continue
|
|
182
161
|
# Check if the edge already exists before adding
|
|
183
162
|
if not self.graph_store.edge_exists(new_from, new_to, edge["type"], direction="ANY"):
|
|
184
163
|
self.graph_store.add_edge(new_from, new_to, edge["type"])
|
|
185
164
|
|
|
186
|
-
self.graph_store.
|
|
187
|
-
self.graph_store.
|
|
165
|
+
self.graph_store.update_node(redundant_a.id, {"status": "archived"})
|
|
166
|
+
self.graph_store.update_node(redundant_b.id, {"status": "archived"})
|
|
167
|
+
self.graph_store.add_edge(redundant_a.id, merged.id, type="MERGED_TO")
|
|
168
|
+
self.graph_store.add_edge(redundant_b.id, merged.id, type="MERGED_TO")
|
|
188
169
|
logger.debug(
|
|
189
|
-
f"
|
|
170
|
+
f"Archive {redundant_a.id} and {redundant_b.id}, and inherit their edges to {merged.id}."
|
|
190
171
|
)
|
|
191
172
|
|
|
192
173
|
def _merge_metadata(
|