synapse-memory-engine 1.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- synapse_memory/__init__.py +31 -0
- synapse_memory/api/fastapi_server.py +1334 -0
- synapse_memory/cli/synapse_cli.py +112 -0
- synapse_memory/client.py +127 -0
- synapse_memory/core/__init__.py +50 -0
- synapse_memory/core/auth.py +1273 -0
- synapse_memory/core/decay_engine.py +123 -0
- synapse_memory/core/deduplicator.py +98 -0
- synapse_memory/core/embedder.py +119 -0
- synapse_memory/core/embedding_provider.py +241 -0
- synapse_memory/core/encryption.py +226 -0
- synapse_memory/core/hybrid_search.py +199 -0
- synapse_memory/core/knapsack_packer.py +103 -0
- synapse_memory/core/memory_manager.py +233 -0
- synapse_memory/core/migrations.py +241 -0
- synapse_memory/core/observability.py +48 -0
- synapse_memory/core/pgvector_db.py +1276 -0
- synapse_memory/core/pruner.py +68 -0
- synapse_memory/core/rate_limiter.py +180 -0
- synapse_memory/core/sqlite_db.py +564 -0
- synapse_memory/examples/langchain_integration.py +12 -0
- synapse_memory/examples/llamaindex_integration.py +12 -0
- synapse_memory/integrations/__init__.py +9 -0
- synapse_memory/integrations/langchain.py +209 -0
- synapse_memory/integrations/llamaindex.py +295 -0
- synapse_memory/main.py +112 -0
- synapse_memory/py.typed +0 -0
- synapse_memory_engine-1.3.0.dist-info/METADATA +75 -0
- synapse_memory_engine-1.3.0.dist-info/RECORD +32 -0
- synapse_memory_engine-1.3.0.dist-info/WHEEL +5 -0
- synapse_memory_engine-1.3.0.dist-info/entry_points.txt +4 -0
- synapse_memory_engine-1.3.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,209 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Production-grade LangChain drop-in memory integration for Synapse Memory.
|
|
3
|
+
Compliant with langchain_core.memory.BaseMemory.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from typing import Dict, Any, List, Optional
|
|
7
|
+
import time
|
|
8
|
+
import logging
|
|
9
|
+
|
|
10
|
+
from synapse_memory.core.sqlite_db import SQLiteMemoryStore
|
|
11
|
+
from synapse_memory.core.embedder import SynapseEmbedder
|
|
12
|
+
from synapse_memory.core.memory_manager import MemoryManager
|
|
13
|
+
from synapse_memory.core.hybrid_search import HybridSearch
|
|
14
|
+
from synapse_memory.core.knapsack_packer import KnapsackPacker
|
|
15
|
+
from synapse_memory.core.decay_engine import DecayEngine
|
|
16
|
+
|
|
17
|
+
logger = logging.getLogger("SynapseLangChain")
|
|
18
|
+
|
|
19
|
+
# Dynamic import for langchain_core with full schema parity fallback
|
|
20
|
+
try:
|
|
21
|
+
from langchain_core.memory import BaseMemory
|
|
22
|
+
from langchain_core.messages import HumanMessage, AIMessage, BaseMessage
|
|
23
|
+
HAS_LANGCHAIN = True
|
|
24
|
+
except ImportError:
|
|
25
|
+
class BaseMemory: # type: ignore[no-redef]
|
|
26
|
+
pass
|
|
27
|
+
|
|
28
|
+
class BaseMessage: # type: ignore[no-redef]
|
|
29
|
+
def __init__(self, content: str):
|
|
30
|
+
self.content = content
|
|
31
|
+
|
|
32
|
+
class HumanMessage(BaseMessage): # type: ignore[no-redef]
|
|
33
|
+
pass
|
|
34
|
+
|
|
35
|
+
class AIMessage(BaseMessage): # type: ignore[no-redef]
|
|
36
|
+
pass
|
|
37
|
+
|
|
38
|
+
HAS_LANGCHAIN = False
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class SynapseLangChainMemory(BaseMemory):
|
|
42
|
+
"""
|
|
43
|
+
Drop-in production memory module for LangChain chains and agents.
|
|
44
|
+
Inherits directly from langchain_core.memory.BaseMemory when available.
|
|
45
|
+
|
|
46
|
+
Features:
|
|
47
|
+
- Backed by persistent encrypted SQLite store.
|
|
48
|
+
- Automatic semantic deduplication on save_context.
|
|
49
|
+
- Real hybrid BM25 + dense vector search on load_memory_variables.
|
|
50
|
+
- Ebbinghaus continuous decay scoring.
|
|
51
|
+
- Exact Knapsack token budget packing into max_token_budget.
|
|
52
|
+
- Support for both string-formatted history and message list formats.
|
|
53
|
+
"""
|
|
54
|
+
|
|
55
|
+
memory_key: str = "chat_history"
|
|
56
|
+
input_key: Optional[str] = None
|
|
57
|
+
output_key: Optional[str] = None
|
|
58
|
+
return_messages: bool = False
|
|
59
|
+
human_prefix: str = "Human"
|
|
60
|
+
ai_prefix: str = "AI"
|
|
61
|
+
max_token_budget: int = 2048
|
|
62
|
+
|
|
63
|
+
def __init__(
|
|
64
|
+
self,
|
|
65
|
+
memory_key: str = "chat_history",
|
|
66
|
+
input_key: Optional[str] = None,
|
|
67
|
+
output_key: Optional[str] = None,
|
|
68
|
+
return_messages: bool = False,
|
|
69
|
+
human_prefix: str = "Human",
|
|
70
|
+
ai_prefix: str = "AI",
|
|
71
|
+
max_token_budget: int = 2048,
|
|
72
|
+
db_path: str = ":memory:",
|
|
73
|
+
store: Optional[SQLiteMemoryStore] = None,
|
|
74
|
+
embedder: Optional[SynapseEmbedder] = None,
|
|
75
|
+
memory_manager: Optional[MemoryManager] = None,
|
|
76
|
+
**kwargs: Any
|
|
77
|
+
):
|
|
78
|
+
if HAS_LANGCHAIN:
|
|
79
|
+
super().__init__(**kwargs)
|
|
80
|
+
|
|
81
|
+
self.memory_key = memory_key
|
|
82
|
+
self.input_key = input_key
|
|
83
|
+
self.output_key = output_key
|
|
84
|
+
self.return_messages = return_messages
|
|
85
|
+
self.human_prefix = human_prefix
|
|
86
|
+
self.ai_prefix = ai_prefix
|
|
87
|
+
self.max_token_budget = max_token_budget
|
|
88
|
+
|
|
89
|
+
# Initialize underlying Synapse core substrate
|
|
90
|
+
self.store = store or SQLiteMemoryStore(db_path=db_path)
|
|
91
|
+
self.embedder = embedder or SynapseEmbedder(provider="local")
|
|
92
|
+
self.manager = memory_manager or MemoryManager(
|
|
93
|
+
store=self.store,
|
|
94
|
+
embedder=self.embedder,
|
|
95
|
+
max_record_limit=kwargs.get("max_record_limit", 500)
|
|
96
|
+
)
|
|
97
|
+
self.searcher = HybridSearch(alpha=0.6, k=60)
|
|
98
|
+
self.packer = KnapsackPacker(token_budget=max_token_budget)
|
|
99
|
+
self.decay = DecayEngine()
|
|
100
|
+
|
|
101
|
+
@property
|
|
102
|
+
def memory_variables(self) -> List[str]:
|
|
103
|
+
"""Exposes memory variables to LangChain prompt templates."""
|
|
104
|
+
return [self.memory_key]
|
|
105
|
+
|
|
106
|
+
def _get_input_output(
|
|
107
|
+
self, inputs: Dict[str, Any], outputs: Dict[str, Any]
|
|
108
|
+
) -> tuple[str, str]:
|
|
109
|
+
if self.input_key:
|
|
110
|
+
user_input = inputs.get(self.input_key, "")
|
|
111
|
+
else:
|
|
112
|
+
prompt_keys = [k for k in inputs.keys() if k not in self.memory_variables]
|
|
113
|
+
user_input = str(inputs[prompt_keys[0]]) if prompt_keys else ""
|
|
114
|
+
|
|
115
|
+
if self.output_key:
|
|
116
|
+
model_output = outputs.get(self.output_key, "")
|
|
117
|
+
else:
|
|
118
|
+
out_keys = list(outputs.keys())
|
|
119
|
+
model_output = str(outputs[out_keys[0]]) if out_keys else ""
|
|
120
|
+
|
|
121
|
+
return str(user_input), str(model_output)
|
|
122
|
+
|
|
123
|
+
def load_memory_variables(self, inputs: Dict[str, Any]) -> Dict[str, Any]:
|
|
124
|
+
"""
|
|
125
|
+
Retrieves relevant historical memories, applies Ebbinghaus decay relevance scores,
|
|
126
|
+
solves context limits via Knapsack DP, and returns formatted context.
|
|
127
|
+
"""
|
|
128
|
+
query_text = (
|
|
129
|
+
inputs.get("input")
|
|
130
|
+
or inputs.get("question")
|
|
131
|
+
or inputs.get("query")
|
|
132
|
+
or (inputs.get(self.input_key) if self.input_key else None)
|
|
133
|
+
or ""
|
|
134
|
+
)
|
|
135
|
+
|
|
136
|
+
all_memories = self.store.get_all_memories()
|
|
137
|
+
if not all_memories:
|
|
138
|
+
return {self.memory_key: [] if self.return_messages else ""}
|
|
139
|
+
|
|
140
|
+
# If query is provided, perform full Hybrid Search
|
|
141
|
+
if query_text.strip():
|
|
142
|
+
query_vector = self.embedder.embed_query(query_text)
|
|
143
|
+
ranked_nodes = self.searcher.search(
|
|
144
|
+
query=query_text,
|
|
145
|
+
corpus=all_memories,
|
|
146
|
+
query_vector=query_vector,
|
|
147
|
+
top_k=min(len(all_memories), 20)
|
|
148
|
+
)
|
|
149
|
+
else:
|
|
150
|
+
# Fallback to recency order if no active query
|
|
151
|
+
ranked_nodes = sorted(
|
|
152
|
+
all_memories,
|
|
153
|
+
key=lambda m: m.get("created_at", 0),
|
|
154
|
+
reverse=True
|
|
155
|
+
)[:20]
|
|
156
|
+
|
|
157
|
+
# Apply continuous temporal decay relevance scoring
|
|
158
|
+
now = time.time()
|
|
159
|
+
for node in ranked_nodes:
|
|
160
|
+
base_score = float(node.get("score", node.get("confidence", 0.8)))
|
|
161
|
+
node["relevance_score"] = self.decay.calculate_relevance(node, now=now) * base_score
|
|
162
|
+
|
|
163
|
+
# Solve Knapsack token constraint
|
|
164
|
+
packed_memories = self.packer.pack(ranked_nodes, token_budget=self.max_token_budget)
|
|
165
|
+
|
|
166
|
+
# Update access counters
|
|
167
|
+
for node in packed_memories:
|
|
168
|
+
self.store.update_access_count(node["id"], 1)
|
|
169
|
+
|
|
170
|
+
# Format output as messages or formatted string
|
|
171
|
+
if self.return_messages:
|
|
172
|
+
messages: List[Any] = []
|
|
173
|
+
for node in packed_memories:
|
|
174
|
+
content = node.get("content", "")
|
|
175
|
+
if " -> " in content and (self.human_prefix in content or "User:" in content):
|
|
176
|
+
parts = content.split(" -> ", 1)
|
|
177
|
+
u_text = parts[0].replace(f"{self.human_prefix}: ", "").replace("User: ", "")
|
|
178
|
+
a_text = parts[1].replace(f"{self.ai_prefix}: ", "").replace("Agent: ", "").replace("AI: ", "")
|
|
179
|
+
messages.append(HumanMessage(content=u_text))
|
|
180
|
+
messages.append(AIMessage(content=a_text))
|
|
181
|
+
else:
|
|
182
|
+
messages.append(HumanMessage(content=content))
|
|
183
|
+
return {self.memory_key: messages}
|
|
184
|
+
else:
|
|
185
|
+
formatted_nodes = [f"[{node.get('category', 'context').upper()}]: {node.get('content', '')}" for node in packed_memories]
|
|
186
|
+
return {self.memory_key: "\n".join(formatted_nodes)}
|
|
187
|
+
|
|
188
|
+
def save_context(self, inputs: Dict[str, Any], outputs: Dict[str, Any]) -> None:
|
|
189
|
+
"""
|
|
190
|
+
Saves user/assistant conversation turn into encrypted memory store with deduplication.
|
|
191
|
+
"""
|
|
192
|
+
user_input, model_output = self._get_input_output(inputs, outputs)
|
|
193
|
+
if not user_input and not model_output:
|
|
194
|
+
return
|
|
195
|
+
|
|
196
|
+
combined_content = f"{self.human_prefix}: {user_input} -> {self.ai_prefix}: {model_output}"
|
|
197
|
+
|
|
198
|
+
# Ingest through memory manager to trigger semantic deduplication and auto-pruning
|
|
199
|
+
self.manager.ingest_with_deduplication(
|
|
200
|
+
content=combined_content,
|
|
201
|
+
category="conversation",
|
|
202
|
+
confidence=0.95
|
|
203
|
+
)
|
|
204
|
+
|
|
205
|
+
def clear(self) -> None:
|
|
206
|
+
"""Clears all stored memories."""
|
|
207
|
+
all_mems = self.store.get_all_memories()
|
|
208
|
+
for mem in all_mems:
|
|
209
|
+
self.store.delete_memory(mem["id"])
|
|
@@ -0,0 +1,295 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Production-grade LlamaIndex drop-in memory integration for Synapse Memory.
|
|
3
|
+
Compliant with llama_index.core.memory.BaseMemory.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from typing import Dict, Any, List, Optional
|
|
7
|
+
import time
|
|
8
|
+
import logging
|
|
9
|
+
|
|
10
|
+
from synapse_memory.core.sqlite_db import SQLiteMemoryStore
|
|
11
|
+
from synapse_memory.core.embedder import SynapseEmbedder
|
|
12
|
+
from synapse_memory.core.memory_manager import MemoryManager
|
|
13
|
+
from synapse_memory.core.hybrid_search import HybridSearch
|
|
14
|
+
from synapse_memory.core.knapsack_packer import KnapsackPacker
|
|
15
|
+
from synapse_memory.core.decay_engine import DecayEngine
|
|
16
|
+
|
|
17
|
+
logger = logging.getLogger("SynapseLlamaIndex")
|
|
18
|
+
|
|
19
|
+
try:
|
|
20
|
+
from llama_index.core.memory import BaseMemory
|
|
21
|
+
from llama_index.core.llms import ChatMessage, MessageRole
|
|
22
|
+
from llama_index.core.bridge.pydantic import Field, PrivateAttr
|
|
23
|
+
HAS_LLAMAINDEX = True
|
|
24
|
+
except ImportError:
|
|
25
|
+
class BaseMemory: # type: ignore[no-redef] # noqa: F811
|
|
26
|
+
pass
|
|
27
|
+
|
|
28
|
+
def Field(default: Any = None, **kwargs: Any) -> Any: # type: ignore[no-redef] # noqa: F811
|
|
29
|
+
return default
|
|
30
|
+
|
|
31
|
+
def PrivateAttr(default: Any = None, **kwargs: Any) -> Any: # type: ignore[no-redef] # noqa: F811
|
|
32
|
+
return default
|
|
33
|
+
|
|
34
|
+
class MessageRole: # type: ignore[no-redef] # noqa: F811
|
|
35
|
+
USER = "user"
|
|
36
|
+
ASSISTANT = "assistant"
|
|
37
|
+
SYSTEM = "system"
|
|
38
|
+
|
|
39
|
+
class ChatMessage: # type: ignore[no-redef] # noqa: F811
|
|
40
|
+
def __init__(
|
|
41
|
+
self,
|
|
42
|
+
role: Any = MessageRole.USER,
|
|
43
|
+
content: str = "",
|
|
44
|
+
additional_kwargs: Optional[Dict[str, Any]] = None,
|
|
45
|
+
):
|
|
46
|
+
self.role = role
|
|
47
|
+
self.content = content
|
|
48
|
+
self.additional_kwargs = additional_kwargs or {}
|
|
49
|
+
|
|
50
|
+
def __repr__(self) -> str:
|
|
51
|
+
return f"ChatMessage(role={self.role}, content={self.content[:30]}...)"
|
|
52
|
+
|
|
53
|
+
HAS_LLAMAINDEX = False
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
class SynapseLlamaIndexMemory(BaseMemory):
|
|
57
|
+
"""
|
|
58
|
+
Drop-in production memory module for LlamaIndex query and chat engines.
|
|
59
|
+
Inherits directly from llama_index.core.memory.BaseMemory when available.
|
|
60
|
+
|
|
61
|
+
Features:
|
|
62
|
+
- Backed by persistent encrypted SQLite store.
|
|
63
|
+
- Automatic semantic deduplication on message ingestion.
|
|
64
|
+
- Real hybrid BM25 + dense vector retrieval.
|
|
65
|
+
- Knapsack token packing within max_token_limit.
|
|
66
|
+
- Full ChatMessage role preservation.
|
|
67
|
+
"""
|
|
68
|
+
|
|
69
|
+
max_token_limit: int = Field(default=3072)
|
|
70
|
+
db_path: str = Field(default=":memory:")
|
|
71
|
+
max_record_limit: int = Field(default=500)
|
|
72
|
+
|
|
73
|
+
_store: Optional[SQLiteMemoryStore] = PrivateAttr(default=None)
|
|
74
|
+
_embedder: Optional[SynapseEmbedder] = PrivateAttr(default=None)
|
|
75
|
+
_manager: Optional[MemoryManager] = PrivateAttr(default=None)
|
|
76
|
+
_searcher: Optional[HybridSearch] = PrivateAttr(default=None)
|
|
77
|
+
_packer: Optional[KnapsackPacker] = PrivateAttr(default=None)
|
|
78
|
+
_decay: Optional[DecayEngine] = PrivateAttr(default=None)
|
|
79
|
+
|
|
80
|
+
def __init__(
|
|
81
|
+
self,
|
|
82
|
+
max_token_limit: int = 3072,
|
|
83
|
+
db_path: str = ":memory:",
|
|
84
|
+
store: Optional[SQLiteMemoryStore] = None,
|
|
85
|
+
embedder: Optional[SynapseEmbedder] = None,
|
|
86
|
+
memory_manager: Optional[MemoryManager] = None,
|
|
87
|
+
max_record_limit: int = 500,
|
|
88
|
+
**kwargs: Any
|
|
89
|
+
):
|
|
90
|
+
if HAS_LLAMAINDEX:
|
|
91
|
+
super().__init__(
|
|
92
|
+
max_token_limit=max_token_limit,
|
|
93
|
+
db_path=db_path,
|
|
94
|
+
max_record_limit=max_record_limit,
|
|
95
|
+
**kwargs,
|
|
96
|
+
)
|
|
97
|
+
else:
|
|
98
|
+
self.max_token_limit = max_token_limit
|
|
99
|
+
self.db_path = db_path
|
|
100
|
+
self.max_record_limit = max_record_limit
|
|
101
|
+
|
|
102
|
+
self._store = store or SQLiteMemoryStore(db_path=db_path)
|
|
103
|
+
self._embedder = embedder or SynapseEmbedder(provider="local")
|
|
104
|
+
self._manager = memory_manager or MemoryManager(
|
|
105
|
+
store=self.store,
|
|
106
|
+
embedder=self.embedder,
|
|
107
|
+
max_record_limit=kwargs.get("max_record_limit", max_record_limit),
|
|
108
|
+
)
|
|
109
|
+
self._searcher = HybridSearch(alpha=0.6, k=60)
|
|
110
|
+
self._packer = KnapsackPacker(token_budget=max_token_limit)
|
|
111
|
+
self._decay = DecayEngine()
|
|
112
|
+
|
|
113
|
+
@property
|
|
114
|
+
def store(self) -> SQLiteMemoryStore:
|
|
115
|
+
store = self._store
|
|
116
|
+
assert store is not None
|
|
117
|
+
return store
|
|
118
|
+
|
|
119
|
+
@store.setter
|
|
120
|
+
def store(self, value: SQLiteMemoryStore) -> None:
|
|
121
|
+
self._store = value
|
|
122
|
+
|
|
123
|
+
@property
|
|
124
|
+
def embedder(self) -> SynapseEmbedder:
|
|
125
|
+
embedder = self._embedder
|
|
126
|
+
assert embedder is not None
|
|
127
|
+
return embedder
|
|
128
|
+
|
|
129
|
+
@embedder.setter
|
|
130
|
+
def embedder(self, value: SynapseEmbedder) -> None:
|
|
131
|
+
self._embedder = value
|
|
132
|
+
|
|
133
|
+
@property
|
|
134
|
+
def manager(self) -> MemoryManager:
|
|
135
|
+
manager = self._manager
|
|
136
|
+
assert manager is not None
|
|
137
|
+
return manager
|
|
138
|
+
|
|
139
|
+
@manager.setter
|
|
140
|
+
def manager(self, value: MemoryManager) -> None:
|
|
141
|
+
self._manager = value
|
|
142
|
+
|
|
143
|
+
@property
|
|
144
|
+
def searcher(self) -> HybridSearch:
|
|
145
|
+
searcher = self._searcher
|
|
146
|
+
assert searcher is not None
|
|
147
|
+
return searcher
|
|
148
|
+
|
|
149
|
+
@searcher.setter
|
|
150
|
+
def searcher(self, value: HybridSearch) -> None:
|
|
151
|
+
self._searcher = value
|
|
152
|
+
|
|
153
|
+
@property
|
|
154
|
+
def packer(self) -> KnapsackPacker:
|
|
155
|
+
packer = self._packer
|
|
156
|
+
assert packer is not None
|
|
157
|
+
return packer
|
|
158
|
+
|
|
159
|
+
@packer.setter
|
|
160
|
+
def packer(self, value: KnapsackPacker) -> None:
|
|
161
|
+
self._packer = value
|
|
162
|
+
|
|
163
|
+
@property
|
|
164
|
+
def decay(self) -> DecayEngine:
|
|
165
|
+
decay = self._decay
|
|
166
|
+
assert decay is not None
|
|
167
|
+
return decay
|
|
168
|
+
|
|
169
|
+
@decay.setter
|
|
170
|
+
def decay(self, value: DecayEngine) -> None:
|
|
171
|
+
self._decay = value
|
|
172
|
+
|
|
173
|
+
@classmethod
|
|
174
|
+
def from_defaults(cls, **kwargs: Any) -> "SynapseLlamaIndexMemory":
|
|
175
|
+
"""Create a memory instance from defaults (llama_index BaseMemory contract)."""
|
|
176
|
+
chat_history = kwargs.pop("chat_history", None)
|
|
177
|
+
instance = cls(**kwargs)
|
|
178
|
+
if chat_history:
|
|
179
|
+
for msg in chat_history:
|
|
180
|
+
instance.put(msg)
|
|
181
|
+
return instance
|
|
182
|
+
|
|
183
|
+
@staticmethod
|
|
184
|
+
def _role_value(role: Any) -> str:
|
|
185
|
+
value = getattr(role, "value", role)
|
|
186
|
+
return str(value) if value is not None else "user"
|
|
187
|
+
|
|
188
|
+
@staticmethod
|
|
189
|
+
def _content_text(content: Any) -> str:
|
|
190
|
+
if isinstance(content, str):
|
|
191
|
+
return content
|
|
192
|
+
if isinstance(content, list):
|
|
193
|
+
parts: List[str] = []
|
|
194
|
+
for block in content:
|
|
195
|
+
if isinstance(block, str):
|
|
196
|
+
parts.append(block)
|
|
197
|
+
elif isinstance(block, dict) and "text" in block:
|
|
198
|
+
parts.append(str(block["text"]))
|
|
199
|
+
else:
|
|
200
|
+
text = getattr(block, "text", None)
|
|
201
|
+
parts.append(str(text) if text is not None else str(block))
|
|
202
|
+
return "".join(parts)
|
|
203
|
+
return str(content) if content is not None else ""
|
|
204
|
+
|
|
205
|
+
def _to_canonical_role(self, role: Any) -> str:
|
|
206
|
+
normalized = self._role_value(role).lower()
|
|
207
|
+
if normalized in ("user", "assistant", "system"):
|
|
208
|
+
return normalized
|
|
209
|
+
return "user"
|
|
210
|
+
|
|
211
|
+
def get(self, input: Optional[str] = None, **kwargs: Any) -> List[ChatMessage]: # type: ignore[override]
|
|
212
|
+
"""
|
|
213
|
+
Retrieves relevant historical chat messages, applying hybrid search and knapsack packing.
|
|
214
|
+
"""
|
|
215
|
+
all_memories = self.store.get_all_memories()
|
|
216
|
+
if not all_memories:
|
|
217
|
+
return []
|
|
218
|
+
|
|
219
|
+
if input and input.strip():
|
|
220
|
+
query_vector = self.embedder.embed_query(input)
|
|
221
|
+
ranked_nodes = self.searcher.search(
|
|
222
|
+
query=input,
|
|
223
|
+
corpus=all_memories,
|
|
224
|
+
query_vector=query_vector,
|
|
225
|
+
top_k=min(len(all_memories), 20)
|
|
226
|
+
)
|
|
227
|
+
else:
|
|
228
|
+
ranked_nodes = sorted(
|
|
229
|
+
all_memories,
|
|
230
|
+
key=lambda m: m.get("created_at", 0),
|
|
231
|
+
reverse=True
|
|
232
|
+
)[:20]
|
|
233
|
+
|
|
234
|
+
now = time.time()
|
|
235
|
+
for node in ranked_nodes:
|
|
236
|
+
base_score = float(node.get("score", node.get("confidence", 0.8)))
|
|
237
|
+
node["relevance_score"] = self.decay.calculate_relevance(node, now=now) * base_score
|
|
238
|
+
|
|
239
|
+
packed_memories = self.packer.pack(ranked_nodes, token_budget=self.max_token_limit)
|
|
240
|
+
|
|
241
|
+
messages: List[ChatMessage] = []
|
|
242
|
+
for node in packed_memories:
|
|
243
|
+
self.store.update_access_count(node["id"], 1)
|
|
244
|
+
content = node.get("content", "")
|
|
245
|
+
role = self._to_canonical_role(node.get("category", "user"))
|
|
246
|
+
messages.append(ChatMessage(role=role, content=content)) # type: ignore[arg-type]
|
|
247
|
+
|
|
248
|
+
return messages
|
|
249
|
+
|
|
250
|
+
def get_all(self) -> List[ChatMessage]:
|
|
251
|
+
"""Returns all messages currently persisted in the memory store."""
|
|
252
|
+
all_memories = sorted(
|
|
253
|
+
self.store.get_all_memories(),
|
|
254
|
+
key=lambda m: m.get("created_at", 0),
|
|
255
|
+
)
|
|
256
|
+
messages: List[ChatMessage] = []
|
|
257
|
+
for node in all_memories:
|
|
258
|
+
role = self._to_canonical_role(node.get("category", "user"))
|
|
259
|
+
messages.append(ChatMessage(role=role, content=node.get("content", ""))) # type: ignore[arg-type]
|
|
260
|
+
return messages
|
|
261
|
+
|
|
262
|
+
def put(self, message: Any) -> None: # type: ignore[override]
|
|
263
|
+
"""
|
|
264
|
+
Stores a ChatMessage (or plain text) into the encrypted store with semantic deduplication.
|
|
265
|
+
"""
|
|
266
|
+
if isinstance(message, ChatMessage):
|
|
267
|
+
content = self._content_text(message.content)
|
|
268
|
+
role = self._to_canonical_role(message.role)
|
|
269
|
+
elif isinstance(message, dict):
|
|
270
|
+
content = self._content_text(message.get("content", ""))
|
|
271
|
+
role = self._to_canonical_role(message.get("role", "user"))
|
|
272
|
+
else:
|
|
273
|
+
content = self._content_text(message)
|
|
274
|
+
role = "user"
|
|
275
|
+
|
|
276
|
+
if not content.strip():
|
|
277
|
+
return
|
|
278
|
+
|
|
279
|
+
self.manager.ingest_with_deduplication(
|
|
280
|
+
content=content,
|
|
281
|
+
category=role,
|
|
282
|
+
confidence=0.95
|
|
283
|
+
)
|
|
284
|
+
|
|
285
|
+
def set(self, messages: List[Any]) -> None: # type: ignore[override]
|
|
286
|
+
"""Resets the store and inserts the provided message sequence."""
|
|
287
|
+
self.reset()
|
|
288
|
+
for msg in messages:
|
|
289
|
+
self.put(msg)
|
|
290
|
+
|
|
291
|
+
def reset(self) -> None: # type: ignore[override]
|
|
292
|
+
"""Clears all stored memories."""
|
|
293
|
+
all_mems = self.store.get_all_memories()
|
|
294
|
+
for mem in all_mems:
|
|
295
|
+
self.store.delete_memory(mem["id"])
|
synapse_memory/main.py
ADDED
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
import time
|
|
2
|
+
from synapse_memory.core.knapsack_packer import KnapsackPacker
|
|
3
|
+
from synapse_memory.core.decay_engine import DecayEngine
|
|
4
|
+
from synapse_memory.core.hybrid_search import HybridSearch
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def run_production_demo():
|
|
8
|
+
print("=" * 70)
|
|
9
|
+
print(" SYNAPSEMEMORY: UNIVERSAL ACTIVE RAG CORE DEMO & TESTS")
|
|
10
|
+
print("=" * 70)
|
|
11
|
+
|
|
12
|
+
# 1. Initialize Engines
|
|
13
|
+
packer = KnapsackPacker()
|
|
14
|
+
decay = DecayEngine()
|
|
15
|
+
search = HybridSearch()
|
|
16
|
+
|
|
17
|
+
print("\n[+] Engines Initialized Successfully:")
|
|
18
|
+
print(" - Knapsack Packer: Adaptive 0/1 Dynamic Programming Solver Active")
|
|
19
|
+
print(" - Temporal Decay: Ebbinghaus Stabilization Curve Active")
|
|
20
|
+
print(" - Hybrid Search: Lexical/Semantic Reciprocal Fusion Active")
|
|
21
|
+
|
|
22
|
+
# 2. Ingest Sample Memory Base representing typical user developer workspace config
|
|
23
|
+
database = [
|
|
24
|
+
{
|
|
25
|
+
"id": "mem_01",
|
|
26
|
+
"content": "User prefers React with TypeScript and uses Vite with Tailwind CSS for layouts. Absolutely bans inline style tags.",
|
|
27
|
+
"category": "preference",
|
|
28
|
+
"confidence": 0.98,
|
|
29
|
+
"created_at": time.time() - 3600, # 1 hour ago
|
|
30
|
+
"access_count": 0,
|
|
31
|
+
"token_cost": 35
|
|
32
|
+
},
|
|
33
|
+
{
|
|
34
|
+
"id": "mem_02",
|
|
35
|
+
"content": "AetherLab database uses PostgreSQL hosted on GCP Cloud SQL with pgvector extensions configured.",
|
|
36
|
+
"category": "infrastructure",
|
|
37
|
+
"confidence": 0.95,
|
|
38
|
+
"created_at": time.time() - 86400 * 2, # 2 days ago (should show significant temporal decay)
|
|
39
|
+
"access_count": 0,
|
|
40
|
+
"token_cost": 40
|
|
41
|
+
},
|
|
42
|
+
{
|
|
43
|
+
"id": "mem_03",
|
|
44
|
+
"content": "Server is listening exclusively on port 3000 behind an Nginx reverse proxy layer.",
|
|
45
|
+
"category": "infrastructure",
|
|
46
|
+
"confidence": 0.96,
|
|
47
|
+
"created_at": time.time(), # just now (high freshness)
|
|
48
|
+
"access_count": 0,
|
|
49
|
+
"token_cost": 25
|
|
50
|
+
},
|
|
51
|
+
{
|
|
52
|
+
"id": "mem_04",
|
|
53
|
+
"content": "User hates purple-to-blue gradients in UI design. Prefers highly-polished warm or cool slate neutrals.",
|
|
54
|
+
"category": "preference",
|
|
55
|
+
"confidence": 0.92,
|
|
56
|
+
"created_at": time.time() - 10, # just now
|
|
57
|
+
"access_count": 0,
|
|
58
|
+
"token_cost": 30
|
|
59
|
+
}
|
|
60
|
+
]
|
|
61
|
+
|
|
62
|
+
print(f"\n[+] Ingested {len(database)} Sample Cognitive Nodes into Memory Store.")
|
|
63
|
+
|
|
64
|
+
# 3. Simulate Query and Dynamic Filtering
|
|
65
|
+
query = "What framework and styling settings should we use for database infrastructure?"
|
|
66
|
+
print(f"\n[?] User Prompt: '{query}'")
|
|
67
|
+
|
|
68
|
+
print("\n[~] Execution Phase 1: Running Hybrid Semantic/Lexical Retrieval...")
|
|
69
|
+
candidates = search.fused_search(query, database, top_k=4)
|
|
70
|
+
|
|
71
|
+
print("\n[~] Execution Phase 2: Applying Ebbinghaus Temporal Forgetting Curve Weights...")
|
|
72
|
+
decayed_candidates = []
|
|
73
|
+
for c in candidates:
|
|
74
|
+
original_score = c["relevance_score"]
|
|
75
|
+
current_score = decay.calculate_retention(
|
|
76
|
+
base_relevance=original_score,
|
|
77
|
+
created_epoch=c["created_at"],
|
|
78
|
+
access_count=c["access_count"]
|
|
79
|
+
)
|
|
80
|
+
c_copy = c.copy()
|
|
81
|
+
c_copy["relevance_score"] = current_score
|
|
82
|
+
decayed_candidates.append(c_copy)
|
|
83
|
+
|
|
84
|
+
# Display computed weight variations
|
|
85
|
+
print(f" - Node {c['id']}: Orig Search Relevance={original_score:.3f} | Current Decay Score={current_score:.3f}")
|
|
86
|
+
|
|
87
|
+
# 4. Solve Knapsack token allocation to fit within a strict token context budget
|
|
88
|
+
token_budget = 70 # deliberately tight budget to force knapsack decision optimization
|
|
89
|
+
print(f"\n[~] Execution Phase 3: Solving Knapsack DP Constraint (Budget: {token_budget} tokens)...")
|
|
90
|
+
packed_memories = packer.pack(decayed_candidates, token_budget)
|
|
91
|
+
|
|
92
|
+
print("\n" + "-" * 70)
|
|
93
|
+
print(" SYNAPSEMEMORY CONTEXT BUDGET ALLOCATION REPORT")
|
|
94
|
+
print("-" * 70)
|
|
95
|
+
print(f"Total Database Candidates Size : {sum(d['token_cost'] for d in database)} tokens")
|
|
96
|
+
print(f"Target Prompt Context Budget : {token_budget} tokens")
|
|
97
|
+
print(f"Optimal Allocation Pack Size : {sum(m['token_cost'] for m in packed_memories)} tokens")
|
|
98
|
+
print(f"Total Overhead Costs Saved : {sum(d['token_cost'] for d in database) - sum(m['token_cost'] for m in packed_memories)} tokens")
|
|
99
|
+
print("-" * 70)
|
|
100
|
+
|
|
101
|
+
print("\n[+] Selected Context Pack Nodes:")
|
|
102
|
+
for idx, node in enumerate(packed_memories):
|
|
103
|
+
print(f" {idx + 1}. [{node['id']}] Category: {node['category'].upper()} | Cost: {node['token_cost']} tokens | Final Relevance: {node['relevance_score']:.3f}")
|
|
104
|
+
print(f" Content: \"{node['content']}\"")
|
|
105
|
+
|
|
106
|
+
print("\n" + "=" * 70)
|
|
107
|
+
print(" DEMONSTRATION RUN COMPLETE - STATUS: PRODUCTION READY")
|
|
108
|
+
print("=" * 70)
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
if __name__ == "__main__":
|
|
112
|
+
run_production_demo()
|
synapse_memory/py.typed
ADDED
|
File without changes
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: synapse-memory-engine
|
|
3
|
+
Version: 1.3.0
|
|
4
|
+
Summary: Production-grade active RAG & cognitive memory engine with multi-tenancy, pgvector HNSW indexing, semantic deduplication, decay-based pruning, and token-aware context packing.
|
|
5
|
+
Home-page: https://github.com/721189/Synapse-Memory
|
|
6
|
+
Author: Synapse Cognitive Systems
|
|
7
|
+
Author-email: Synapse Cognitive Systems <singhshivam20009@gmail.com>
|
|
8
|
+
Project-URL: Homepage, https://github.com/721189/Synapse-Memory
|
|
9
|
+
Project-URL: Bug Tracker, https://github.com/721189/Synapse-Memory/issues
|
|
10
|
+
Project-URL: Documentation, https://github.com/721189/Synapse-Memory#readme
|
|
11
|
+
Project-URL: Repository, https://github.com/721189/Synapse-Memory.git
|
|
12
|
+
Requires-Python: >=3.10
|
|
13
|
+
Description-Content-Type: text/markdown
|
|
14
|
+
Requires-Dist: fastapi>=0.115.0
|
|
15
|
+
Requires-Dist: pydantic>=2.10.0
|
|
16
|
+
Requires-Dist: uvicorn>=0.34.0
|
|
17
|
+
Requires-Dist: cryptography>=42.0.0
|
|
18
|
+
Requires-Dist: requests>=2.31.0
|
|
19
|
+
Requires-Dist: numpy>=1.26.0
|
|
20
|
+
Requires-Dist: urllib3>=2.8.0
|
|
21
|
+
Provides-Extra: local-models
|
|
22
|
+
Requires-Dist: sentence-transformers>=5.0.0; extra == "local-models"
|
|
23
|
+
Provides-Extra: pgvector
|
|
24
|
+
Requires-Dist: psycopg2-binary>=2.9.9.1; extra == "pgvector"
|
|
25
|
+
Requires-Dist: pgvector>=0.4.0; extra == "pgvector"
|
|
26
|
+
Provides-Extra: redis
|
|
27
|
+
Requires-Dist: redis>=5.0.0; extra == "redis"
|
|
28
|
+
Provides-Extra: langchain
|
|
29
|
+
Requires-Dist: langchain-core>=0.3.0; extra == "langchain"
|
|
30
|
+
Provides-Extra: llamaindex
|
|
31
|
+
Requires-Dist: llama-index-core>=0.13.0; extra == "llamaindex"
|
|
32
|
+
Provides-Extra: all
|
|
33
|
+
Requires-Dist: sentence-transformers>=5.0.0; extra == "all"
|
|
34
|
+
Requires-Dist: psycopg2-binary>=2.9.9.1; extra == "all"
|
|
35
|
+
Requires-Dist: pgvector>=0.4.0; extra == "all"
|
|
36
|
+
Requires-Dist: langchain-core>=0.3.0; extra == "all"
|
|
37
|
+
Requires-Dist: llama-index-core>=0.13.0; extra == "all"
|
|
38
|
+
Requires-Dist: redis>=5.0.0; extra == "all"
|
|
39
|
+
Provides-Extra: dev
|
|
40
|
+
Requires-Dist: flake8>=7.0.0; extra == "dev"
|
|
41
|
+
Requires-Dist: mypy>=1.13.0; extra == "dev"
|
|
42
|
+
Requires-Dist: bandit>=1.8.0; extra == "dev"
|
|
43
|
+
Requires-Dist: pytest>=8.0.0; extra == "dev"
|
|
44
|
+
Requires-Dist: httpx>=0.27.0; extra == "dev"
|
|
45
|
+
Requires-Dist: types-requests>=2.32.0; extra == "dev"
|
|
46
|
+
Dynamic: author
|
|
47
|
+
Dynamic: home-page
|
|
48
|
+
Dynamic: requires-python
|
|
49
|
+
|
|
50
|
+
# Synapse Memory Python SDK
|
|
51
|
+
|
|
52
|
+
Core Python SDK and algorithms for the Synapse Cognitive Memory substrate.
|
|
53
|
+
|
|
54
|
+
## Features
|
|
55
|
+
|
|
56
|
+
- **Encrypted SQLite Storage**: Fernet AES-128-CBC with HMAC-SHA256 authenticated encryption at rest.
|
|
57
|
+
- **Dynamic Programming Knapsack Packing**: Exact 0/1 Knapsack optimizer for packing high-relevance memories into strict token envelopes.
|
|
58
|
+
- **Ebbinghaus Decay Engine**: Continuous temporal decay with positive/negative reinforcement factors.
|
|
59
|
+
- **Semantic Deduplication**: Dual lexical Jaccard and dense cosine similarity detection to prevent node fragmentation.
|
|
60
|
+
- **Hybrid Retrieval**: BM25 sparse lexical ranking fused with dense vector embeddings via Reciprocal Rank Fusion (RRF).
|
|
61
|
+
- **Framework Integrations**: True drop-in adapters for LangChain (`BaseMemory`) and LlamaIndex (`BaseMemory`).
|
|
62
|
+
|
|
63
|
+
## Installation
|
|
64
|
+
|
|
65
|
+
```bash
|
|
66
|
+
pip install synapse-memory-engine # from PyPI
|
|
67
|
+
# or, from a checkout of this repository:
|
|
68
|
+
pip install -e .
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
## Testing
|
|
72
|
+
|
|
73
|
+
```bash
|
|
74
|
+
python3 -m unittest discover -s tests
|
|
75
|
+
```
|