memtide 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- memtide/__init__.py +29 -0
- memtide/__main__.py +4 -0
- memtide/cli.py +96 -0
- memtide/config.py +192 -0
- memtide/consolidation.py +134 -0
- memtide/decay.py +89 -0
- memtide/embeddings.py +187 -0
- memtide/engine.py +806 -0
- memtide/gating.py +119 -0
- memtide/llm.py +184 -0
- memtide/multimodal.py +360 -0
- memtide/pgstore.py +529 -0
- memtide/retrieval.py +309 -0
- memtide/server.py +506 -0
- memtide/slots.py +73 -0
- memtide/storage.py +85 -0
- memtide/types.py +212 -0
- memtide/vectorstore.py +134 -0
- memtide-0.1.0.dist-info/METADATA +308 -0
- memtide-0.1.0.dist-info/RECORD +24 -0
- memtide-0.1.0.dist-info/WHEEL +5 -0
- memtide-0.1.0.dist-info/entry_points.txt +2 -0
- memtide-0.1.0.dist-info/licenses/LICENSE +21 -0
- memtide-0.1.0.dist-info/top_level.txt +1 -0
memtide/__init__.py
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
"""Memtide — a stdlib-core agent memory engine.
|
|
2
|
+
|
|
3
|
+
Atomic, auditable, forgetting-aware memory for LLM agents:
|
|
4
|
+
- Mem0-style atomic-fact extraction & write-time consolidation
|
|
5
|
+
- Letta-style core memory block you inject into the system prompt
|
|
6
|
+
- Zep-style temporal audit (nothing is silently lost; updates/invalidations tracked)
|
|
7
|
+
- Hybrid retrieval: vector + BM25 full-text + entity + Ebbinghaus retention, RRF-fused
|
|
8
|
+
|
|
9
|
+
Production-ready agent memory with PostgreSQL, Qdrant, and any
|
|
10
|
+
OpenAI-compatible LLM/embedding endpoints.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from .config import MemoryConfig
|
|
14
|
+
from .engine import MemoryEngine
|
|
15
|
+
from .types import AddResult, Event, ExtractedFact, Memory, MemoryType, SearchResult
|
|
16
|
+
|
|
17
|
+
__version__ = "0.1.0"
|
|
18
|
+
|
|
19
|
+
__all__ = [
|
|
20
|
+
"MemoryEngine",
|
|
21
|
+
"MemoryConfig",
|
|
22
|
+
"Memory",
|
|
23
|
+
"MemoryType",
|
|
24
|
+
"Event",
|
|
25
|
+
"AddResult",
|
|
26
|
+
"ExtractedFact",
|
|
27
|
+
"SearchResult",
|
|
28
|
+
"__version__",
|
|
29
|
+
]
|
memtide/__main__.py
ADDED
memtide/cli.py
ADDED
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
"""memtide CLI — manage the memory store from the shell (env-configured).
|
|
2
|
+
|
|
3
|
+
python -m memtide add "我叫李雷,住在杭州" # write memories
|
|
4
|
+
python -m memtide search "用户住哪" # hybrid retrieval
|
|
5
|
+
python -m memtide list # dump all memories
|
|
6
|
+
python -m memtide context "用户喜欢什么咖啡" # render core block
|
|
7
|
+
python -m memtide history # audit log
|
|
8
|
+
python -m memtide stats
|
|
9
|
+
python -m memtide delete <memory_id> # soft-delete one memory
|
|
10
|
+
python -m memtide serve --port 8300 # start the REST API server
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import argparse
|
|
16
|
+
import json
|
|
17
|
+
import sys
|
|
18
|
+
|
|
19
|
+
from .config import MemoryConfig
|
|
20
|
+
from .engine import MemoryEngine
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def _engine(args) -> MemoryEngine:
|
|
24
|
+
from .config import config_from_env
|
|
25
|
+
|
|
26
|
+
return MemoryEngine(config_from_env())
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def main(argv=None) -> int:
|
|
30
|
+
p = argparse.ArgumentParser(prog="memtide", description=__doc__,
|
|
31
|
+
formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
32
|
+
sub = p.add_subparsers(dest="cmd", required=True)
|
|
33
|
+
|
|
34
|
+
sp = sub.add_parser("add", help="store memories from a conversation string")
|
|
35
|
+
sp.add_argument("text")
|
|
36
|
+
sp.add_argument("--user", default="default")
|
|
37
|
+
|
|
38
|
+
sp = sub.add_parser("search", help="hybrid retrieval")
|
|
39
|
+
sp.add_argument("query")
|
|
40
|
+
sp.add_argument("--user", default="default")
|
|
41
|
+
sp.add_argument("-k", type=int, default=5)
|
|
42
|
+
|
|
43
|
+
sp = sub.add_parser("list", help="dump valid memories")
|
|
44
|
+
sp.add_argument("--user", default="default")
|
|
45
|
+
|
|
46
|
+
sp = sub.add_parser("context", help="render core memory block")
|
|
47
|
+
sp.add_argument("query", nargs="?", default=None)
|
|
48
|
+
sp.add_argument("--user", default="default")
|
|
49
|
+
|
|
50
|
+
sub.add_parser("history", help="audit log")
|
|
51
|
+
sub.add_parser("stats")
|
|
52
|
+
|
|
53
|
+
sp = sub.add_parser("delete", help="soft-delete one memory")
|
|
54
|
+
sp.add_argument("memory_id")
|
|
55
|
+
|
|
56
|
+
sp = sub.add_parser("serve", help="start the REST API server")
|
|
57
|
+
sp.add_argument("--host", default="127.0.0.1")
|
|
58
|
+
sp.add_argument("--port", type=int, default=8300)
|
|
59
|
+
|
|
60
|
+
args = p.parse_args(argv)
|
|
61
|
+
engine = _engine(args)
|
|
62
|
+
|
|
63
|
+
try:
|
|
64
|
+
if args.cmd == "serve":
|
|
65
|
+
engine.close()
|
|
66
|
+
from .server import serve_forever
|
|
67
|
+
|
|
68
|
+
serve_forever(host=args.host, port=args.port)
|
|
69
|
+
return 0
|
|
70
|
+
if args.cmd == "add":
|
|
71
|
+
res = engine.add(args.text, user_id=args.user)
|
|
72
|
+
print(json.dumps(res.to_dict(), ensure_ascii=False, indent=2))
|
|
73
|
+
elif args.cmd == "search":
|
|
74
|
+
hits = engine.search(args.query, user_id=args.user, limit=args.k)
|
|
75
|
+
for h in hits:
|
|
76
|
+
print(json.dumps(h.to_dict(), ensure_ascii=False))
|
|
77
|
+
elif args.cmd == "list":
|
|
78
|
+
for m in engine.get_all(args.user):
|
|
79
|
+
print(json.dumps(m.to_dict(), ensure_ascii=False))
|
|
80
|
+
elif args.cmd == "context":
|
|
81
|
+
print(engine.render_context(user_id=args.user, query=args.query))
|
|
82
|
+
elif args.cmd == "history":
|
|
83
|
+
for h in engine.get_history(limit=50):
|
|
84
|
+
print(json.dumps(h, ensure_ascii=False))
|
|
85
|
+
elif args.cmd == "stats":
|
|
86
|
+
print(json.dumps(engine.stats(), ensure_ascii=False, indent=2))
|
|
87
|
+
elif args.cmd == "delete":
|
|
88
|
+
ok = engine.delete(args.memory_id)
|
|
89
|
+
print("deleted" if ok else "not found")
|
|
90
|
+
finally:
|
|
91
|
+
engine.close()
|
|
92
|
+
return 0
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
if __name__ == "__main__":
|
|
96
|
+
sys.exit(main())
|
memtide/config.py
ADDED
|
@@ -0,0 +1,192 @@
|
|
|
1
|
+
"""Configuration for a MemoryEngine instance."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import os
|
|
6
|
+
from dataclasses import dataclass, field
|
|
7
|
+
from typing import Optional
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
@dataclass
|
|
11
|
+
class MemoryConfig:
|
|
12
|
+
# --- storage (PostgreSQL only) ----------------------------------------
|
|
13
|
+
storage_backend: str = "postgres"
|
|
14
|
+
pg_dsn: str = "" # e.g. postgresql://memtide:pw@localhost:5432/memtide
|
|
15
|
+
embedding_dim: int = 256 # probe hint only; the live dim comes from embedder.dim, never written back
|
|
16
|
+
|
|
17
|
+
# --- vector store (Qdrant only) -----------------------------------------
|
|
18
|
+
vector_backend: str = "qdrant"
|
|
19
|
+
qdrant_url: str = "http://localhost:6333"
|
|
20
|
+
qdrant_collection: str = "memtide"
|
|
21
|
+
|
|
22
|
+
# --- predictive-coding gate (gating.py) ---------------------------------
|
|
23
|
+
# The memory corpus is the prior; only prediction error is encoded.
|
|
24
|
+
gate_enabled: bool = True
|
|
25
|
+
gate_redundant_bits: float = 0.5 # surprise <= this -> REJECT (fully predicted)
|
|
26
|
+
gate_novel_bits: float = 2.5 # surprise >= this -> NOVEL (importance boost)
|
|
27
|
+
gate_importance_boost: float = 0.10
|
|
28
|
+
gate_slot_floor: float = 0.40 # min cosine to treat same-slot memory as predictor
|
|
29
|
+
# slot-scoped prior: for slotted facts (location/role/...), surprise is
|
|
30
|
+
# computed against SAME-slot memories only — "user lives in X" is not
|
|
31
|
+
# predicted by "user works at Y" just because the phrasing matches
|
|
32
|
+
gate_slot_scoped: bool = True
|
|
33
|
+
|
|
34
|
+
# --- write path -------------------------------------------------------
|
|
35
|
+
max_facts_per_turn: int = 12 # cap extraction output
|
|
36
|
+
dedup_threshold: float = 0.94 # cosine >= this => duplicate (NOOP/UPDATE)
|
|
37
|
+
conflict_threshold: float = 0.55 # cosine >= this + conflict signals => UPDATE
|
|
38
|
+
similarity_candidates: int = 5 # existing memories compared per new fact
|
|
39
|
+
|
|
40
|
+
# --- retrieval (hybrid fusion) ----------------------------------------
|
|
41
|
+
semantic_topk: int = 40 # Qdrant vector candidates
|
|
42
|
+
bm25_topk: int = 40 # PostgreSQL pg_search BM25 candidates
|
|
43
|
+
rrf_k: int = 60 # reciprocal-rank-fusion constant
|
|
44
|
+
# per-channel RRF weights: the loose entity channel counts less than the
|
|
45
|
+
# two main channels; expanded query variants count less than the original
|
|
46
|
+
entity_channel_weight: float = 0.5
|
|
47
|
+
expansion_variant_weight: float = 0.5
|
|
48
|
+
weights: dict = field(default_factory=lambda: {
|
|
49
|
+
"rrf": 1.0,
|
|
50
|
+
"semantic": 0.15, # raw cosine bonus on top of RRF
|
|
51
|
+
"bm25": 0.05, # exact keyword-hit bonus (hit = 1.0 else 0.0)
|
|
52
|
+
"entity": 0.03, # entity-hit bonus (hit = 1.0 else 0.0)
|
|
53
|
+
"recency": 0.10, # retention (Ebbinghaus) bonus
|
|
54
|
+
"importance": 0.05,
|
|
55
|
+
})
|
|
56
|
+
final_topk: int = 10
|
|
57
|
+
# post-fusion filters (search(memory_type=, slot=))
|
|
58
|
+
# MMR diversity: 0 = off; e.g. 0.7 blends score and dissimilarity
|
|
59
|
+
mmr_lambda: float = 0.0
|
|
60
|
+
# optional cross-encoder re-ranking of the fused top-N ("none" | "http";
|
|
61
|
+
# http = Jina/Cohere-style POST {rerank_base_url}/rerank)
|
|
62
|
+
rerank_backend: str = "none"
|
|
63
|
+
rerank_base_url: str = ""
|
|
64
|
+
rerank_model: str = ""
|
|
65
|
+
rerank_api_key: Optional[str] = None
|
|
66
|
+
rerank_topk: int = 20
|
|
67
|
+
# LLM query expansion (real-LLM deployments only): retrieve for the query
|
|
68
|
+
# plus an English translation + a paraphrase, fuse all rankings
|
|
69
|
+
query_expansion: bool = False
|
|
70
|
+
|
|
71
|
+
# --- forgetting / reinforcement ----------------------------------------
|
|
72
|
+
half_life_days: float = 45.0 # base Ebbinghaus half-life
|
|
73
|
+
reinforcement_gain: float = 0.4 # each access slows decay: hl *= 1+g*ln(1+n)
|
|
74
|
+
retention_floor: float = 0.02 # below this a memory is "forgotten" (hidden)
|
|
75
|
+
max_half_life_mult: float = 4.0 # cap on the reinforcement stretch
|
|
76
|
+
# episodic memories fade faster and are forgotten sooner than facts
|
|
77
|
+
episodic_half_life_mult: float = 0.5
|
|
78
|
+
episodic_floor: float = 0.05
|
|
79
|
+
|
|
80
|
+
# --- background consolidation (consolidation.py) --------------------------
|
|
81
|
+
consolidation_min_cluster: int = 3 # smallest cluster worth distilling
|
|
82
|
+
consolidation_similarity: float = 0.45 # real embeddings: same-topic ~0.55, cross-topic ~0.28
|
|
83
|
+
# abstractions forget slower than episodes (effective half-life multiplier
|
|
84
|
+
# for memories with source="consolidation")
|
|
85
|
+
consolidation_half_life_mult: float = 3.0
|
|
86
|
+
|
|
87
|
+
# --- core memory (Letta-style always-injected block) ---------------------
|
|
88
|
+
core_max_chars: int = 1200
|
|
89
|
+
|
|
90
|
+
# --- backends ------------------------------------------------------------
|
|
91
|
+
# llm: any OpenAI-compatible endpoint (API key required)
|
|
92
|
+
llm_backend: str = "openai"
|
|
93
|
+
llm_base_url: str = "https://api.openai.com/v1"
|
|
94
|
+
embedding_base_url: str = "" # empty -> llm_base_url
|
|
95
|
+
llm_model: str = "gpt-4o-mini"
|
|
96
|
+
llm_api_key: Optional[str] = None
|
|
97
|
+
# embedding: "auto" tries dashscope -> openai (by available keys)
|
|
98
|
+
embedding_backend: str = "auto" # auto|openai|dashscope
|
|
99
|
+
embedding_model: str = "text-embedding-3-small"
|
|
100
|
+
dashscope_api_key: Optional[str] = None
|
|
101
|
+
dashscope_base_url: str = "https://dashscope.aliyuncs.com/compatible-mode/v1"
|
|
102
|
+
dashscope_embedding_model: str = "qwen3.7-text-embedding"
|
|
103
|
+
|
|
104
|
+
# --- multimodal (multimodal.py) ----------------------------------------
|
|
105
|
+
# Media parts are described to text (vision model / optional STT) and the
|
|
106
|
+
# original bytes are stored content-addressed under media_dir.
|
|
107
|
+
multimodal_enabled: bool = True
|
|
108
|
+
media_dir: str = "memtide_media" # content-addressed asset store
|
|
109
|
+
max_media_bytes: int = 10 * 1024 * 1024
|
|
110
|
+
# SECURITY: reading media from local paths is disabled by default — a REST
|
|
111
|
+
# deployment would otherwise let any caller read arbitrary local files
|
|
112
|
+
# ({"path": "/etc/passwd"} -> GET /media/<sha>). Enable only for trusted
|
|
113
|
+
# embedders (CLI / in-process agents).
|
|
114
|
+
media_allow_paths: bool = False
|
|
115
|
+
# vision: empty base_url/model -> fall back to the main LLM endpoint
|
|
116
|
+
# (works with any OpenAI-compatible vision model, e.g. qwen-vl-max)
|
|
117
|
+
vision_base_url: str = ""
|
|
118
|
+
vision_model: str = ""
|
|
119
|
+
vision_api_key: Optional[str] = None
|
|
120
|
+
# speech-to-text: empty stt_model disables transcription (audio kept as
|
|
121
|
+
# reference-only attachments)
|
|
122
|
+
stt_base_url: str = ""
|
|
123
|
+
stt_model: str = ""
|
|
124
|
+
|
|
125
|
+
# --- slot normalisation (slots.py) --------------------------------------
|
|
126
|
+
# Open-hint slots: canonicalise aliases (city/住址 -> location) so the
|
|
127
|
+
# same meaning with different names still matches. Extra user aliases
|
|
128
|
+
# merge over the built-in table.
|
|
129
|
+
slot_aliases: dict = field(default_factory=dict)
|
|
130
|
+
|
|
131
|
+
# --- service / ops -------------------------------------------------------
|
|
132
|
+
api_key: Optional[str] = None # when set, REST requires it (X-API-Key)
|
|
133
|
+
auto_reflect_seconds: int = 0 # >0: run consolidate_background periodically
|
|
134
|
+
|
|
135
|
+
def resolve_api_key(self) -> Optional[str]:
|
|
136
|
+
if self.llm_api_key:
|
|
137
|
+
return self.llm_api_key
|
|
138
|
+
return os.environ.get("OPENAI_API_KEY") or os.environ.get("LLM_API_KEY")
|
|
139
|
+
|
|
140
|
+
def resolve_dashscope_key(self) -> Optional[str]:
|
|
141
|
+
if self.dashscope_api_key:
|
|
142
|
+
return self.dashscope_api_key
|
|
143
|
+
return os.environ.get("DASHSCOPE_API_KEY")
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def config_from_env() -> "MemoryConfig":
|
|
147
|
+
"""Build a config from MEMTIDE_* / LLM_* / EMBEDDING_* / DASHSCOPE_* env
|
|
148
|
+
variables (used by the REST server and Docker deployment)."""
|
|
149
|
+
cfg = MemoryConfig(
|
|
150
|
+
storage_backend=(os.environ.get("MEMTIDE_STORAGE") or "postgres"),
|
|
151
|
+
pg_dsn=(os.environ.get("MEMTIDE_PG_DSN") or ""),
|
|
152
|
+
vector_backend=(os.environ.get("MEMTIDE_VECTOR_BACKEND") or "qdrant"),
|
|
153
|
+
qdrant_url=(os.environ.get("MEMTIDE_QDRANT_URL") or "http://localhost:6333"),
|
|
154
|
+
qdrant_collection=(os.environ.get("MEMTIDE_QDRANT_COLLECTION") or "memtide"),
|
|
155
|
+
llm_backend=(os.environ.get("LLM_BACKEND") or "openai"),
|
|
156
|
+
llm_base_url=(os.environ.get("LLM_BASE_URL") or "https://api.openai.com/v1"),
|
|
157
|
+
llm_model=(os.environ.get("LLM_MODEL") or "gpt-4o-mini"),
|
|
158
|
+
embedding_backend=(os.environ.get("EMBEDDING_BACKEND") or "auto"),
|
|
159
|
+
embedding_model=(os.environ.get("EMBEDDING_MODEL") or "text-embedding-3-small"),
|
|
160
|
+
dashscope_embedding_model=(os.environ.get("DASHSCOPE_EMBEDDING_MODEL")
|
|
161
|
+
or "qwen3.7-text-embedding"),
|
|
162
|
+
dashscope_base_url=os.environ.get(
|
|
163
|
+
"DASHSCOPE_BASE_URL",
|
|
164
|
+
"https://dashscope.aliyuncs.com/compatible-mode/v1"),
|
|
165
|
+
multimodal_enabled=(os.environ.get("MEMTIDE_MULTIMODAL") or "1") not in ("0", "false", "no"),
|
|
166
|
+
media_dir=(os.environ.get("MEMTIDE_MEDIA_DIR") or "memtide_media"),
|
|
167
|
+
media_allow_paths=(os.environ.get("MEMTIDE_MEDIA_ALLOW_PATHS") or "0") in ("1", "true", "yes"),
|
|
168
|
+
vision_base_url=(os.environ.get("MEMTIDE_VISION_BASE_URL") or ""),
|
|
169
|
+
vision_model=(os.environ.get("MEMTIDE_VISION_MODEL") or ""),
|
|
170
|
+
stt_base_url=(os.environ.get("MEMTIDE_STT_BASE_URL") or ""),
|
|
171
|
+
stt_model=(os.environ.get("MEMTIDE_STT_MODEL") or ""),
|
|
172
|
+
)
|
|
173
|
+
if cfg.storage_backend != "postgres":
|
|
174
|
+
raise ValueError("MEMTIDE_STORAGE 必须为 postgres;PostgreSQL 是唯一存储后端")
|
|
175
|
+
if not cfg.pg_dsn:
|
|
176
|
+
raise ValueError("请设置 MEMTIDE_PG_DSN(例如 postgresql://user:password@host:5432/db)")
|
|
177
|
+
if cfg.vector_backend != "qdrant":
|
|
178
|
+
raise ValueError("MEMTIDE_VECTOR_BACKEND 必须为 qdrant;Qdrant 是唯一向量后端")
|
|
179
|
+
if os.environ.get("LLM_API_KEY"):
|
|
180
|
+
cfg.llm_api_key = os.environ["LLM_API_KEY"]
|
|
181
|
+
if os.environ.get("DASHSCOPE_API_KEY"):
|
|
182
|
+
cfg.dashscope_api_key = os.environ["DASHSCOPE_API_KEY"]
|
|
183
|
+
if os.environ.get("MEMTIDE_VISION_API_KEY"):
|
|
184
|
+
cfg.vision_api_key = os.environ["MEMTIDE_VISION_API_KEY"]
|
|
185
|
+
if os.environ.get("MEMTIDE_API_KEY"):
|
|
186
|
+
cfg.api_key = os.environ["MEMTIDE_API_KEY"]
|
|
187
|
+
if os.environ.get("MEMTIDE_AUTO_REFLECT"):
|
|
188
|
+
try:
|
|
189
|
+
cfg.auto_reflect_seconds = max(60, int(os.environ["MEMTIDE_AUTO_REFLECT"]))
|
|
190
|
+
except ValueError:
|
|
191
|
+
pass
|
|
192
|
+
return cfg
|
memtide/consolidation.py
ADDED
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
"""Background consolidation (LangMem-style reflection).
|
|
2
|
+
|
|
3
|
+
A periodic pass over the memory bank that does higher-level updating no
|
|
4
|
+
single write can do:
|
|
5
|
+
|
|
6
|
+
1. CLUSTER greedy density clustering: a memory plus every memory within
|
|
7
|
+
cosine ``threshold`` of it forms a same-topic cluster (only
|
|
8
|
+
clusters with >= ``min_cluster`` members count; already-
|
|
9
|
+
consolidated summaries are excluded to keep one abstraction level
|
|
10
|
+
per pass).
|
|
11
|
+
2. SUMMARIZE members are distilled into one summary memory (LLM when
|
|
12
|
+
configured, deterministic template otherwise — same contract).
|
|
13
|
+
3. SUPERSEDE each member is invalidated (``invalid_at``) and linked to the
|
|
14
|
+
summary via ``superseded_by``; a CONSOLIDATE event lands in the
|
|
15
|
+
audit history, so nothing is lost and the chain is replayable.
|
|
16
|
+
|
|
17
|
+
The summary inherits the strongest member's importance (plus a small bonus),
|
|
18
|
+
which lifts it into the core-memory block — repeatedly observed patterns get
|
|
19
|
+
promoted, exactly the "distillation" LangMem describes. Because superseded
|
|
20
|
+
members are gone from FTS and the vector scan, this also *shrinks* retrieval
|
|
21
|
+
space while keeping information reachable through the summary.
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
from __future__ import annotations
|
|
25
|
+
|
|
26
|
+
import json
|
|
27
|
+
import re
|
|
28
|
+
from typing import Any, Dict, List, Optional
|
|
29
|
+
|
|
30
|
+
from .embeddings import cosine, pack, unpack
|
|
31
|
+
from .types import Memory, MemoryType
|
|
32
|
+
|
|
33
|
+
CONSOLIDATION_SUMMARY_PROMPT = """You are the background memory reflection module of an AI agent.
|
|
34
|
+
The facts below are same-topic memories about one user. Distill them into ONE
|
|
35
|
+
short summary memory (third person, keeping every distinct detail that
|
|
36
|
+
matters; drop repetition). Write it in the language the facts mostly use;
|
|
37
|
+
1-2 sentences, up to 4 for a large cluster.
|
|
38
|
+
|
|
39
|
+
Output STRICT JSON: {{"summary": "..."}}
|
|
40
|
+
|
|
41
|
+
Facts:
|
|
42
|
+
{facts}"""
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _is_zh(text: str) -> bool:
|
|
46
|
+
return (len(re.findall(r"[\u4e00-\u9fff]", text))
|
|
47
|
+
> len(re.findall(r"[a-zA-Z]", text)))
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def find_clusters(memories: List[Memory], vectors: Dict[str, List[float]],
|
|
51
|
+
threshold: float, min_cluster: int) -> List[List[Memory]]:
|
|
52
|
+
"""Greedy density clustering over memory vectors."""
|
|
53
|
+
remaining = list(memories)
|
|
54
|
+
clusters: List[List[Memory]] = []
|
|
55
|
+
while remaining:
|
|
56
|
+
seed = remaining[0]
|
|
57
|
+
group = [m for m in remaining
|
|
58
|
+
if cosine(vectors[seed.id], vectors[m.id]) >= threshold]
|
|
59
|
+
if len(group) >= min_cluster:
|
|
60
|
+
clusters.append(group)
|
|
61
|
+
remaining = [m for m in remaining if m not in group]
|
|
62
|
+
else:
|
|
63
|
+
remaining = remaining[1:]
|
|
64
|
+
return clusters
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def consolidate(engine, user_id: str = "default", agent_id: Optional[str] = None,
|
|
68
|
+
run_id: Optional[str] = None, min_cluster: Optional[int] = None,
|
|
69
|
+
threshold: Optional[float] = None) -> Dict[str, Any]:
|
|
70
|
+
"""Run one reflection pass; returns an auditable report."""
|
|
71
|
+
cfg = engine.cfg
|
|
72
|
+
min_cluster = min_cluster if min_cluster is not None else cfg.consolidation_min_cluster
|
|
73
|
+
threshold = threshold if threshold is not None else cfg.consolidation_similarity
|
|
74
|
+
|
|
75
|
+
memories = engine.store.all_valid(user_id, agent_id, run_id)
|
|
76
|
+
# one abstraction level per pass: never re-merge summaries
|
|
77
|
+
memories = [m for m in memories if not m.metadata.get("consolidated")]
|
|
78
|
+
if len(memories) < min_cluster:
|
|
79
|
+
return {"clusters": 0, "summaries": [], "members_absorbed": 0}
|
|
80
|
+
|
|
81
|
+
vectors = {}
|
|
82
|
+
blobs = engine.store.get_embeddings([m.id for m in memories])
|
|
83
|
+
for m in memories:
|
|
84
|
+
vec = unpack(blobs.get(m.id), cfg.embedding_dim)
|
|
85
|
+
if vec is not None:
|
|
86
|
+
vectors[m.id] = vec
|
|
87
|
+
memories = [m for m in memories if m.id in vectors]
|
|
88
|
+
|
|
89
|
+
report: Dict[str, Any] = {"clusters": 0, "summaries": [], "members_absorbed": 0}
|
|
90
|
+
for group in find_clusters(memories, vectors, threshold, min_cluster):
|
|
91
|
+
group = sorted(group, key=lambda m: m.created_at)
|
|
92
|
+
texts = [m.text for m in group]
|
|
93
|
+
summary_text: Optional[str] = None
|
|
94
|
+
data = engine.llm.complete_json(
|
|
95
|
+
"You are the background memory reflection module of an AI agent.",
|
|
96
|
+
CONSOLIDATION_SUMMARY_PROMPT.format(
|
|
97
|
+
facts=json.dumps(texts, ensure_ascii=False)),
|
|
98
|
+
)
|
|
99
|
+
if data and data.get("summary"):
|
|
100
|
+
summary_text = str(data["summary"]).strip()
|
|
101
|
+
if not summary_text:
|
|
102
|
+
# deterministic fallback promised by the module contract: never
|
|
103
|
+
# let one bad LLM response kill the whole consolidation pass
|
|
104
|
+
summary_text = " | ".join(texts)[:500]
|
|
105
|
+
|
|
106
|
+
entities = sorted({e for m in group for e in m.entities})
|
|
107
|
+
summary = Memory(
|
|
108
|
+
text=summary_text,
|
|
109
|
+
memory_type=MemoryType.FACT,
|
|
110
|
+
user_id=user_id,
|
|
111
|
+
agent_id=agent_id,
|
|
112
|
+
run_id=run_id,
|
|
113
|
+
entities=entities[:8],
|
|
114
|
+
metadata={
|
|
115
|
+
"consolidated": True,
|
|
116
|
+
"members": [m.id for m in group],
|
|
117
|
+
"gate": "consolidated",
|
|
118
|
+
},
|
|
119
|
+
importance=min(0.95, max(m.importance for m in group) + 0.05),
|
|
120
|
+
source="consolidation",
|
|
121
|
+
)
|
|
122
|
+
engine.store.insert(summary, pack(summary_vec := engine.embedder.embed(summary_text)))
|
|
123
|
+
engine.vector_store.upsert(
|
|
124
|
+
summary.id, summary_vec,
|
|
125
|
+
{"memory_id": summary.id, "user_id": user_id,
|
|
126
|
+
"agent_id": agent_id, "run_id": run_id})
|
|
127
|
+
for m in group:
|
|
128
|
+
engine.store.supersede(m.id, summary.id)
|
|
129
|
+
engine.vector_store.delete(m.id)
|
|
130
|
+
report["clusters"] += 1
|
|
131
|
+
report["members_absorbed"] += len(group)
|
|
132
|
+
report["summaries"].append({"id": summary.id, "text": summary_text,
|
|
133
|
+
"members": [m.id for m in group]})
|
|
134
|
+
return report
|
memtide/decay.py
ADDED
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
"""Forgetting & reinforcement (cognitive-science flavored).
|
|
2
|
+
|
|
3
|
+
Retention follows a Ebbinghaus-style exponential decay whose half-life is
|
|
4
|
+
stretched by each successful retrieval (spacing effect):
|
|
5
|
+
|
|
6
|
+
half_life_eff = half_life_days * min(1 + gain * ln(1 + access_count), max_mult)
|
|
7
|
+
retention = 0.5 ** (age_days / half_life_eff)
|
|
8
|
+
|
|
9
|
+
``age_days`` counts from when the fact became true (valid_at/created_at):
|
|
10
|
+
re-reading a memory reinforces it (longer half-life via access_count) but no
|
|
11
|
+
longer rejuvenates its age — otherwise hot memories would live forever.
|
|
12
|
+
Episodic memories fade faster than facts (episodic_mult < 1).
|
|
13
|
+
|
|
14
|
+
Memories below their floor are treated as forgotten: they stay in the
|
|
15
|
+
database (auditable, Zep-style) but stop surfacing in retrieval and the core
|
|
16
|
+
memory block, until a future query re-excites them.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
import math
|
|
22
|
+
from datetime import datetime, timezone
|
|
23
|
+
from typing import Optional
|
|
24
|
+
|
|
25
|
+
from .types import Memory
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _parse(ts: Optional[str]) -> Optional[datetime]:
|
|
29
|
+
if not ts:
|
|
30
|
+
return None
|
|
31
|
+
try:
|
|
32
|
+
return datetime.fromisoformat(ts)
|
|
33
|
+
except ValueError:
|
|
34
|
+
return None
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def age_days(mem: Memory, now: Optional[datetime] = None) -> float:
|
|
38
|
+
# Age counts from when the fact became true — NOT last_accessed: a hit
|
|
39
|
+
# reinforces (see effective_half_life) but must not reset the clock.
|
|
40
|
+
now = now or datetime.now(timezone.utc)
|
|
41
|
+
ref = _parse(mem.valid_at) or _parse(mem.created_at)
|
|
42
|
+
if ref is None:
|
|
43
|
+
return 0.0
|
|
44
|
+
if ref.tzinfo is None:
|
|
45
|
+
ref = ref.replace(tzinfo=timezone.utc)
|
|
46
|
+
return max(0.0, (now - ref).total_seconds() / 86400.0)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def effective_half_life(mem: Memory, base_days: float, gain: float,
|
|
50
|
+
consolidation_mult: float = 1.0,
|
|
51
|
+
episodic_mult: float = 0.5,
|
|
52
|
+
max_mult: float = 4.0) -> float:
|
|
53
|
+
"""Half-life stretched by retrieval (spacing effect), capped at
|
|
54
|
+
``max_mult`` × base so hot memories can't become effectively immortal.
|
|
55
|
+
Abstractions (source="consolidation") fade slower than episodes when a
|
|
56
|
+
multiplier > 1 is configured — the distilled summary outlives its members.
|
|
57
|
+
Episodic memories fade faster (episodic_mult < 1)."""
|
|
58
|
+
stretch = min(1.0 + gain * math.log1p(mem.access_count), max_mult)
|
|
59
|
+
hl = base_days * stretch
|
|
60
|
+
if mem.memory_type == "episodic":
|
|
61
|
+
hl *= episodic_mult
|
|
62
|
+
if consolidation_mult != 1.0 and mem.source == "consolidation":
|
|
63
|
+
hl *= consolidation_mult
|
|
64
|
+
return hl
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def retention(mem: Memory, base_days: float, gain: float, now: Optional[datetime] = None,
|
|
68
|
+
consolidation_mult: float = 1.0,
|
|
69
|
+
episodic_mult: float = 0.5,
|
|
70
|
+
max_mult: float = 4.0) -> float:
|
|
71
|
+
"""Memory strength in [0, 1] — used for ranking and forgetting decisions."""
|
|
72
|
+
a = age_days(mem, now)
|
|
73
|
+
if a <= 0.0:
|
|
74
|
+
return 1.0
|
|
75
|
+
hl = effective_half_life(mem, base_days, gain, consolidation_mult,
|
|
76
|
+
episodic_mult, max_mult)
|
|
77
|
+
return 0.5 ** (a / hl)
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def is_forgotten(mem: Memory, base_days: float, gain: float, floor: float,
|
|
81
|
+
now: Optional[datetime] = None,
|
|
82
|
+
consolidation_mult: float = 1.0,
|
|
83
|
+
episodic_floor: Optional[float] = None,
|
|
84
|
+
episodic_mult: float = 0.5,
|
|
85
|
+
max_mult: float = 4.0) -> bool:
|
|
86
|
+
if episodic_floor is not None and mem.memory_type == "episodic":
|
|
87
|
+
floor = episodic_floor
|
|
88
|
+
return retention(mem, base_days, gain, now, consolidation_mult,
|
|
89
|
+
episodic_mult, max_mult) < floor
|