mymemo 0.1.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mymemo-0.1.1/.gitignore +7 -0
- mymemo-0.1.1/PKG-INFO +70 -0
- mymemo-0.1.1/README.md +47 -0
- mymemo-0.1.1/examples/demo.py +58 -0
- mymemo-0.1.1/mymemo/__init__.py +38 -0
- mymemo-0.1.1/mymemo/client.py +28 -0
- mymemo-0.1.1/mymemo/config.py +26 -0
- mymemo-0.1.1/mymemo/recaller.py +61 -0
- mymemo-0.1.1/mymemo/store.py +205 -0
- mymemo-0.1.1/mymemo/types.py +92 -0
- mymemo-0.1.1/pyproject.toml +39 -0
- mymemo-0.1.1/tests/test_store.py +116 -0
mymemo-0.1.1/.gitignore
ADDED
mymemo-0.1.1/PKG-INFO
ADDED
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: mymemo
|
|
3
|
+
Version: 0.1.1
|
|
4
|
+
Summary: Python SDK for OmniMem — Agent memory ledger on Turso/libSQL
|
|
5
|
+
Project-URL: Homepage, https://github.com/OntarioLT/OmniMem
|
|
6
|
+
Project-URL: Repository, https://github.com/OntarioLT/OmniMem
|
|
7
|
+
Project-URL: Issues, https://github.com/OntarioLT/OmniMem/issues
|
|
8
|
+
Author: OntarioLT
|
|
9
|
+
License-Expression: MIT
|
|
10
|
+
Keywords: agent,fts5,libsql,memory,rag,turso,vector
|
|
11
|
+
Classifier: Development Status :: 3 - Alpha
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
15
|
+
Classifier: Topic :: Software Development :: Libraries
|
|
16
|
+
Requires-Python: >=3.11
|
|
17
|
+
Requires-Dist: libsql>=0.1.0
|
|
18
|
+
Requires-Dist: pydantic>=2.0
|
|
19
|
+
Provides-Extra: dev
|
|
20
|
+
Requires-Dist: pytest-asyncio>=0.23; extra == 'dev'
|
|
21
|
+
Requires-Dist: pytest>=8.0; extra == 'dev'
|
|
22
|
+
Description-Content-Type: text/markdown
|
|
23
|
+
|
|
24
|
+
# mymemo
|
|
25
|
+
|
|
26
|
+
Python SDK for [MyMemo](https://github.com/OntarioLT/OmniMem) — Agent memory ledger on Turso/libSQL.
|
|
27
|
+
|
|
28
|
+
## Install
|
|
29
|
+
|
|
30
|
+
```bash
|
|
31
|
+
pip install mymemo
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
## Quick Start
|
|
35
|
+
|
|
36
|
+
```python
|
|
37
|
+
from mymemo import MyMemoConfig, create_clients, MemoryStore, rank
|
|
38
|
+
from mymemo.types import MemoryFactDraft, IncidentBase, SearchQuery
|
|
39
|
+
import time
|
|
40
|
+
|
|
41
|
+
# Configure
|
|
42
|
+
config = MyMemoConfig(
|
|
43
|
+
routing={"primary": "wss://your-db.turso.io", "replica": "file:./local.db"}
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
# Connect
|
|
47
|
+
clients = create_clients(config)
|
|
48
|
+
store = MemoryStore(clients.writer, dimension=384)
|
|
49
|
+
|
|
50
|
+
# Write memory
|
|
51
|
+
meta = IncidentBase(scope="my-agent", incident_id="1", timestamp=int(time.time()))
|
|
52
|
+
store.write(MemoryFactDraft(content="User prefers dark mode", category="user_preference"), meta)
|
|
53
|
+
|
|
54
|
+
# Search
|
|
55
|
+
results = store.get_candidates(SearchQuery(scope="my-agent", keyword="dark"))
|
|
56
|
+
ranked = rank(results, SearchQuery(scope="my-agent"))
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
## Features
|
|
60
|
+
|
|
61
|
+
- **Idempotent writes** — SHA256 deterministic IDs, no duplicates
|
|
62
|
+
- **Hybrid search** — FTS5 full-text + vector similarity
|
|
63
|
+
- **Adaptive ranking** — time decay, category-specific tuning
|
|
64
|
+
- **GC** — automatic eviction of stale memories
|
|
65
|
+
- **NaN defense** — rejects invalid vectors at write time
|
|
66
|
+
- **Shared database** — compatible with the TypeScript core SDK
|
|
67
|
+
|
|
68
|
+
## License
|
|
69
|
+
|
|
70
|
+
MIT
|
mymemo-0.1.1/README.md
ADDED
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
# mymemo
|
|
2
|
+
|
|
3
|
+
Python SDK for [MyMemo](https://github.com/OntarioLT/OmniMem) — Agent memory ledger on Turso/libSQL.
|
|
4
|
+
|
|
5
|
+
## Install
|
|
6
|
+
|
|
7
|
+
```bash
|
|
8
|
+
pip install mymemo
|
|
9
|
+
```
|
|
10
|
+
|
|
11
|
+
## Quick Start
|
|
12
|
+
|
|
13
|
+
```python
|
|
14
|
+
from mymemo import MyMemoConfig, create_clients, MemoryStore, rank
|
|
15
|
+
from mymemo.types import MemoryFactDraft, IncidentBase, SearchQuery
|
|
16
|
+
import time
|
|
17
|
+
|
|
18
|
+
# Configure
|
|
19
|
+
config = MyMemoConfig(
|
|
20
|
+
routing={"primary": "wss://your-db.turso.io", "replica": "file:./local.db"}
|
|
21
|
+
)
|
|
22
|
+
|
|
23
|
+
# Connect
|
|
24
|
+
clients = create_clients(config)
|
|
25
|
+
store = MemoryStore(clients.writer, dimension=384)
|
|
26
|
+
|
|
27
|
+
# Write memory
|
|
28
|
+
meta = IncidentBase(scope="my-agent", incident_id="1", timestamp=int(time.time()))
|
|
29
|
+
store.write(MemoryFactDraft(content="User prefers dark mode", category="user_preference"), meta)
|
|
30
|
+
|
|
31
|
+
# Search
|
|
32
|
+
results = store.get_candidates(SearchQuery(scope="my-agent", keyword="dark"))
|
|
33
|
+
ranked = rank(results, SearchQuery(scope="my-agent"))
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
## Features
|
|
37
|
+
|
|
38
|
+
- **Idempotent writes** — SHA256 deterministic IDs, no duplicates
|
|
39
|
+
- **Hybrid search** — FTS5 full-text + vector similarity
|
|
40
|
+
- **Adaptive ranking** — time decay, category-specific tuning
|
|
41
|
+
- **GC** — automatic eviction of stale memories
|
|
42
|
+
- **NaN defense** — rejects invalid vectors at write time
|
|
43
|
+
- **Shared database** — compatible with the TypeScript core SDK
|
|
44
|
+
|
|
45
|
+
## License
|
|
46
|
+
|
|
47
|
+
MIT
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
"""MyMemo Python SDK Demo — 全流程演示"""
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
import libsql as libsql
|
|
5
|
+
from mymemo import MemoryStore, rank
|
|
6
|
+
from mymemo.types import MemoryFactDraft, IncidentBase, SearchQuery
|
|
7
|
+
import time
|
|
8
|
+
|
|
9
|
+
DB_PATH = "file:./mymemo_demo.db"
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def main():
|
|
13
|
+
# 清理旧 demo 数据库
|
|
14
|
+
if os.path.exists("mymemo_demo.db"):
|
|
15
|
+
os.remove("mymemo_demo.db")
|
|
16
|
+
|
|
17
|
+
# 连接本地 libSQL 数据库
|
|
18
|
+
conn = libsql.connect(DB_PATH)
|
|
19
|
+
store = MemoryStore(conn, dimension=384)
|
|
20
|
+
|
|
21
|
+
# 写入多条记忆
|
|
22
|
+
meta = IncidentBase(scope="demo", incident_id="demo-1", timestamp=int(time.time()))
|
|
23
|
+
memories = [
|
|
24
|
+
("用户喜欢深色模式", "user_preference"),
|
|
25
|
+
("项目使用 TypeScript", "code_snippet"),
|
|
26
|
+
("上周修复了登录 bug", "correction"),
|
|
27
|
+
("TypeScript 类型系统很强大", "insight"),
|
|
28
|
+
("用户在上海工作", "user_preference"),
|
|
29
|
+
]
|
|
30
|
+
for content, category in memories:
|
|
31
|
+
store.write(MemoryFactDraft(content=content, category=category), meta)
|
|
32
|
+
print(f"✅ 写入 {len(memories)} 条记忆")
|
|
33
|
+
|
|
34
|
+
# 搜索
|
|
35
|
+
results = store.get_candidates(SearchQuery(scope="demo", keyword="TypeScript"))
|
|
36
|
+
print(f"\n🔍 搜索 'TypeScript' → {len(results)} 条结果")
|
|
37
|
+
for r in results:
|
|
38
|
+
print(f" [{r.category}] {r.content}")
|
|
39
|
+
|
|
40
|
+
# 混合排序
|
|
41
|
+
ranked = rank(results, SearchQuery(scope="demo"))
|
|
42
|
+
print(f"\n📊 混合排序(文本+向量+时间衰减):")
|
|
43
|
+
for r in ranked:
|
|
44
|
+
print(f" score={r.final_score:.3f} | {r.content}")
|
|
45
|
+
|
|
46
|
+
# 归档
|
|
47
|
+
store.archive(results[0].id, "demo:已读", "user")
|
|
48
|
+
print(f"\n🗄️ 归档: {results[0].content}")
|
|
49
|
+
|
|
50
|
+
# GC
|
|
51
|
+
archived = store.run_gc("demo")
|
|
52
|
+
print(f"\n🗑️ GC 淘汰 {archived} 条记忆")
|
|
53
|
+
|
|
54
|
+
print(f"\n📁 数据库文件: mymemo_demo.db")
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
if __name__ == "__main__":
|
|
58
|
+
main()
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
"""mymemo — Python SDK for MyMemo agent memory ledger."""
|
|
2
|
+
|
|
3
|
+
from .config import MyMemoConfig
|
|
4
|
+
from .client import MyMemoClients, create_clients
|
|
5
|
+
from .store import MemoryStore
|
|
6
|
+
from .recaller import rank
|
|
7
|
+
from .types import (
|
|
8
|
+
CandidateMemory,
|
|
9
|
+
CheckpointIncident,
|
|
10
|
+
ExtractedFactIncident,
|
|
11
|
+
Incident,
|
|
12
|
+
IncidentBase,
|
|
13
|
+
MemoryFactDraft,
|
|
14
|
+
Message,
|
|
15
|
+
MessageBatchIncident,
|
|
16
|
+
RawEventIncident,
|
|
17
|
+
RankedMemory,
|
|
18
|
+
SearchQuery,
|
|
19
|
+
)
|
|
20
|
+
|
|
21
|
+
__all__ = [
|
|
22
|
+
"MyMemoConfig",
|
|
23
|
+
"MyMemoClients",
|
|
24
|
+
"create_clients",
|
|
25
|
+
"MemoryStore",
|
|
26
|
+
"rank",
|
|
27
|
+
"CandidateMemory",
|
|
28
|
+
"CheckpointIncident",
|
|
29
|
+
"ExtractedFactIncident",
|
|
30
|
+
"Incident",
|
|
31
|
+
"IncidentBase",
|
|
32
|
+
"MemoryFactDraft",
|
|
33
|
+
"Message",
|
|
34
|
+
"MessageBatchIncident",
|
|
35
|
+
"RawEventIncident",
|
|
36
|
+
"RankedMemory",
|
|
37
|
+
"SearchQuery",
|
|
38
|
+
]
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import libsql as libsql
|
|
4
|
+
|
|
5
|
+
from .config import MyMemoConfig
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class MyMemoClients:
|
|
9
|
+
def __init__(self, writer: libsql.Connection, reader: libsql.Connection):
|
|
10
|
+
self.writer = writer
|
|
11
|
+
self.reader = reader
|
|
12
|
+
|
|
13
|
+
def close(self):
|
|
14
|
+
self.writer.close()
|
|
15
|
+
self.reader.close()
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def create_clients(config: MyMemoConfig) -> MyMemoClients:
|
|
19
|
+
writer = libsql.connect(
|
|
20
|
+
config.routing.primary,
|
|
21
|
+
auth_token=config.routing.token,
|
|
22
|
+
)
|
|
23
|
+
reader = libsql.connect(
|
|
24
|
+
config.routing.replica,
|
|
25
|
+
sync_url=config.routing.primary,
|
|
26
|
+
sync_interval=config.sync_interval,
|
|
27
|
+
)
|
|
28
|
+
return OmniMemClients(writer, reader)
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from pydantic import BaseModel, Field
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class EmbedderConfig(BaseModel):
|
|
7
|
+
dimension: int = Field(384, pattern=r"^(384|768|1024|1536)$")
|
|
8
|
+
quantization: str = Field("f8", pattern=r"^(f8|f32)$")
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class RoutingConfig(BaseModel):
|
|
12
|
+
primary: str # wss://xxx.turso.io
|
|
13
|
+
replica: str # file:///path/to/local.db
|
|
14
|
+
token: str | None = None
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class CurationConfig(BaseModel):
|
|
18
|
+
default_hybrid_weights: dict[str, float] | None = None
|
|
19
|
+
min_final_score: float = Field(0.15, ge=0, le=1)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class MyMemoConfig(BaseModel):
|
|
23
|
+
embedder: EmbedderConfig = EmbedderConfig()
|
|
24
|
+
routing: RoutingConfig
|
|
25
|
+
sync_interval: int = Field(5000, ge=1000, le=60000)
|
|
26
|
+
curation: CurationConfig = CurationConfig()
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import math
|
|
4
|
+
|
|
5
|
+
from .types import CandidateMemory, RankedMemory, SearchQuery
|
|
6
|
+
|
|
7
|
+
DEFAULT_CATEGORY_LAMBDA: dict[str, float] = {
|
|
8
|
+
"user_preference": 0.2,
|
|
9
|
+
"code_snippet": 0.6,
|
|
10
|
+
"correction": 0.5,
|
|
11
|
+
"insight": 0.3,
|
|
12
|
+
"ephemeral": 0.4,
|
|
13
|
+
"__default__": 0.4,
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
MIN_FINAL_SCORE = 0.15
|
|
17
|
+
FTS_EXP_CLAMP = 1e-8
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def rank(candidates: list[CandidateMemory], query: SearchQuery) -> list[RankedMemory]:
|
|
21
|
+
results: list[RankedMemory] = []
|
|
22
|
+
for c in candidates:
|
|
23
|
+
lam = (
|
|
24
|
+
(query.category_lambda or {}).get(c.category)
|
|
25
|
+
or DEFAULT_CATEGORY_LAMBDA.get(c.category)
|
|
26
|
+
or DEFAULT_CATEGORY_LAMBDA["__default__"]
|
|
27
|
+
)
|
|
28
|
+
|
|
29
|
+
fts_score = math.exp(-lam * abs(c.raw_fts_score))
|
|
30
|
+
if fts_score < FTS_EXP_CLAMP:
|
|
31
|
+
fts_score = 0.0
|
|
32
|
+
|
|
33
|
+
vec_score = min(max(c.raw_vec_score, 0.0), 1.0)
|
|
34
|
+
decay_base = (query.decay_map or {}).get(c.category, 0.98)
|
|
35
|
+
decay_factor = math.pow(decay_base, max(0, c.days_since_retrieved))
|
|
36
|
+
|
|
37
|
+
w = query.hybrid_weights or {"vec": 0.6, "fts": 0.4}
|
|
38
|
+
final_score = (vec_score * w["vec"] + fts_score * w["fts"]) * decay_factor * c.weight
|
|
39
|
+
|
|
40
|
+
if final_score >= MIN_FINAL_SCORE:
|
|
41
|
+
results.append(
|
|
42
|
+
RankedMemory(
|
|
43
|
+
id=c.id,
|
|
44
|
+
content=c.content,
|
|
45
|
+
category=c.category,
|
|
46
|
+
weight=c.weight,
|
|
47
|
+
version=c.version,
|
|
48
|
+
preserve_reason=c.preserve_reason,
|
|
49
|
+
successful_recall_count=c.successful_recall_count,
|
|
50
|
+
raw_fts_score=c.raw_fts_score,
|
|
51
|
+
raw_vec_score=c.raw_vec_score,
|
|
52
|
+
days_since_retrieved=c.days_since_retrieved,
|
|
53
|
+
fts_score=fts_score,
|
|
54
|
+
vec_score=vec_score,
|
|
55
|
+
final_score=final_score,
|
|
56
|
+
decay_factor=decay_factor,
|
|
57
|
+
)
|
|
58
|
+
)
|
|
59
|
+
|
|
60
|
+
results.sort(key=lambda r: r.final_score, reverse=True)
|
|
61
|
+
return results
|
|
@@ -0,0 +1,205 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import hashlib
|
|
4
|
+
import json
|
|
5
|
+
import math
|
|
6
|
+
import time
|
|
7
|
+
from typing import Any
|
|
8
|
+
|
|
9
|
+
import libsql as libsql
|
|
10
|
+
|
|
11
|
+
from .types import (
|
|
12
|
+
CandidateMemory,
|
|
13
|
+
IncidentBase,
|
|
14
|
+
MemoryFactDraft,
|
|
15
|
+
SearchQuery,
|
|
16
|
+
)
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
DDL_TEMPLATES = [
|
|
20
|
+
"""CREATE TABLE IF NOT EXISTS memories (
|
|
21
|
+
id TEXT PRIMARY KEY,
|
|
22
|
+
content TEXT NOT NULL,
|
|
23
|
+
embedding F8_BLOB({dim}),
|
|
24
|
+
category TEXT NOT NULL,
|
|
25
|
+
weight REAL DEFAULT 1.0,
|
|
26
|
+
version INTEGER DEFAULT 1,
|
|
27
|
+
status TEXT DEFAULT 'active',
|
|
28
|
+
preserve_reason TEXT CHECK (preserve_reason IN (
|
|
29
|
+
'user_locked', 'correction', 'core_rule', 'system_proven', 'admin_override'
|
|
30
|
+
)),
|
|
31
|
+
preserved_by TEXT,
|
|
32
|
+
successful_recall_count INTEGER DEFAULT 0,
|
|
33
|
+
created_at INTEGER NOT NULL
|
|
34
|
+
CHECK (created_at > 1000000000 AND created_at < 2000000000),
|
|
35
|
+
last_retrieved INTEGER,
|
|
36
|
+
retrieval_count INTEGER DEFAULT 0,
|
|
37
|
+
scope TEXT NOT NULL,
|
|
38
|
+
incident_id TEXT,
|
|
39
|
+
metadata JSON
|
|
40
|
+
)""",
|
|
41
|
+
"CREATE INDEX IF NOT EXISTS idx_memories_scope_status ON memories(scope, status, category)",
|
|
42
|
+
"""CREATE VIRTUAL TABLE IF NOT EXISTS memories_fts USING fts5(
|
|
43
|
+
content, category,
|
|
44
|
+
content='memories', content_rowid='rowid',
|
|
45
|
+
tokenize='unicode61 remove_diacritics 1'
|
|
46
|
+
)""",
|
|
47
|
+
"""CREATE TRIGGER IF NOT EXISTS memories_ai AFTER INSERT ON memories BEGIN
|
|
48
|
+
INSERT INTO memories_fts(rowid, content, category) VALUES (new.rowid, new.content, new.category);
|
|
49
|
+
END""",
|
|
50
|
+
"""CREATE TRIGGER IF NOT EXISTS memories_au AFTER UPDATE ON memories BEGIN
|
|
51
|
+
INSERT INTO memories_fts(memories_fts, rowid, content, category) VALUES ('delete', old.rowid, old.content, old.category);
|
|
52
|
+
INSERT INTO memories_fts(rowid, content, category) VALUES (new.rowid, new.content, new.category);
|
|
53
|
+
END""",
|
|
54
|
+
"""CREATE TRIGGER IF NOT EXISTS memories_ad AFTER DELETE ON memories BEGIN
|
|
55
|
+
INSERT INTO memories_fts(memories_fts, rowid, content, category) VALUES ('delete', old.rowid, old.content, old.category);
|
|
56
|
+
END""",
|
|
57
|
+
"""CREATE TABLE IF NOT EXISTS tasks (
|
|
58
|
+
id TEXT PRIMARY KEY,
|
|
59
|
+
scope TEXT NOT NULL,
|
|
60
|
+
description TEXT,
|
|
61
|
+
status TEXT,
|
|
62
|
+
complexity_score REAL DEFAULT 1.0,
|
|
63
|
+
tokens_used INTEGER,
|
|
64
|
+
errors INTEGER,
|
|
65
|
+
created_at INTEGER NOT NULL
|
|
66
|
+
)""",
|
|
67
|
+
"""CREATE TABLE IF NOT EXISTS memory_task_attributions (
|
|
68
|
+
memory_id TEXT,
|
|
69
|
+
task_id TEXT,
|
|
70
|
+
similarity REAL,
|
|
71
|
+
credit REAL,
|
|
72
|
+
PRIMARY KEY (memory_id, task_id),
|
|
73
|
+
FOREIGN KEY (memory_id) REFERENCES memories(id),
|
|
74
|
+
FOREIGN KEY (task_id) REFERENCES tasks(id)
|
|
75
|
+
)""",
|
|
76
|
+
]
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
class MemoryStore:
|
|
80
|
+
def __init__(self, conn: libsql.Connection, dimension: int = 384):
|
|
81
|
+
self.conn = conn
|
|
82
|
+
self.dimension = dimension
|
|
83
|
+
self._init_schema()
|
|
84
|
+
|
|
85
|
+
def _init_schema(self):
|
|
86
|
+
for ddl in DDL_TEMPLATES:
|
|
87
|
+
self.conn.execute(ddl.format(dim=self.dimension))
|
|
88
|
+
self.conn.commit()
|
|
89
|
+
|
|
90
|
+
def _sha256(self, data: str) -> str:
|
|
91
|
+
return hashlib.sha256(data.encode()).hexdigest()
|
|
92
|
+
|
|
93
|
+
def write(self, draft: MemoryFactDraft, meta: IncidentBase) -> str:
|
|
94
|
+
# L2 normalize embedding if present
|
|
95
|
+
raw_vector: list[float] = getattr(draft, "embedding", []) or []
|
|
96
|
+
normalized: list[float] = []
|
|
97
|
+
if raw_vector:
|
|
98
|
+
norm = math.sqrt(sum(v * v for v in raw_vector))
|
|
99
|
+
normalized = [v / norm for v in raw_vector]
|
|
100
|
+
if not all(math.isfinite(v) for v in normalized):
|
|
101
|
+
raise ValueError(
|
|
102
|
+
f"[OmniMem] Invalid vector (NaN/Infinity) for incident {meta.incident_id}"
|
|
103
|
+
)
|
|
104
|
+
|
|
105
|
+
# Deterministic ID
|
|
106
|
+
content_hash = self._sha256(draft.content)
|
|
107
|
+
memory_id = self._sha256(f"{meta.scope}|{meta.incident_id}|{content_hash}")
|
|
108
|
+
|
|
109
|
+
# Idempotent write
|
|
110
|
+
self.conn.execute(
|
|
111
|
+
"""INSERT OR IGNORE INTO memories
|
|
112
|
+
(id, content, embedding, category, scope, incident_id, created_at,
|
|
113
|
+
weight, version, preserve_reason, successful_recall_count)
|
|
114
|
+
VALUES (?, ?, vector8(?), ?, ?, ?, strftime('%s','now'), ?, ?, ?, ?)""",
|
|
115
|
+
tuple([
|
|
116
|
+
memory_id,
|
|
117
|
+
draft.content,
|
|
118
|
+
json.dumps(normalized),
|
|
119
|
+
draft.category,
|
|
120
|
+
meta.scope,
|
|
121
|
+
meta.incident_id,
|
|
122
|
+
draft.weight,
|
|
123
|
+
draft.version,
|
|
124
|
+
draft.preserve_reason,
|
|
125
|
+
draft.successful_recall_count,
|
|
126
|
+
]),
|
|
127
|
+
)
|
|
128
|
+
self.conn.commit()
|
|
129
|
+
return memory_id
|
|
130
|
+
|
|
131
|
+
def archive(self, memory_id: str, reason: str, by: str) -> None:
|
|
132
|
+
self.conn.execute(
|
|
133
|
+
"""UPDATE memories SET status='archived', preserved_by=?,
|
|
134
|
+
metadata=json_set(COALESCE(metadata,'{}'),'$.archive_reason',?)
|
|
135
|
+
WHERE id=?""",
|
|
136
|
+
(by, reason, memory_id),
|
|
137
|
+
)
|
|
138
|
+
self.conn.commit()
|
|
139
|
+
|
|
140
|
+
def archive_batch(self, memory_ids: list[str], reason: str, by: str) -> None:
|
|
141
|
+
if not memory_ids:
|
|
142
|
+
return
|
|
143
|
+
placeholders = ",".join(["?"] * len(memory_ids))
|
|
144
|
+
sql = (
|
|
145
|
+
"UPDATE memories SET status='archived', preserved_by=?,"
|
|
146
|
+
"metadata=json_set(COALESCE(metadata,'{}'),'$.archive_reason',?)"
|
|
147
|
+
f"WHERE id IN ({placeholders})"
|
|
148
|
+
)
|
|
149
|
+
self.conn.execute(sql, tuple([by, reason, *memory_ids]))
|
|
150
|
+
self.conn.commit()
|
|
151
|
+
|
|
152
|
+
def get_candidates(self, query: SearchQuery, max_candidates: int = 200) -> list[CandidateMemory]:
|
|
153
|
+
keyword = query.keyword or "*"
|
|
154
|
+
vector_json = json.dumps(query.vector or [])
|
|
155
|
+
|
|
156
|
+
rows = self.conn.execute(
|
|
157
|
+
"""WITH fts_candidates AS (
|
|
158
|
+
SELECT m.rowid, m.id, m.content, m.category, m.weight, m.version,
|
|
159
|
+
m.last_retrieved, m.created_at,
|
|
160
|
+
bm25(memories_fts) as raw_fts_score
|
|
161
|
+
FROM memories m
|
|
162
|
+
JOIN memories_fts ON m.rowid = memories_fts.rowid
|
|
163
|
+
WHERE m.scope = ? AND m.status = 'active' AND memories_fts MATCH ?
|
|
164
|
+
ORDER BY bm25(memories_fts) ASC
|
|
165
|
+
LIMIT ?
|
|
166
|
+
)
|
|
167
|
+
SELECT fc.id, fc.content, fc.category, fc.weight, fc.version,
|
|
168
|
+
fc.raw_fts_score,
|
|
169
|
+
(1.0 - vector_distance_cos(m.embedding, vector8(?))) as raw_vec_score,
|
|
170
|
+
(strftime('%s','now') - fc.last_retrieved) / 86400.0 as days_since_retrieved
|
|
171
|
+
FROM fts_candidates fc
|
|
172
|
+
JOIN memories m ON fc.id = m.id""",
|
|
173
|
+
(query.scope, keyword, max_candidates, vector_json),
|
|
174
|
+
).fetchall()
|
|
175
|
+
|
|
176
|
+
return [
|
|
177
|
+
CandidateMemory(
|
|
178
|
+
id=row[0],
|
|
179
|
+
content=row[1],
|
|
180
|
+
category=row[2],
|
|
181
|
+
weight=row[3],
|
|
182
|
+
version=row[4],
|
|
183
|
+
preserve_reason=None,
|
|
184
|
+
successful_recall_count=0,
|
|
185
|
+
raw_fts_score=row[5],
|
|
186
|
+
raw_vec_score=row[6] or 0.0,
|
|
187
|
+
days_since_retrieved=row[7] or 0.0,
|
|
188
|
+
)
|
|
189
|
+
for row in rows
|
|
190
|
+
]
|
|
191
|
+
|
|
192
|
+
def run_gc(self, scope: str) -> int:
|
|
193
|
+
cursor = self.conn.execute(
|
|
194
|
+
"""UPDATE memories SET status='archived', preserved_by='evolver:gc'
|
|
195
|
+
WHERE scope = ? AND status='active' AND preserve_reason IS NULL
|
|
196
|
+
AND (strftime('%s','now') - created_at) > 86400
|
|
197
|
+
AND (
|
|
198
|
+
(retrieval_count = 0 AND (strftime('%s','now') - created_at) > 2592000)
|
|
199
|
+
OR (retrieval_count > 10 AND weight < 0.2)
|
|
200
|
+
OR (version < 5 AND retrieval_count = 0 AND (strftime('%s','now') - created_at) > 7776000)
|
|
201
|
+
)""",
|
|
202
|
+
(scope,),
|
|
203
|
+
)
|
|
204
|
+
self.conn.commit()
|
|
205
|
+
return cursor.rowcount
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass, field
|
|
4
|
+
from typing import Literal
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
@dataclass
|
|
8
|
+
class IncidentBase:
|
|
9
|
+
scope: str
|
|
10
|
+
incident_id: str
|
|
11
|
+
timestamp: int
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@dataclass
|
|
15
|
+
class Message:
|
|
16
|
+
role: Literal["user", "assistant", "system"]
|
|
17
|
+
content: str
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
@dataclass
|
|
21
|
+
class MessageBatchIncident(IncidentBase):
|
|
22
|
+
type: Literal["message_batch"] = "message_batch"
|
|
23
|
+
messages: list[Message] = field(default_factory=list)
|
|
24
|
+
session_context: dict | None = None
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
@dataclass
|
|
28
|
+
class CheckpointIncident(IncidentBase):
|
|
29
|
+
type: Literal["checkpoint"] = "checkpoint"
|
|
30
|
+
level: str = "auto"
|
|
31
|
+
content: str = ""
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
@dataclass
|
|
35
|
+
class ExtractedFactIncident(IncidentBase):
|
|
36
|
+
type: Literal["extracted_fact"] = "extracted_fact"
|
|
37
|
+
statement: str = ""
|
|
38
|
+
confidence: float = 0.0
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
@dataclass
|
|
42
|
+
class RawEventIncident(IncidentBase):
|
|
43
|
+
type: Literal["raw_event"] = "raw_event"
|
|
44
|
+
payload: dict = field(default_factory=dict)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
Incident = MessageBatchIncident | CheckpointIncident | ExtractedFactIncident | RawEventIncident
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
@dataclass
|
|
51
|
+
class SearchQuery:
|
|
52
|
+
scope: str
|
|
53
|
+
keyword: str | None = None
|
|
54
|
+
vector: list[float] | None = None
|
|
55
|
+
category: list[str] | None = None
|
|
56
|
+
limit: int = 20
|
|
57
|
+
freshness: Literal["eventual", "strong"] = "eventual"
|
|
58
|
+
decay_map: dict[str, float] | None = None
|
|
59
|
+
hybrid_weights: dict[str, float] | None = None
|
|
60
|
+
category_lambda: dict[str, float] | None = None
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
@dataclass
|
|
64
|
+
class CandidateMemory:
|
|
65
|
+
id: str
|
|
66
|
+
content: str
|
|
67
|
+
category: str
|
|
68
|
+
weight: float
|
|
69
|
+
version: int
|
|
70
|
+
preserve_reason: str | None
|
|
71
|
+
successful_recall_count: int
|
|
72
|
+
raw_fts_score: float
|
|
73
|
+
raw_vec_score: float
|
|
74
|
+
days_since_retrieved: float
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
@dataclass
|
|
78
|
+
class RankedMemory(CandidateMemory):
|
|
79
|
+
fts_score: float = 0.0
|
|
80
|
+
vec_score: float = 0.0
|
|
81
|
+
final_score: float = 0.0
|
|
82
|
+
decay_factor: float = 1.0
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
@dataclass
|
|
86
|
+
class MemoryFactDraft:
|
|
87
|
+
content: str
|
|
88
|
+
category: str
|
|
89
|
+
weight: float = 1.0
|
|
90
|
+
version: int = 1
|
|
91
|
+
preserve_reason: str | None = None
|
|
92
|
+
successful_recall_count: int = 0
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "mymemo"
|
|
7
|
+
version = "0.1.1"
|
|
8
|
+
description = "Python SDK for OmniMem — Agent memory ledger on Turso/libSQL"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = "MIT"
|
|
11
|
+
requires-python = ">=3.11"
|
|
12
|
+
authors = [{ name = "OntarioLT" }]
|
|
13
|
+
keywords = ["agent", "memory", "rag", "turso", "libsql", "fts5", "vector"]
|
|
14
|
+
classifiers = [
|
|
15
|
+
"Development Status :: 3 - Alpha",
|
|
16
|
+
"Intended Audience :: Developers",
|
|
17
|
+
"License :: OSI Approved :: MIT License",
|
|
18
|
+
"Programming Language :: Python :: 3.11",
|
|
19
|
+
"Topic :: Software Development :: Libraries",
|
|
20
|
+
]
|
|
21
|
+
dependencies = [
|
|
22
|
+
"libsql>=0.1.0",
|
|
23
|
+
"pydantic>=2.0",
|
|
24
|
+
]
|
|
25
|
+
|
|
26
|
+
[project.urls]
|
|
27
|
+
Homepage = "https://github.com/OntarioLT/OmniMem"
|
|
28
|
+
Repository = "https://github.com/OntarioLT/OmniMem"
|
|
29
|
+
Issues = "https://github.com/OntarioLT/OmniMem/issues"
|
|
30
|
+
|
|
31
|
+
[project.optional-dependencies]
|
|
32
|
+
dev = [
|
|
33
|
+
"pytest>=8.0",
|
|
34
|
+
"pytest-asyncio>=0.23",
|
|
35
|
+
]
|
|
36
|
+
|
|
37
|
+
[tool.pytest.ini_options]
|
|
38
|
+
asyncio_mode = "auto"
|
|
39
|
+
addopts = "-v"
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
import hashlib
|
|
2
|
+
import math
|
|
3
|
+
import time
|
|
4
|
+
|
|
5
|
+
import libsql as libsql
|
|
6
|
+
import pytest
|
|
7
|
+
|
|
8
|
+
from mymemo.store import MemoryStore
|
|
9
|
+
from mymemo.types import (
|
|
10
|
+
IncidentBase,
|
|
11
|
+
MemoryFactDraft,
|
|
12
|
+
SearchQuery,
|
|
13
|
+
CandidateMemory,
|
|
14
|
+
)
|
|
15
|
+
from mymemo.recaller import rank
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@pytest.fixture
|
|
19
|
+
def store():
|
|
20
|
+
conn = libsql.connect(":memory:")
|
|
21
|
+
return MemoryStore(conn, dimension=384)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
@pytest.fixture
|
|
25
|
+
def store_with_data(store: MemoryStore):
|
|
26
|
+
meta = IncidentBase(scope="test", incident_id="inc1", timestamp=int(time.time()))
|
|
27
|
+
store.write(MemoryFactDraft(content="TypeScript is great", category="insight"), meta)
|
|
28
|
+
store.write(MemoryFactDraft(content="Python is also great", category="insight"), meta)
|
|
29
|
+
return store
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class TestMemoryStore:
|
|
33
|
+
def test_idempotent_writes(self, store: MemoryStore):
|
|
34
|
+
"""相同内容 + 相同 scope/incident → 返回相同 SHA256 ID"""
|
|
35
|
+
meta = IncidentBase(scope="s", incident_id="i1", timestamp=1000000001)
|
|
36
|
+
id1 = store.write(MemoryFactDraft(content="hello", category="a"), meta)
|
|
37
|
+
id2 = store.write(MemoryFactDraft(content="hello", category="a"), meta)
|
|
38
|
+
assert id1 == id2
|
|
39
|
+
|
|
40
|
+
def test_different_content_different_id(self, store: MemoryStore):
|
|
41
|
+
"""不同内容产生不同的确定性 ID"""
|
|
42
|
+
meta = IncidentBase(scope="s", incident_id="i1", timestamp=1000000001)
|
|
43
|
+
id1 = store.write(MemoryFactDraft(content="a", category="x"), meta)
|
|
44
|
+
id2 = store.write(MemoryFactDraft(content="b", category="x"), meta)
|
|
45
|
+
assert id1 != id2
|
|
46
|
+
|
|
47
|
+
def test_soft_archive(self, store: MemoryStore):
|
|
48
|
+
"""软删除 — 设置 status=archived,不物理删除"""
|
|
49
|
+
meta = IncidentBase(scope="s", incident_id="i1", timestamp=1000000001)
|
|
50
|
+
mid = store.write(MemoryFactDraft(content="x", category="c"), meta)
|
|
51
|
+
store.archive(mid, "test", "user")
|
|
52
|
+
row = store.conn.execute("SELECT status FROM memories WHERE id=?", (mid,)).fetchone()
|
|
53
|
+
assert row[0] == "archived"
|
|
54
|
+
|
|
55
|
+
def test_batch_archive(self, store: MemoryStore):
|
|
56
|
+
"""批量归档 — 单次查询处理多条记忆"""
|
|
57
|
+
meta = IncidentBase(scope="s", incident_id="i1", timestamp=1000000001)
|
|
58
|
+
id1 = store.write(MemoryFactDraft(content="a", category="c"), meta)
|
|
59
|
+
id2 = store.write(MemoryFactDraft(content="b", category="c"), meta)
|
|
60
|
+
store.archive_batch([id1, id2], "batch", "sys")
|
|
61
|
+
rows = store.conn.execute("SELECT status FROM memories WHERE id IN (?,?)", (id1, id2)).fetchall()
|
|
62
|
+
assert all(r[0] == "archived" for r in rows)
|
|
63
|
+
|
|
64
|
+
def test_hybrid_search(self, store_with_data: MemoryStore):
|
|
65
|
+
"""FTS5 全文搜索返回匹配结果"""
|
|
66
|
+
results = store_with_data.get_candidates(SearchQuery(scope="test", keyword="TypeScript"))
|
|
67
|
+
assert len(results) >= 1
|
|
68
|
+
assert any("TypeScript" in r.content for r in results)
|
|
69
|
+
|
|
70
|
+
def test_empty_search_result(self, store: MemoryStore):
|
|
71
|
+
"""无匹配时返回空列表"""
|
|
72
|
+
results = store.get_candidates(SearchQuery(scope="nonexistent", keyword="x"))
|
|
73
|
+
assert results == []
|
|
74
|
+
|
|
75
|
+
def test_gc_archives_eligible_memories(self, store: MemoryStore):
|
|
76
|
+
"""GC 淘汰过期、未检索、低权重的记忆"""
|
|
77
|
+
store.conn.execute(
|
|
78
|
+
"INSERT INTO memories (id, content, category, scope, incident_id, created_at, retrieval_count, weight, version) "
|
|
79
|
+
"VALUES ('gc1', 'old', 'ephemeral', 'gc_scope', 'inc', 1000000001, 0, 1.0, 1)"
|
|
80
|
+
)
|
|
81
|
+
store.conn.commit()
|
|
82
|
+
count = store.run_gc("gc_scope")
|
|
83
|
+
assert count >= 1
|
|
84
|
+
|
|
85
|
+
def test_nan_vector_rejected(self, store: MemoryStore):
|
|
86
|
+
"""NaN/Infinity 向量被拒绝,防止脏数据入库"""
|
|
87
|
+
meta = IncidentBase(scope="s", incident_id="i1", timestamp=1000000001)
|
|
88
|
+
draft = MemoryFactDraft(content="x", category="c")
|
|
89
|
+
draft.embedding = [float("nan"), 0.0, 0.0] # type: ignore
|
|
90
|
+
with pytest.raises(ValueError, match="NaN/Infinity"):
|
|
91
|
+
store.write(draft, meta)
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
class TestAdaptiveRecaller:
|
|
95
|
+
def test_ranking_by_hybrid_score(self):
|
|
96
|
+
"""按文本+向量混合评分排序"""
|
|
97
|
+
c1 = CandidateMemory("1", "a", "insight", 1.0, 1, None, 0, -5.0, 0.3, 0)
|
|
98
|
+
c2 = CandidateMemory("2", "b", "insight", 1.0, 1, None, 0, -0.5, 0.9, 0)
|
|
99
|
+
q = SearchQuery(scope="test")
|
|
100
|
+
ranked = rank([c1, c2], q)
|
|
101
|
+
assert ranked[0].id == "2"
|
|
102
|
+
|
|
103
|
+
def test_filters_irrelevant_results(self):
|
|
104
|
+
"""低分结果被过滤(score < 0.15)"""
|
|
105
|
+
c = CandidateMemory("1", "a", "insight", 1.0, 1, None, 0, -100, 0.01, 0)
|
|
106
|
+
q = SearchQuery(scope="test")
|
|
107
|
+
ranked = rank([c], q)
|
|
108
|
+
assert len(ranked) == 0
|
|
109
|
+
|
|
110
|
+
def test_time_decay(self):
|
|
111
|
+
"""时间衰减 — 新记忆排名高于旧记忆"""
|
|
112
|
+
c_fresh = CandidateMemory("f", "a", "insight", 1.0, 1, None, 0, -0.1, 0.9, 0)
|
|
113
|
+
c_old = CandidateMemory("o", "a", "insight", 1.0, 1, None, 0, -0.1, 0.9, 30)
|
|
114
|
+
q = SearchQuery(scope="test")
|
|
115
|
+
ranked = rank([c_fresh, c_old], q)
|
|
116
|
+
assert ranked[0].id == "f"
|