ragleap-memory 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
"""ragleap-memory: durable, tenant-scoped long-term memory for LLM applications.
|
|
2
|
+
|
|
3
|
+
Needs SQLite with FTS5. Not independently audited. See docs/threat-model.md."""
|
|
4
|
+
from ragleap_memory.model import (KINDS, TRUSTS, InvalidEntryError, MemoryEntry, MemoryStoreError, NotFoundError,
|
|
5
|
+
SearchHit)
|
|
6
|
+
from ragleap_memory.sqlite_store import SCHEMA_VERSION, SQLiteStore
|
|
7
|
+
|
|
8
|
+
__version__ = "0.1.0"
|
|
9
|
+
|
|
10
|
+
__all__ = [
|
|
11
|
+
"InvalidEntryError", "KINDS", "MemoryEntry", "MemoryStoreError", "NotFoundError", "SCHEMA_VERSION",
|
|
12
|
+
"SQLiteStore", "SearchHit", "TRUSTS", "__version__",
|
|
13
|
+
]
|
ragleap_memory/model.py
ADDED
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
"""Data model for ragleap-memory: entries, search hits, trust labels and errors."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import re
|
|
5
|
+
from dataclasses import dataclass, field
|
|
6
|
+
from typing import Any, Dict, Tuple
|
|
7
|
+
|
|
8
|
+
from ragleap_security import Scope
|
|
9
|
+
|
|
10
|
+
KINDS = ("episode", "fact", "summary")
|
|
11
|
+
# Lowest to highest. "owner": written by the owner. "model": written by a model. "untrusted": derived
|
|
12
|
+
# from content nobody vouched for. The default for new entries is the lowest.
|
|
13
|
+
TRUSTS = ("untrusted", "model", "owner")
|
|
14
|
+
|
|
15
|
+
ID_RE = re.compile(r"[0-9a-f]{32}\Z")
|
|
16
|
+
SOURCE_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.:@/-]{0,127}\Z")
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class MemoryStoreError(Exception):
|
|
20
|
+
"""Base class for memory store errors. Messages never contain entry text."""
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class InvalidEntryError(MemoryStoreError, ValueError):
|
|
24
|
+
"""An entry or argument is invalid."""
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class NotFoundError(MemoryStoreError):
|
|
28
|
+
"""No such entry, or it is not visible to the scope that asked. The two cases look the same."""
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def trust_rank(trust: str) -> int:
|
|
32
|
+
return TRUSTS.index(trust)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
@dataclass(frozen=True)
|
|
36
|
+
class MemoryEntry:
|
|
37
|
+
id: str
|
|
38
|
+
scope: Scope
|
|
39
|
+
kind: str
|
|
40
|
+
text: str
|
|
41
|
+
source: str
|
|
42
|
+
trust: str
|
|
43
|
+
importance: float
|
|
44
|
+
created_at: str
|
|
45
|
+
expires_at: "str | None" = None
|
|
46
|
+
superseded_by: "str | None" = None
|
|
47
|
+
derived_from: Tuple[str, ...] = ()
|
|
48
|
+
metadata: Dict[str, Any] = field(default_factory=dict)
|
|
49
|
+
|
|
50
|
+
def to_dict(self) -> Dict[str, Any]:
|
|
51
|
+
return {
|
|
52
|
+
"id": self.id, "scope": self.scope.key, "kind": self.kind, "text": self.text,
|
|
53
|
+
"source": self.source, "trust": self.trust, "importance": self.importance,
|
|
54
|
+
"created_at": self.created_at, "expires_at": self.expires_at,
|
|
55
|
+
"superseded_by": self.superseded_by, "derived_from": list(self.derived_from),
|
|
56
|
+
"metadata": self.metadata,
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
@dataclass(frozen=True)
|
|
61
|
+
class SearchHit:
|
|
62
|
+
entry: MemoryEntry
|
|
63
|
+
score: float # higher is more relevant (negated SQLite bm25)
|
|
@@ -0,0 +1,322 @@
|
|
|
1
|
+
"""SQLiteStore: durable memory in one SQLite file (or in memory), with FTS5 keyword search.
|
|
2
|
+
|
|
3
|
+
Every operation takes a Scope and is filtered by it inside the SQL, not by the caller. A scope sees
|
|
4
|
+
the entries whose scope it covers: a tenant-wide scope sees everything in the tenant, a narrower scope
|
|
5
|
+
sees only its own subject/agent/session, and an entry without a subject is not visible to a scope that
|
|
6
|
+
names one."""
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import contextlib
|
|
10
|
+
import json
|
|
11
|
+
import os
|
|
12
|
+
import re
|
|
13
|
+
import sqlite3
|
|
14
|
+
import threading
|
|
15
|
+
import uuid
|
|
16
|
+
from datetime import datetime, timezone
|
|
17
|
+
from typing import Any, Callable, Dict, Iterable, Iterator, List, Optional, Sequence
|
|
18
|
+
|
|
19
|
+
from ragleap_security import (AuditLog, Scope, check_json_shape, check_size, require_scope)
|
|
20
|
+
|
|
21
|
+
from .model import (ID_RE, KINDS, SOURCE_RE, TRUSTS, InvalidEntryError, MemoryEntry, MemoryStoreError,
|
|
22
|
+
NotFoundError, SearchHit, trust_rank)
|
|
23
|
+
|
|
24
|
+
SCHEMA_VERSION = 1
|
|
25
|
+
_SCOPE_FIELDS = ("tenant", "subject", "agent", "session")
|
|
26
|
+
_TOKEN = re.compile(r"\w+")
|
|
27
|
+
|
|
28
|
+
_SCHEMA = """
|
|
29
|
+
CREATE TABLE IF NOT EXISTS memories(
|
|
30
|
+
rowid INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
31
|
+
id TEXT NOT NULL UNIQUE,
|
|
32
|
+
tenant TEXT NOT NULL, subject TEXT, agent TEXT, session TEXT,
|
|
33
|
+
kind TEXT NOT NULL CHECK(kind IN ('episode','fact','summary')),
|
|
34
|
+
text TEXT NOT NULL,
|
|
35
|
+
source TEXT NOT NULL,
|
|
36
|
+
trust TEXT NOT NULL CHECK(trust IN ('untrusted','model','owner')),
|
|
37
|
+
importance REAL NOT NULL CHECK(importance >= 0 AND importance <= 1),
|
|
38
|
+
created_at TEXT NOT NULL,
|
|
39
|
+
expires_at TEXT,
|
|
40
|
+
superseded_by TEXT,
|
|
41
|
+
derived_from TEXT NOT NULL,
|
|
42
|
+
metadata TEXT NOT NULL
|
|
43
|
+
);
|
|
44
|
+
CREATE INDEX IF NOT EXISTS memories_scope ON memories(tenant, subject, agent, session);
|
|
45
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS memories_fts USING fts5(
|
|
46
|
+
text, content='memories', content_rowid='rowid', tokenize='unicode61 remove_diacritics 2');
|
|
47
|
+
CREATE TRIGGER IF NOT EXISTS memories_ai AFTER INSERT ON memories BEGIN
|
|
48
|
+
INSERT INTO memories_fts(rowid, text) VALUES (new.rowid, new.text);
|
|
49
|
+
END;
|
|
50
|
+
CREATE TRIGGER IF NOT EXISTS memories_ad AFTER DELETE ON memories BEGIN
|
|
51
|
+
INSERT INTO memories_fts(memories_fts, rowid, text) VALUES ('delete', old.rowid, old.text);
|
|
52
|
+
END;
|
|
53
|
+
"""
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _iso(dt: datetime) -> str:
|
|
57
|
+
return dt.astimezone(timezone.utc).isoformat(timespec="microseconds")
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def _scope_clause(scope: Scope, alias: str = "") -> "tuple[str, list]":
|
|
61
|
+
"""SQL condition that matches exactly the entries `scope` covers."""
|
|
62
|
+
p = f"{alias}." if alias else ""
|
|
63
|
+
parts, params = [f"{p}tenant = ?"], [scope.tenant]
|
|
64
|
+
for f in _SCOPE_FIELDS[1:]:
|
|
65
|
+
v = getattr(scope, f)
|
|
66
|
+
if v is not None:
|
|
67
|
+
parts.append(f"{p}{f} = ?")
|
|
68
|
+
params.append(v)
|
|
69
|
+
return " AND ".join(parts), params
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _row_to_entry(row: sqlite3.Row) -> MemoryEntry:
|
|
73
|
+
return MemoryEntry(
|
|
74
|
+
id=row["id"],
|
|
75
|
+
scope=Scope(row["tenant"], row["subject"], row["agent"], row["session"]),
|
|
76
|
+
kind=row["kind"], text=row["text"], source=row["source"], trust=row["trust"],
|
|
77
|
+
importance=row["importance"], created_at=row["created_at"], expires_at=row["expires_at"],
|
|
78
|
+
superseded_by=row["superseded_by"], derived_from=tuple(json.loads(row["derived_from"])),
|
|
79
|
+
metadata=json.loads(row["metadata"]),
|
|
80
|
+
)
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
class SQLiteStore:
|
|
84
|
+
def __init__(self, path: str = ":memory:", *, audit: Optional[AuditLog] = None,
|
|
85
|
+
clock: Optional[Callable[[], datetime]] = None, max_text_bytes: int = 8192,
|
|
86
|
+
max_results: int = 50):
|
|
87
|
+
self.path = os.fspath(path)
|
|
88
|
+
self._audit = audit
|
|
89
|
+
self._clock = clock or (lambda: datetime.now(timezone.utc))
|
|
90
|
+
self.max_text_bytes = max_text_bytes
|
|
91
|
+
self.max_results = max_results
|
|
92
|
+
self._lock = threading.RLock()
|
|
93
|
+
if self.path != ":memory:" and not os.path.exists(self.path):
|
|
94
|
+
os.close(os.open(self.path, os.O_WRONLY | os.O_CREAT, 0o600))
|
|
95
|
+
self._db = sqlite3.connect(self.path, check_same_thread=False, isolation_level=None)
|
|
96
|
+
self._db.row_factory = sqlite3.Row
|
|
97
|
+
self._db.execute("PRAGMA busy_timeout = 5000")
|
|
98
|
+
self._db.execute("PRAGMA secure_delete = ON")
|
|
99
|
+
if self.path != ":memory:":
|
|
100
|
+
self._db.execute("PRAGMA journal_mode = WAL")
|
|
101
|
+
version = self._db.execute("PRAGMA user_version").fetchone()[0]
|
|
102
|
+
if version > SCHEMA_VERSION:
|
|
103
|
+
self._db.close()
|
|
104
|
+
raise MemoryStoreError(f"database schema version {version} is newer than this library supports")
|
|
105
|
+
try:
|
|
106
|
+
self._db.executescript(_SCHEMA)
|
|
107
|
+
except sqlite3.OperationalError as e:
|
|
108
|
+
self._db.close()
|
|
109
|
+
raise MemoryStoreError("SQLite with FTS5 is required: %s" % e) from None
|
|
110
|
+
self._db.execute(f"PRAGMA user_version = {SCHEMA_VERSION}")
|
|
111
|
+
|
|
112
|
+
# -- helpers -----------------------------------------------------------------------------------
|
|
113
|
+
|
|
114
|
+
@contextlib.contextmanager
|
|
115
|
+
def _tx(self) -> Iterator[sqlite3.Connection]:
|
|
116
|
+
with self._lock:
|
|
117
|
+
self._db.execute("BEGIN IMMEDIATE")
|
|
118
|
+
try:
|
|
119
|
+
yield self._db
|
|
120
|
+
except BaseException:
|
|
121
|
+
self._db.execute("ROLLBACK")
|
|
122
|
+
raise
|
|
123
|
+
else:
|
|
124
|
+
self._db.execute("COMMIT")
|
|
125
|
+
|
|
126
|
+
def _log(self, event: str, scope: Scope, data: Dict[str, Any]) -> None:
|
|
127
|
+
if self._audit is not None: # inside the transaction: if the audit write fails, the change is rolled back
|
|
128
|
+
self._audit.append(event, data, scope)
|
|
129
|
+
|
|
130
|
+
def _visible(self, db: sqlite3.Connection, scope: Scope, entry_id: str) -> sqlite3.Row:
|
|
131
|
+
clause, params = _scope_clause(scope)
|
|
132
|
+
row = db.execute(f"SELECT * FROM memories WHERE id = ? AND {clause}", [entry_id, *params]).fetchone()
|
|
133
|
+
if row is None:
|
|
134
|
+
raise NotFoundError("no such entry")
|
|
135
|
+
return row
|
|
136
|
+
|
|
137
|
+
def _validate_new(self, scope: Any, text: Any, kind: str, source: str, trust: str, importance: Any,
|
|
138
|
+
expires_at: Any, metadata: Any) -> "tuple[Scope, str | None, str]":
|
|
139
|
+
scope = require_scope(scope)
|
|
140
|
+
if kind not in KINDS:
|
|
141
|
+
raise InvalidEntryError("invalid kind")
|
|
142
|
+
if trust not in TRUSTS:
|
|
143
|
+
raise InvalidEntryError("invalid trust")
|
|
144
|
+
if not isinstance(source, str) or not SOURCE_RE.match(source):
|
|
145
|
+
raise InvalidEntryError("invalid source")
|
|
146
|
+
if isinstance(importance, bool) or not isinstance(importance, (int, float)) or not 0 <= importance <= 1:
|
|
147
|
+
raise InvalidEntryError("importance must be a number from 0 to 1")
|
|
148
|
+
if not isinstance(text, str) or not text.strip():
|
|
149
|
+
raise InvalidEntryError("text must be a non-empty string")
|
|
150
|
+
check_size(text, self.max_text_bytes, "text")
|
|
151
|
+
exp = None
|
|
152
|
+
if expires_at is not None:
|
|
153
|
+
if not isinstance(expires_at, datetime) or expires_at.tzinfo is None:
|
|
154
|
+
raise InvalidEntryError("expires_at must be a timezone-aware datetime")
|
|
155
|
+
exp = _iso(expires_at)
|
|
156
|
+
if metadata is None:
|
|
157
|
+
metadata = {}
|
|
158
|
+
if not isinstance(metadata, dict):
|
|
159
|
+
raise InvalidEntryError("metadata must be a dict")
|
|
160
|
+
check_json_shape(metadata, max_depth=8, max_nodes=1000)
|
|
161
|
+
try:
|
|
162
|
+
meta_json = json.dumps(metadata, sort_keys=True)
|
|
163
|
+
except (TypeError, ValueError):
|
|
164
|
+
raise InvalidEntryError("metadata must be JSON serializable") from None
|
|
165
|
+
check_size(meta_json, 4096, "metadata")
|
|
166
|
+
return scope, exp, meta_json
|
|
167
|
+
|
|
168
|
+
def _insert(self, db: sqlite3.Connection, scope: Scope, text: str, kind: str, source: str, trust: str,
|
|
169
|
+
importance: float, exp: Optional[str], derived: Sequence[str], meta_json: str) -> MemoryEntry:
|
|
170
|
+
entry_id = uuid.uuid4().hex
|
|
171
|
+
created = _iso(self._clock())
|
|
172
|
+
db.execute(
|
|
173
|
+
"INSERT INTO memories(id, tenant, subject, agent, session, kind, text, source, trust, importance,"
|
|
174
|
+
" created_at, expires_at, superseded_by, derived_from, metadata) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,NULL,?,?)",
|
|
175
|
+
[entry_id, scope.tenant, scope.subject, scope.agent, scope.session, kind, text, source, trust,
|
|
176
|
+
float(importance), created, exp, json.dumps(list(derived)), meta_json])
|
|
177
|
+
return _row_to_entry(db.execute("SELECT * FROM memories WHERE id = ?", [entry_id]).fetchone())
|
|
178
|
+
|
|
179
|
+
# -- public API --------------------------------------------------------------------------------
|
|
180
|
+
|
|
181
|
+
def add(self, scope: Scope, text: str, *, kind: str = "fact", source: str = "api", trust: str = "untrusted",
|
|
182
|
+
importance: float = 0.5, expires_at: Optional[datetime] = None, derived_from: Iterable[str] = (),
|
|
183
|
+
metadata: Optional[Dict[str, Any]] = None) -> MemoryEntry:
|
|
184
|
+
"""Store an entry in exactly `scope`. trust defaults to the lowest level ("untrusted"): say
|
|
185
|
+
"model" or "owner" explicitly. Every id in derived_from must be visible to `scope`."""
|
|
186
|
+
scope, exp, meta_json = self._validate_new(scope, text, kind, source, trust, importance, expires_at, metadata)
|
|
187
|
+
derived = list(derived_from)
|
|
188
|
+
if len(derived) > 100 or len(set(derived)) != len(derived) or not all(isinstance(d, str) and ID_RE.match(d) for d in derived):
|
|
189
|
+
raise InvalidEntryError("derived_from must be up to 100 distinct entry ids")
|
|
190
|
+
with self._tx() as db:
|
|
191
|
+
for d in derived:
|
|
192
|
+
self._visible(db, scope, d)
|
|
193
|
+
entry = self._insert(db, scope, text, kind, source, trust, importance, exp, derived, meta_json)
|
|
194
|
+
self._log("memory.add", scope, {"id": entry.id, "kind": kind, "trust": trust})
|
|
195
|
+
return entry
|
|
196
|
+
|
|
197
|
+
def get(self, scope: Scope, entry_id: str) -> MemoryEntry:
|
|
198
|
+
scope = require_scope(scope)
|
|
199
|
+
with self._lock:
|
|
200
|
+
return _row_to_entry(self._visible(self._db, scope, entry_id))
|
|
201
|
+
|
|
202
|
+
def count(self, scope: Scope) -> int:
|
|
203
|
+
scope = require_scope(scope)
|
|
204
|
+
clause, params = _scope_clause(scope)
|
|
205
|
+
with self._lock:
|
|
206
|
+
return self._db.execute(f"SELECT COUNT(*) FROM memories WHERE {clause}", params).fetchone()[0]
|
|
207
|
+
|
|
208
|
+
def search(self, scope: Scope, query: str, *, k: int = 5, kinds: Optional[Sequence[str]] = None,
|
|
209
|
+
min_trust: str = "untrusted", include_superseded: bool = False,
|
|
210
|
+
include_expired: bool = False) -> List[SearchHit]:
|
|
211
|
+
"""Keyword search (FTS5, bm25) over the entries `scope` can see. The query is split into
|
|
212
|
+
words; any word may match. Operators in the query are treated as plain words."""
|
|
213
|
+
scope = require_scope(scope)
|
|
214
|
+
if not isinstance(query, str):
|
|
215
|
+
raise InvalidEntryError("query must be a string")
|
|
216
|
+
check_size(query, 1000, "query")
|
|
217
|
+
if isinstance(k, bool) or not isinstance(k, int) or k < 1:
|
|
218
|
+
raise InvalidEntryError("k must be a positive int")
|
|
219
|
+
k = min(k, self.max_results)
|
|
220
|
+
if min_trust not in TRUSTS:
|
|
221
|
+
raise InvalidEntryError("invalid min_trust")
|
|
222
|
+
kinds = tuple(kinds) if kinds is not None else KINDS
|
|
223
|
+
if not kinds or any(x not in KINDS for x in kinds):
|
|
224
|
+
raise InvalidEntryError("invalid kinds")
|
|
225
|
+
tokens = _TOKEN.findall(query)[:32]
|
|
226
|
+
if not tokens:
|
|
227
|
+
return []
|
|
228
|
+
match = " OR ".join('"%s"' % t for t in tokens)
|
|
229
|
+
clause, params = _scope_clause(scope, "m")
|
|
230
|
+
trusts = [t for t in TRUSTS if trust_rank(t) >= trust_rank(min_trust)]
|
|
231
|
+
sql = (f"SELECT m.*, bm25(memories_fts) AS score FROM memories_fts JOIN memories m ON m.rowid = memories_fts.rowid"
|
|
232
|
+
f" WHERE memories_fts MATCH ? AND {clause}"
|
|
233
|
+
f" AND m.kind IN ({','.join('?' * len(kinds))}) AND m.trust IN ({','.join('?' * len(trusts))})")
|
|
234
|
+
args: list = [match, *params, *kinds, *trusts]
|
|
235
|
+
if not include_superseded:
|
|
236
|
+
sql += " AND m.superseded_by IS NULL"
|
|
237
|
+
if not include_expired:
|
|
238
|
+
sql += " AND (m.expires_at IS NULL OR m.expires_at > ?)"
|
|
239
|
+
args.append(_iso(self._clock()))
|
|
240
|
+
sql += " ORDER BY score, m.created_at DESC LIMIT ?"
|
|
241
|
+
args.append(k)
|
|
242
|
+
with self._lock:
|
|
243
|
+
rows = self._db.execute(sql, args).fetchall()
|
|
244
|
+
return [SearchHit(_row_to_entry(r), -float(r["score"])) for r in rows]
|
|
245
|
+
|
|
246
|
+
def supersede(self, scope: Scope, old_id: str, text: str, *, kind: Optional[str] = None, source: str = "api",
|
|
247
|
+
trust: str = "untrusted", importance: Optional[float] = None,
|
|
248
|
+
expires_at: Optional[datetime] = None, metadata: Optional[Dict[str, Any]] = None) -> MemoryEntry:
|
|
249
|
+
"""Replace an entry whose content changed. The new entry gets the old entry's exact scope; the
|
|
250
|
+
old one stays in the store (see export, include_superseded) and points at its replacement."""
|
|
251
|
+
scope = require_scope(scope)
|
|
252
|
+
with self._tx() as db:
|
|
253
|
+
old = self._visible(db, scope, old_id)
|
|
254
|
+
if old["superseded_by"] is not None:
|
|
255
|
+
raise InvalidEntryError("entry is already superseded")
|
|
256
|
+
old_scope = Scope(old["tenant"], old["subject"], old["agent"], old["session"])
|
|
257
|
+
kind = kind or old["kind"]
|
|
258
|
+
importance = old["importance"] if importance is None else importance
|
|
259
|
+
_, exp, meta_json = self._validate_new(old_scope, text, kind, source, trust, importance, expires_at, metadata)
|
|
260
|
+
new = self._insert(db, old_scope, text, kind, source, trust, importance, exp, [old_id], meta_json)
|
|
261
|
+
db.execute("UPDATE memories SET superseded_by = ? WHERE id = ?", [new.id, old_id])
|
|
262
|
+
self._log("memory.supersede", scope, {"old": old_id, "new": new.id})
|
|
263
|
+
return new
|
|
264
|
+
|
|
265
|
+
def forget(self, scope: Scope, ids: Iterable[str]) -> int:
|
|
266
|
+
"""Delete the listed entries that `scope` can see; returns how many were deleted. They stop
|
|
267
|
+
appearing in searches at once, but traces can stay in the database file until compact()
|
|
268
|
+
or erase() runs."""
|
|
269
|
+
scope = require_scope(scope)
|
|
270
|
+
ids = list(ids)
|
|
271
|
+
if len(ids) > 1000 or not all(isinstance(i, str) and ID_RE.match(i) for i in ids):
|
|
272
|
+
raise InvalidEntryError("ids must be up to 1000 entry ids")
|
|
273
|
+
if not ids:
|
|
274
|
+
return 0
|
|
275
|
+
clause, params = _scope_clause(scope)
|
|
276
|
+
with self._tx() as db:
|
|
277
|
+
n = db.execute(f"DELETE FROM memories WHERE id IN ({','.join('?' * len(ids))}) AND {clause}",
|
|
278
|
+
[*ids, *params]).rowcount
|
|
279
|
+
self._log("memory.forget", scope, {"count": n})
|
|
280
|
+
return n
|
|
281
|
+
|
|
282
|
+
def export(self, scope: Scope) -> List[Dict[str, Any]]:
|
|
283
|
+
"""Every entry `scope` can see (superseded and expired ones too) as plain dicts."""
|
|
284
|
+
scope = require_scope(scope)
|
|
285
|
+
clause, params = _scope_clause(scope)
|
|
286
|
+
with self._tx() as db:
|
|
287
|
+
rows = db.execute(f"SELECT * FROM memories WHERE {clause} ORDER BY created_at, id", params).fetchall()
|
|
288
|
+
self._log("memory.export", scope, {"count": len(rows)})
|
|
289
|
+
return [_row_to_entry(r).to_dict() for r in rows]
|
|
290
|
+
|
|
291
|
+
def erase(self, scope: Scope) -> int:
|
|
292
|
+
"""Delete everything `scope` can see, then rewrite the search index and checkpoint the
|
|
293
|
+
journal so the deleted text is no longer in the database file. Returns the number deleted."""
|
|
294
|
+
scope = require_scope(scope)
|
|
295
|
+
clause, params = _scope_clause(scope)
|
|
296
|
+
with self._tx() as db:
|
|
297
|
+
n = db.execute(f"DELETE FROM memories WHERE {clause}", params).rowcount
|
|
298
|
+
self._log("memory.erase", scope, {"count": n})
|
|
299
|
+
self.compact()
|
|
300
|
+
return n
|
|
301
|
+
|
|
302
|
+
def sweep_expired(self, scope: Scope) -> int:
|
|
303
|
+
"""Delete expired entries inside `scope`; returns how many."""
|
|
304
|
+
scope = require_scope(scope)
|
|
305
|
+
clause, params = _scope_clause(scope)
|
|
306
|
+
with self._tx() as db:
|
|
307
|
+
n = db.execute(f"DELETE FROM memories WHERE {clause} AND expires_at IS NOT NULL AND expires_at <= ?",
|
|
308
|
+
[*params, _iso(self._clock())]).rowcount
|
|
309
|
+
self._log("memory.sweep", scope, {"count": n})
|
|
310
|
+
return n
|
|
311
|
+
|
|
312
|
+
def compact(self) -> None:
|
|
313
|
+
"""Rewrite the full-text index and checkpoint the journal. Needed to remove deleted text from
|
|
314
|
+
the file; erase() calls it. Can take a while on a large store."""
|
|
315
|
+
with self._lock:
|
|
316
|
+
self._db.execute("INSERT INTO memories_fts(memories_fts) VALUES ('optimize')")
|
|
317
|
+
if self.path != ":memory:":
|
|
318
|
+
self._db.execute("PRAGMA wal_checkpoint(TRUNCATE)")
|
|
319
|
+
|
|
320
|
+
def close(self) -> None:
|
|
321
|
+
with self._lock:
|
|
322
|
+
self._db.close()
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: ragleap-memory
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Durable, tenant-scoped long-term memory for LLM applications: typed entries with trust labels, SQLite full-text search, supersession of changed facts, per-scope export and erase, and an optional tamper-evident audit log. Standard library plus ragleap-security. Single-tenant is one fixed scope. Not independently audited.
|
|
5
|
+
Project-URL: Homepage, https://github.com/ragleap/ragleap-core
|
|
6
|
+
Project-URL: Repository, https://github.com/ragleap/ragleap-core
|
|
7
|
+
Project-URL: Issues, https://github.com/ragleap/ragleap-core/issues
|
|
8
|
+
Author: RagLeap
|
|
9
|
+
License-Expression: MIT
|
|
10
|
+
Keywords: byok,fts5,llm,long-term-memory,memory,multi-tenant,self-hosted,sqlite
|
|
11
|
+
Classifier: Development Status :: 3 - Alpha
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
19
|
+
Requires-Python: >=3.10
|
|
20
|
+
Requires-Dist: ragleap-security>=0.1.0
|
|
21
|
+
Provides-Extra: test
|
|
22
|
+
Requires-Dist: pytest>=8.0.0; extra == 'test'
|
|
23
|
+
Description-Content-Type: text/markdown
|
|
24
|
+
|
|
25
|
+
# ragleap-memory
|
|
26
|
+
|
|
27
|
+
Durable, tenant-scoped long-term memory for LLM applications. Alpha. **Not independently audited**: see `docs/threat-model.md` for exactly what is and is not defended. Single-tenant use is simply one fixed `Scope`.
|
|
28
|
+
|
|
29
|
+
## What is in 0.1.0
|
|
30
|
+
|
|
31
|
+
- `SQLiteStore`: one SQLite file (or memory), FTS5 keyword search, no server. Needs SQLite built with FTS5; the store refuses to start without it.
|
|
32
|
+
- Every call takes a `Scope` (from ragleap-security) and is filtered by it inside the SQL. A tenant-wide scope sees its whole tenant; a narrower scope sees only its own subject, agent and session.
|
|
33
|
+
- Typed entries (`episode`, `fact`, `summary`) with a trust label (`untrusted`, `model`, `owner`). New entries default to `untrusted`.
|
|
34
|
+
- `supersede` for facts that change (history is kept), `forget`, `export`, `erase` (removes the text from the file), `sweep_expired`, and an optional `AuditLog` that records operations without entry text.
|
|
35
|
+
|
|
36
|
+
```python
|
|
37
|
+
from ragleap_memory import SQLiteStore
|
|
38
|
+
from ragleap_security import Scope
|
|
39
|
+
|
|
40
|
+
store = SQLiteStore("memory.db")
|
|
41
|
+
alice = Scope("acme", subject="alice")
|
|
42
|
+
|
|
43
|
+
old = store.add(alice, "Alice prefers email", trust="owner")
|
|
44
|
+
new = store.supersede(alice, old.id, "Alice prefers phone calls", trust="owner")
|
|
45
|
+
|
|
46
|
+
for hit in store.search(alice, "how should we contact alice", k=3, min_trust="model"):
|
|
47
|
+
print(hit.entry.trust, hit.entry.text)
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
The store labels trust but does not decide what to do with retrieved text. A consumer that puts memory into a prompt must treat anything below `owner` as untrusted input.
|
|
51
|
+
|
|
52
|
+
## Verifying it yourself
|
|
53
|
+
|
|
54
|
+
```
|
|
55
|
+
pip install -e ../ragleap-security -e ".[test]"
|
|
56
|
+
pytest
|
|
57
|
+
python scripts/mutation_check.py # breaks one protection at a time; every mutation must be killed
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
## Not in 0.1.0
|
|
61
|
+
|
|
62
|
+
Embeddings and vector search, consolidation and retention policies, importance-based ranking, Postgres, encryption at rest, a review queue and the ragleap-agents adapter are planned (issue #900). Search quality has not been measured on any benchmark.
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
ragleap_memory/__init__.py,sha256=bE8v0yxGGcMow4uu_r2HVked1YT2diyReKOkelbvgjI,588
|
|
2
|
+
ragleap_memory/model.py,sha256=GR8bBBa6zkxzrmHB9nSvJ93Xly14_2rYUa2eUjVA0Ys,1964
|
|
3
|
+
ragleap_memory/sqlite_store.py,sha256=Kav5zJ9zPSUrEfQ_wwBjr46bzhMgQJDm0_FTuuThpPQ,16534
|
|
4
|
+
ragleap_memory-0.1.0.dist-info/METADATA,sha256=8mSd_Ivweo3xlnfV9tQs4I-Z1qjEHCwa7GsrNVfyusk,3295
|
|
5
|
+
ragleap_memory-0.1.0.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
|
|
6
|
+
ragleap_memory-0.1.0.dist-info/RECORD,,
|