memdebug 0.2.0__py3-none-any.whl

memdebug/adapters/mem0.py

@@ -0,0 +1,324 @@
+"""Read-only adapter for self-hosted Mem0 (verified against mem0ai 2.2.1).
+
+Where the data comes from:
+  * live memories     -> Mem0's own get_all()  (public API)
+  * the global feed   -> Mem0's history database, opened strictly read-only, because the
+                         public API cannot list the history of memories that were deleted.
+  * one memory's log  -> Mem0's own history()  (public API)
+
+This adapter never writes to Mem0 or to its database. Everything it reads is untrusted.
+
+Mem0 facts this relies on (2.2.1):
+  * history rows: id, memory_id, old_memory, new_memory, event, created_at, updated_at,
+    is_deleted, actor_id, role. Events are 'ADD', 'UPDATE', 'DELETE'.
+  * for UPDATE and DELETE rows, created_at is when the *memory* was created and updated_at is
+    when the *event* happened; for ADD both are the creation time.
+  * rows have no scope column; scope has to come from the live listing.
+"""
+from __future__ import annotations
+
+import json
+import logging
+import os
+import sqlite3
+import sys
+from contextlib import closing
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+from typing import Any, Callable
+
+from pydantic import ValidationError
+
+from ..errors import AdapterError, MemdebugError, UnsupportedSchemaError
+from ..models import MAX_ID_CHARS, Memory, MemoryEvent, Op, Source
+from ..textsafe import bound_text, safe_text
+from .base import HistoryRead, LiveMemories
+from .common import Warnings as _Warnings
+from .common import clean_id as _clean_id
+from .common import decode as _decode
+from .common import parse_ts as _parse_ts
+from .common import require_regular_file as _require_regular_file
+
+logger = logging.getLogger(__name__)
+
+REQUIRED_COLUMNS = ("id", "memory_id", "old_memory", "new_memory", "event", "created_at", "updated_at")
+OPTIONAL_COLUMNS = ("actor_id", "role")  # fixed names; column names are never taken from data
+SCOPE_KEYS = ("user_id", "agent_id", "run_id")
+_OPS = {"ADD": Op.ADD, "UPDATE": Op.UPDATE, "DELETE": Op.DELETE}
+_MAX_CONFIG_BYTES = 1_000_000
+
+
+def _text_or_none(value: object) -> tuple[bool, str | None]:
+    if value is None:
+        return True, None
+    if isinstance(value, str):
+        return True, bound_text(value)
+    return False, None  # BLOB, number or anything else SQLite's loose typing allowed in
+
+
+def validate_scope(scope: object) -> dict[str, str]:
+    if not isinstance(scope, dict) or not scope:
+        raise AdapterError("a scope is required: give user_id, agent_id or run_id")
+    clean: dict[str, str] = {}
+    for key, value in scope.items():
+        if key not in SCOPE_KEYS:
+            raise AdapterError(f"unsupported scope key: {safe_text(key, 30)}")
+        if _clean_id(value) is None:
+            raise AdapterError(f"scope value for {key} must be a non-empty string of at most {MAX_ID_CHARS} characters")
+        clean[key] = value
+    return clean
+
+
+class Mem0Adapter:
+    name = "mem0"
+    capabilities = {"history", "global_feed"}
+
+    def __init__(
+        self,
+        memory: Any,
+        history_db_path: str | Path,
+        *,
+        max_rows: int = 100_000,
+        live_limit: int = 10_000,
+        max_total_chars: int = 100_000_000,
+        clock: Callable[[], datetime] | None = None,
+    ):
+        for attribute in ("get_all", "history"):
+            if not callable(getattr(memory, attribute, None)):
+                raise AdapterError(f"memory object has no callable {attribute}(); expected a mem0 Memory")
+        if not (isinstance(max_rows, int) and max_rows > 0 and isinstance(live_limit, int) and live_limit > 0):
+            raise AdapterError("max_rows and live_limit must be positive integers")
+        self._memory = memory
+        self._path = Path(history_db_path)
+        self._max_rows = max_rows
+        self._live_limit = live_limit
+        self._max_total_chars = max_total_chars
+        self._clock = clock or (lambda: datetime.now(timezone.utc))
+        with closing(self._connect_ro()) as conn:  # fail early on a wrong file
+            self._check_schema(conn)
+
+    # -- read-only database access ----------------------------------------------------
+
+    def _connect_ro(self) -> sqlite3.Connection:
+        path = _require_regular_file(self._path)
+        try:
+            # mode=ro: never creates the file, never writes. query_only is a second lock.
+            conn = sqlite3.connect(f"{path.as_uri()}?mode=ro", uri=True, timeout=5)
+            conn.text_factory = _decode
+            conn.execute("PRAGMA query_only = ON")
+            return conn
+        except sqlite3.Error as exc:
+            raise AdapterError(f"cannot open history file read-only: {exc}") from exc
+
+    @staticmethod
+    def _check_schema(conn: sqlite3.Connection) -> set[str]:
+        try:
+            present = {row[1] for row in conn.execute("PRAGMA table_info(history)")}
+        except sqlite3.Error as exc:
+            raise AdapterError(f"cannot read history file: {exc}") from exc
+        if not present:
+            raise UnsupportedSchemaError("no 'history' table found; is this Mem0's history database?")
+        missing = [c for c in REQUIRED_COLUMNS if c not in present]
+        if missing:
+            raise UnsupportedSchemaError(f"history table lacks expected columns: {', '.join(missing)}")
+        return present
+
+    # -- row -> event -------------------------------------------------------------------
+
+    def _event_from_row(
+        self, row: dict[str, Any], label: str, ref_fallback: str, now: datetime, warnings: _Warnings
+    ) -> tuple[MemoryEvent | None, str]:
+        ref = _clean_id(row.get("id")) or ref_fallback
+        memory_id = _clean_id(row.get("memory_id"))
+        if memory_id is None:
+            warnings.add(f"{label}: missing or invalid memory_id; row skipped")
+            return None, ref
+        raw_event = row.get("event")
+        op = _OPS.get(raw_event.strip().upper()) if isinstance(raw_event, str) else None
+        if op is None:
+            warnings.add(f"{label}: unknown event {safe_text(raw_event, 30)!r}; row skipped")
+            return None, ref
+        ok_old, old = _text_or_none(row.get("old_memory"))
+        ok_new, new = _text_or_none(row.get("new_memory"))
+        if not (ok_old and ok_new):
+            warnings.add(f"{label}: memory text is not text; row skipped")
+            return None, ref
+        if op is Op.UPDATE and new is None:
+            warnings.add(f"{label}: UPDATE without new text")
+
+        created = _parse_ts(row.get("created_at"))
+        updated = _parse_ts(row.get("updated_at"))
+        ts = (created or updated) if op is Op.ADD else (updated or created)
+        observed = False
+        if ts is None:
+            ts, observed = now, True
+            warnings.add(f"{label}: no usable timestamp; using the time it was read")
+        elif ts > now + timedelta(days=1):
+            warnings.add(f"{label}: timestamp is in the future")
+
+        actor = _clean_id(row.get("actor_id"))
+        role = _clean_id(row.get("role"))
+        source = Source(actor_id=actor, role=role) if (actor or role) else None
+        try:
+            event = MemoryEvent(
+                backend=self.name, memory_id=memory_id, op=op, ts=ts, ts_observed=observed,
+                backend_ref=ref, before=old, after=new, source=source,
+            )
+        except ValidationError:
+            warnings.add(f"{label}: row failed validation; skipped")
+            return None, ref
+        return event, ref
+
+    # -- the adapter interface -------------------------------------------------------------
+
+    def read_history(self, max_rows: int) -> HistoryRead:
+        if not (isinstance(max_rows, int) and max_rows > 0):
+            raise AdapterError("max_rows must be a positive integer")
+        warnings = _Warnings()
+        now = self._clock()
+        pairs: list[tuple[MemoryEvent, int]] = []
+        refs: set[str] = set()
+        skipped = 0
+        truncated = False
+        total_chars = 0
+        with closing(self._connect_ro()) as conn:
+            present = self._check_schema(conn)
+            names = list(REQUIRED_COLUMNS) + [c for c in OPTIONAL_COLUMNS if c in present]
+            # Identifiers come from the fixed tuples above; values are always bound parameters.
+            query = f"SELECT rowid, {', '.join(names)} FROM history ORDER BY rowid LIMIT ?"
+            try:
+                for index, db_row in enumerate(conn.execute(query, (max_rows + 1,))):
+                    if index >= max_rows:
+                        truncated = True
+                        break
+                    rowid = db_row[0]
+                    row = dict(zip(names, db_row[1:], strict=False))
+                    event, ref = self._event_from_row(row, f"history rowid {rowid}", f"rowid:{rowid}", now, warnings)
+                    refs.add(ref)
+                    if event is None:
+                        skipped += 1
+                        continue
+                    total_chars += len(event.before or "") + len(event.after or "")
+                    if total_chars > self._max_total_chars:
+                        truncated = True
+                        warnings.add("history text is larger than the memory budget; the rest was not read")
+                        break
+                    pairs.append((event, rowid))
+            except sqlite3.Error as exc:
+                raise AdapterError(f"reading history failed: {exc}") from exc
+        pairs.sort(key=lambda pair: (pair[0].ts, pair[1]))
+        if truncated:
+            warnings.add(f"history has more than {max_rows} rows; only the oldest {max_rows} were read")
+        return HistoryRead(
+            events=[event for event, _ in pairs], refs=refs, truncated=truncated,
+            skipped=skipped, warnings=warnings.as_list(),
+        )
+
+    def list_memories(self, scope: dict[str, str]) -> LiveMemories:
+        clean = validate_scope(scope)
+        try:
+            result = self._memory.get_all(filters=dict(clean), top_k=self._live_limit, show_expired=True)
+        except Exception as exc:  # any failure inside the backend becomes one controlled error
+            raise AdapterError(f"mem0 get_all failed: {type(exc).__name__}: {safe_text(exc, 200)}") from exc
+        if not isinstance(result, dict) or not isinstance(result.get("results"), list):
+            raise AdapterError("mem0 get_all returned an unexpected shape; is this mem0ai 2.x?")
+        items = result["results"]
+        warnings = _Warnings()
+        complete = len(items) < self._live_limit
+        if not complete:
+            warnings.add(
+                f"listing reached the limit of {self._live_limit}; absence will not be reported as deletion"
+            )
+        memories: list[Memory] = []
+        seen: set[str] = set()
+        for index, item in enumerate(items):
+            memory_id = _clean_id(item.get("id")) if isinstance(item, dict) else None
+            text = item.get("memory") if isinstance(item, dict) else None
+            if memory_id is None or not isinstance(text, str):
+                warnings.add(f"live item {index}: missing id or text; skipped")
+                complete = False  # a hidden memory could be mistaken for a deleted one
+                continue
+            if memory_id in seen:
+                warnings.add(f"live item {index}: duplicate id; skipped")
+                continue
+            seen.add(memory_id)
+            item_scope = {k: item[k] for k in SCOPE_KEYS if _clean_id(item.get(k))}
+            try:
+                memories.append(Memory(id=memory_id, text=bound_text(text), scope={**clean, **item_scope}))
+            except ValidationError:
+                warnings.add(f"live item {index}: failed validation; skipped")
+                complete = False
+        return LiveMemories(memories=memories, complete=complete, warnings=warnings.as_list())
+
+    def history(self, memory_id: str) -> list[MemoryEvent]:
+        if _clean_id(memory_id) is None:
+            raise AdapterError(f"memory_id must be a non-empty string of at most {MAX_ID_CHARS} characters")
+        try:
+            rows = self._memory.history(memory_id)
+        except Exception as exc:
+            raise AdapterError(f"mem0 history failed: {type(exc).__name__}: {safe_text(exc, 200)}") from exc
+        if not isinstance(rows, list):
+            raise AdapterError("mem0 history returned an unexpected shape")
+        warnings = _Warnings()
+        now = self._clock()
+        pairs: list[tuple[MemoryEvent, int]] = []
+        for index, row in enumerate(rows[: self._max_rows]):
+            if not isinstance(row, dict):
+                warnings.add(f"history item {index}: not a record; skipped")
+                continue
+            event, _ = self._event_from_row(row, f"history item {index}", f"api:{index}", now, warnings)
+            if event is not None:
+                pairs.append((event, index))
+        for message in warnings.as_list():
+            logger.warning("%s", message)
+        pairs.sort(key=lambda pair: (pair[0].ts, pair[1]))
+        return [event for event, _ in pairs]
+
+
+# -- starting mem0 -------------------------------------------------------------------------
+
+
+def ensure_mem0_telemetry_off() -> list[str]:
+    """Mem0 sends anonymous usage events by default. Turn that off before it is imported."""
+    os.environ.setdefault("MEM0_TELEMETRY", "false")
+    notes: list[str] = []
+    if os.environ["MEM0_TELEMETRY"].strip().lower() in ("true", "1", "yes"):
+        notes.append("MEM0_TELEMETRY is on in your environment; mem0 will send anonymous usage reports")
+    else:
+        module = sys.modules.get("mem0.memory.telemetry")
+        if module is not None and getattr(module, "MEM0_TELEMETRY", False):
+            notes.append(
+                "mem0 was already imported with telemetry on; start a fresh process with "
+                "MEM0_TELEMETRY=false to stop its anonymous usage reports"
+            )
+    return notes
+
+
+def _load_json_config(path: Path) -> dict[str, Any]:
+    resolved = _require_regular_file(path)
+    if resolved.stat().st_size > _MAX_CONFIG_BYTES:
+        raise AdapterError("mem0 config file is too large")
+    try:
+        config = json.loads(resolved.read_text(encoding="utf-8"))
+    except (OSError, ValueError) as exc:
+        raise AdapterError("mem0 config file is not valid JSON") from exc
+    if not isinstance(config, dict):
+        raise AdapterError("mem0 config must be a JSON object")
+    return config
+
+
+def build_mem0_memory(config_path: str | Path | None = None) -> tuple[Any, list[str]]:
+    """Create a mem0 Memory. Errors never echo the config, which may hold API keys."""
+    notes = ensure_mem0_telemetry_off()
+    try:
+        from mem0 import Memory
+    except ImportError as exc:
+        raise AdapterError("mem0ai is not installed; run: pip install 'mem0ai>=2'") from exc
+    try:
+        if config_path is None:
+            return Memory(), notes
+        return Memory.from_config(_load_json_config(Path(config_path))), notes
+    except MemdebugError:
+        raise
+    except Exception as exc:
+        raise AdapterError(f"could not start mem0: {type(exc).__name__}") from exc

memdebug/adapters/openwebui.py

@@ -0,0 +1,209 @@
+"""Open WebUI's memory feature, read from a COPY of its database (webui.db).
+
+Open WebUI keeps what it remembers about a user in a `memory` table and records no change history, so memdebug records
+what it sees between two looks. This adapter only reads: the file is opened read-only (SQLite's read-only mode plus
+`query_only`), only the `memory` and `user` tables are touched, and every row is validated. It never reads chats,
+passwords, API keys or settings. Work on a copy of the database, not the live file inside the container.
+"""
+from __future__ import annotations
+
+import bisect
+import json
+import re
+import sqlite3
+from contextlib import closing
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Callable
+
+from ..errors import AdapterError
+from ..models import Memory, MemoryEvent, Source, SourceKind
+from ..textsafe import bound_text, safe_text
+from .base import HistoryRead, LiveMemories
+from .common import Warnings, clean_id, decode, require_regular_file
+
+MAX_ID_CHARS = 256
+_REQUIRED = {"id", "user_id", "content"}
+_PLAIN = re.compile(r"^[A-Za-z0-9_.:/ -]{1,40}\Z")
+WINDOW_SECONDS = 120          # a chat message this close to a memory's last change counts as "around the same time"
+MAX_CHAT_ROWS = 200_000       # chat records compared with; more than this and the comparison is skipped
+_OPTIONAL = ("type", "meta", "created_at", "updated_at")
+
+
+def _plain(value: object) -> str | None:
+    """A short label made of plain characters, or just its size if it is anything else. Never free text."""
+    if value is None or value == "":
+        return None
+    text = str(value)
+    return text if _PLAIN.match(text) else f"(text, {len(text)} chars)"
+
+
+def _epoch(value: object) -> float | None:
+    """Seconds since 1970, from seconds or milliseconds; anything else is not trusted as a time."""
+    if isinstance(value, bool) or not isinstance(value, (int, float)):
+        return None
+    for low, high, divisor in ((1e9, 1e11, 1), (1e12, 1e14, 1000)):
+        if low <= value < high:
+            return value / divisor
+    return None
+
+
+class OpenWebUIAdapter:
+    name = "openwebui"
+    capabilities: set[str] = set()  # no history to read
+
+    def __init__(self, db_path: str | Path, *, user_id: str | None = None, max_rows: int = 50_000,
+                 clock: Callable[[], datetime] | None = None):
+        self._path = Path(db_path)
+        self._max_rows = max_rows
+        self._clock = clock or (lambda: datetime.now(timezone.utc))
+        if not (isinstance(max_rows, int) and max_rows > 0):
+            raise AdapterError("max_rows must be positive")
+        wanted = None if user_id is None else clean_id(user_id)
+        if user_id is not None and wanted is None:
+            raise AdapterError("the user id must be a non-empty string of reasonable length")
+        with closing(self._connect()) as db:
+            columns = self._columns(db, "memory")
+            if not _REQUIRED <= columns:
+                raise AdapterError("this does not look like Open WebUI's database: the memory table lacks the expected columns")
+            self._user = wanted or self._only_user(db)
+
+    # -- reading ---------------------------------------------------------------------------------------------------
+
+    def _connect(self) -> sqlite3.Connection:
+        path = require_regular_file(self._path)
+        try:
+            conn = sqlite3.connect(f"{path.as_uri()}?mode=ro", uri=True, timeout=5)
+            conn.text_factory = decode
+            conn.execute("PRAGMA query_only = ON")
+            return conn
+        except sqlite3.Error as exc:
+            raise AdapterError(f"cannot open the Open WebUI database read-only: {safe_text(exc, 100)}") from exc
+
+    @staticmethod
+    def _columns(db: sqlite3.Connection, table: str) -> set[str]:
+        try:
+            return {str(row[1]) for row in db.execute(f'PRAGMA table_info("{table}")')}
+        except sqlite3.Error as exc:
+            raise AdapterError(f"cannot read the database structure: {safe_text(exc, 100)}") from exc
+
+    def _only_user(self, db: sqlite3.Connection) -> str:
+        """The user whose memories to read, when none was named: the single user of this installation."""
+        try:
+            owners = {clean_id(row[0]) for row in db.execute('SELECT DISTINCT "user_id" FROM "memory" LIMIT 3')}
+            owners.discard(None)
+            if len(owners) == 1:
+                return str(next(iter(owners)))
+            if len(owners) > 1:
+                raise AdapterError("memories of several users are stored here; say which one with --user-id")
+            if "user" in {r[0] for r in db.execute("SELECT name FROM sqlite_master WHERE type='table'")} and "id" in self._columns(db, "user"):
+                users = {clean_id(row[0]) for row in db.execute('SELECT "id" FROM "user" LIMIT 3')}
+                users.discard(None)
+                if len(users) == 1:
+                    return str(next(iter(users)))
+        except sqlite3.Error as exc:
+            raise AdapterError(f"cannot read the Open WebUI database: {safe_text(exc, 100)}") from exc
+        raise AdapterError("cannot tell whose memories to read; say which user with --user-id")
+
+    @property
+    def user(self) -> str:
+        return self._user
+
+    @property
+    def scope(self) -> dict[str, str]:
+        return {"user_id": self._user}
+
+    def list_memories(self, scope: dict[str, str]) -> LiveMemories:
+        if scope != self.scope:
+            raise AdapterError(f"scope must be {{'user_id': {safe_text(self._user, 40)!r}}} for this database")
+        warnings = Warnings()
+        memories: list[Memory] = []
+        complete = True
+        with closing(self._connect()) as db:
+            have = self._columns(db, "memory")
+            extra = [name for name in _OPTIONAL if name in have]
+            order = '"created_at", "id"' if "created_at" in have else '"id"'
+            try:
+                rows = db.execute(f'SELECT "id", "content"{"".join(f", {chr(34)}{n}{chr(34)}" for n in extra)} FROM "memory" '
+                                  f'WHERE "user_id" = ? ORDER BY {order} LIMIT ?', (self._user, self._max_rows + 1)).fetchall()
+            except sqlite3.Error as exc:
+                raise AdapterError(f"cannot read the memories: {safe_text(exc, 100)}") from exc
+            chat = self._chat_times(db)
+        if len(rows) > self._max_rows:
+            rows = rows[:self._max_rows]
+            complete = False
+            warnings.add(f"more than {self._max_rows} memories; the rest were not read")
+        for raw_id, content, *details in rows:
+            memory_id = clean_id(raw_id)
+            if memory_id is None or not isinstance(content, str):
+                complete = False
+                warnings.add("a memory row without a usable id or text was skipped")
+                continue
+            facts = dict(zip(extra, details, strict=True))
+            memories.append(Memory(id=memory_id, text=bound_text(content), scope=dict(scope), source=self._source(facts, chat)))
+        return LiveMemories(memories=memories, complete=complete, warnings=warnings.as_list())
+
+    @staticmethod
+    def _chat_times(db: sqlite3.Connection) -> list[tuple[float, str]] | None:
+        """When each chat message happened and its role, nothing else, or None if that cannot be compared. The message text is never read."""
+        try:
+            tables = {row[0] for row in db.execute("SELECT name FROM sqlite_master WHERE type='table'")}
+            if "chat_message" not in tables or not {"role", "created_at"} <= OpenWebUIAdapter._columns(db, "chat_message"):
+                return None
+            rows = db.execute('SELECT "created_at", "role" FROM "chat_message" WHERE "created_at" IS NOT NULL '
+                              'ORDER BY "created_at" LIMIT ?', (MAX_CHAT_ROWS + 1,)).fetchall()
+        except sqlite3.Error:
+            return None
+        if len(rows) > MAX_CHAT_ROWS:
+            return None
+        times = [(t, _plain(role) or "unknown") for raw, role in rows if (t := _epoch(raw)) is not None]
+        return sorted(times)
+
+    @staticmethod
+    def _source(facts: dict, chat: list[tuple[float, str]] | None) -> Source | None:
+        """What Open WebUI itself records about a memory, and how close a chat was to it. A description, never a verdict."""
+        meta = facts.get("meta")
+        data = None
+        if isinstance(meta, str) and len(meta) <= 20_000:
+            try:
+                data = json.loads(meta)
+            except ValueError:
+                data = None
+        label = _plain(data.get("created_by")) if isinstance(data, dict) else None
+        kind = _plain(facts.get("type"))
+        touched = _epoch(facts.get("updated_at")) or _epoch(facts.get("created_at"))
+        if label is None and kind is None and touched is None:
+            return None
+        parts: list[str] = []
+        near: list[tuple[float, str]] = []
+        if touched is not None:
+            parts.append("last touched " + datetime.fromtimestamp(touched, timezone.utc).strftime("%Y-%m-%d %H:%M UTC"))
+            if chat is None:
+                parts.append("chat records could not be compared")
+            elif not chat:
+                parts.append("there are no chat messages to compare with")
+            else:
+                low = bisect.bisect_left(chat, (touched - WINDOW_SECONDS, ""))
+                high = bisect.bisect_right(chat, (touched + WINDOW_SECONDS, "\uffff"))
+                near = chat[low:high]
+                position = bisect.bisect_left(chat, (touched, ""))
+                gap = min(abs(chat[i][0] - touched) for i in (position - 1, position) if 0 <= i < len(chat))
+                if near:
+                    parts.append(f"{len(near)} chat message{'' if len(near) == 1 else 's'} within {WINDOW_SECONDS} s (nearest {gap:.0f} s away)")
+                else:
+                    parts.append(f"no chat message within {WINDOW_SECONDS} s (nearest is {gap:.0f} s away)")
+        if label == "manual" and not near and touched is not None and chat:
+            parts.append("consistent with being added by hand in the settings")
+        elif label == "manual" and near:
+            parts.append("labelled manual, but a chat was active at the time")
+        elif label != "manual" and near:
+            parts.append(f"{'not labelled manual' if label is None else 'labelled ' + label}, and a chat was active at the time: "
+                         "check what the assistant had just read")
+        return Source(kind=SourceKind.UNKNOWN, actor_id=f"created_by={label}" if label else None,
+                      role=f"type={kind}" if kind else None, note="; ".join(parts)[:390] or None)
+
+    def read_history(self, max_rows: int) -> HistoryRead:
+        return HistoryRead(events=[], refs=set(), truncated=False)
+
+    def history(self, memory_id: str) -> list[MemoryEvent]:
+        return []