memorytalk 0.4.2__tar.gz → 0.5.0__tar.gz

{memorytalk-0.4.2 → memorytalk-0.5.0}/PKG-INFO

@@ -1,7 +1,7 @@
 Metadata-Version: 2.4
 Name: memorytalk
-Version: 0.4.2
-Summary: Persistent cross-session memory for AI agents via Talk-Card architecture (v2)
+Version: 0.5.0
+Summary: Persistent cross-session memory for AI agents — Talk-Card architecture with forum-dynamics sinking/floating (v3)
 License-Expression: Apache-2.0
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
@@ -20,6 +20,7 @@ Requires-Dist: aiosqlite>=0.19.0
 Requires-Dist: aiofiles>=23.0.0
 Requires-Dist: rich>=13.0.0
 Requires-Dist: questionary>=2.0.0
+Requires-Dist: watchdog>=4.0.0
 Provides-Extra: dev
 Requires-Dist: pytest>=7.4.0; extra == "dev"
 Requires-Dist: pytest-cov>=4.1.0; extra == "dev"

memorytalk-0.5.0/memorytalk/__init__.py

@@ -0,0 +1,2 @@
+"""memory-talk v3 — Talk-Card memory system with forum dynamics."""
+__version__ = "0.5.0"

memorytalk-0.5.0/memorytalk/adapters/__init__.py

@@ -0,0 +1,6 @@
+"""Source-platform adapters — produce ingest payloads from local files."""
+from memorytalk.adapters.base import ADAPTERS, BaseAdapter, get_adapter, register
+# Import for side-effect: registers the claude-code adapter into ADAPTERS.
+from memorytalk.adapters import claude_code  # noqa: F401
+
+__all__ = ["ADAPTERS", "BaseAdapter", "get_adapter", "register"]

memorytalk-0.5.0/memorytalk/adapters/base.py

@@ -0,0 +1,82 @@
+"""Base adapter — declares "where conversations come from" without saying
+"how sync drives them".
+
+An adapter is the in-process port for one upstream platform (Claude Code
+on disk, a Codex API, …). It exposes three things to ``SyncWatcher``:
+
+  - ``watch_roots()``  — directories the watchdog should observe
+                         (file-source-only; remote adapters return [])
+  - ``list_sources()`` — yield every ``SourceProbe`` known right now,
+                         used by the cold-scan / backfill loop
+  - ``probe(source_id)``        — cheap inspection of one artifact
+  - ``read_after(...)`` — pull rounds strictly after a cursor
+
+Sync owns the cursor state (sha + last_round_id + line_offset) in its
+own ``sync.db``. Adapters are stateless ports.
+"""
+from __future__ import annotations
+from abc import ABC, abstractmethod
+from pathlib import Path
+from typing import Iterator
+
+from memorytalk.schemas import ReadAfterResult, SourceProbe
+
+
+class BaseAdapter(ABC):
+    source_name: str
+
+    @abstractmethod
+    def watch_roots(self) -> list[Path]:
+        """Filesystem directories the sync watchdog should observe.
+        Return ``[]`` for remote / non-filesystem adapters."""
+
+    @abstractmethod
+    def list_sources(self) -> Iterator[SourceProbe]:
+        """Enumerate every upstream artifact this adapter currently
+        knows about. Yields one ``SourceProbe`` per session.
+
+        Sync uses this on backfill to walk the entire upstream surface;
+        watchdog events bypass it and call ``probe`` on a single id.
+        """
+
+    @abstractmethod
+    def probe(self, source_id: str) -> SourceProbe | None:
+        """Inspect a single source artifact by its adapter-side id.
+
+        Returns ``None`` if the artifact no longer exists or isn't a
+        recognized session. The watcher calls this after debouncing a
+        file event."""
+
+    @abstractmethod
+    def read_after(
+        self,
+        source_id: str,
+        after_round_id: str | None,
+        hint_line_offset: int = 0,
+    ) -> ReadAfterResult:
+        """Read rounds strictly after ``after_round_id``.
+
+        ``hint_line_offset`` is the sync-side cached cursor offset.
+        Adapters that can validate it (e.g. by parsing the next record
+        at that offset and confirming its round_id == after_round_id)
+        SHOULD use it as a fast-seek hint; if validation fails they
+        MUST fall back to scanning from the start.
+
+        ``after_round_id=None`` means "read from the very beginning"
+        — used on first ingest of a previously-unseen session.
+        """
+
+
+ADAPTERS: dict[str, type[BaseAdapter]] = {}
+
+
+def register(cls: type[BaseAdapter]) -> type[BaseAdapter]:
+    ADAPTERS[cls.source_name] = cls
+    return cls
+
+
+def get_adapter(name: str) -> BaseAdapter:
+    cls = ADAPTERS.get(name)
+    if not cls:
+        raise ValueError(f"unknown adapter: {name}")
+    return cls()

memorytalk-0.5.0/memorytalk/adapters/claude_code.py

@@ -0,0 +1,236 @@
+"""Claude Code adapter.
+
+Reads ``~/.claude/projects/*/*.jsonl`` conversation files and produces
+ingest payloads. Each ``.jsonl`` file is one session; each line is one
+platform message; ``round_id = uuid`` from the message.
+
+Round-level layout:
+
+- ``content`` block preservation matches the v3 ContentBlock schema —
+  tool_use / tool_result remain typed so future search / display can
+  render them with their original semantics (text projection is built
+  separately for FTS).
+- ``metadata.cwd`` is extracted from the first message that carries
+  a ``cwd`` field so the explore namespace check works without the
+  server having to dig into each round.
+
+Cursor semantics:
+
+- ``source_id`` = absolute file path string.
+- ``sha256``    = sha256 of the file's raw bytes (used by sync to
+  short-circuit "did anything change").
+- ``after_round_id`` is the platform uuid of the last round the server
+  already stored. ``hint_line_offset`` is the line number sync cached
+  the last time it read this file — we use it as a fast-seek hint
+  but always validate before trusting it.
+"""
+from __future__ import annotations
+import hashlib
+import json
+from pathlib import Path
+from typing import Iterator
+from urllib.parse import unquote
+
+from memorytalk.adapters.base import BaseAdapter, register
+from memorytalk.schemas import (
+    ContentBlock, ReadAfterResult, RoundInput, SourceProbe,
+)
+
+
+@register
+class ClaudeCodeAdapter(BaseAdapter):
+    source_name = "claude-code"
+
+    DEFAULT_ROOT = Path.home() / ".claude" / "projects"
+
+    # ────────── sync-facing surface ──────────
+
+    def watch_roots(self) -> list[Path]:
+        return [self.DEFAULT_ROOT]
+
+    def list_sources(self) -> Iterator[SourceProbe]:
+        if not self.DEFAULT_ROOT.exists():
+            return
+        for path in sorted(self.DEFAULT_ROOT.rglob("*.jsonl")):
+            probe = self.probe(str(path))
+            if probe is not None:
+                yield probe
+
+    def probe(self, source_id: str) -> SourceProbe | None:
+        path = Path(source_id)
+        if path.suffix != ".jsonl":
+            return None
+        try:
+            raw_bytes = path.read_bytes()
+        except FileNotFoundError:
+            return None
+        if not raw_bytes:
+            return None
+        sha256 = hashlib.sha256(raw_bytes).hexdigest()
+        session_id = path.stem  # platform uuid, no prefix
+        project_name = unquote(path.parent.name)
+
+        # Pull created_at + cwd from the first parseable message.
+        created_at: str | None = None
+        cwd: str | None = None
+        for line in raw_bytes.decode("utf-8", errors="replace").splitlines():
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                msg = json.loads(line)
+            except json.JSONDecodeError:
+                continue
+            if created_at is None and msg.get("timestamp"):
+                created_at = msg["timestamp"]
+            if cwd is None and msg.get("cwd"):
+                cwd = msg["cwd"]
+            if created_at is not None and cwd is not None:
+                break
+
+        metadata: dict = {"project": project_name, "path": str(path)}
+        if cwd:
+            metadata["cwd"] = cwd
+
+        return SourceProbe(
+            source_id=str(path),
+            session_id=session_id,
+            sha256=sha256,
+            created_at=created_at or "",
+            metadata=metadata,
+        )
+
+    def read_after(
+        self,
+        source_id: str,
+        after_round_id: str | None,
+        hint_line_offset: int = 0,
+    ) -> ReadAfterResult:
+        path = Path(source_id)
+        if path.suffix != ".jsonl":
+            return ReadAfterResult(rounds=[], next_line_offset=0)
+        try:
+            raw = path.read_text(encoding="utf-8", errors="replace")
+        except FileNotFoundError:
+            return ReadAfterResult(rounds=[], next_line_offset=0)
+        lines = raw.splitlines()
+
+        start_line = self._locate_start(lines, after_round_id, hint_line_offset)
+
+        rounds: list[RoundInput] = []
+        cursor = start_line
+        for cursor in range(start_line, len(lines)):
+            line = lines[cursor].strip()
+            if not line:
+                continue
+            try:
+                msg = json.loads(line)
+            except json.JSONDecodeError:
+                continue
+            parsed = self._parse_message(msg)
+            if parsed is None:
+                continue
+            rounds.append(parsed)
+
+        return ReadAfterResult(rounds=rounds, next_line_offset=len(lines))
+
+    # ────────── seek / parse helpers ──────────
+
+    def _locate_start(
+        self,
+        lines: list[str],
+        after_round_id: str | None,
+        hint_line_offset: int,
+    ) -> int:
+        """Return the index of the first line whose round should be
+        yielded (i.e., the line strictly after the one carrying
+        ``after_round_id``).
+
+        Strategy:
+          - If ``after_round_id`` is None → start at 0 (full read).
+          - Otherwise: trust ``hint_line_offset`` only if line[hint-1]
+            actually carries ``after_round_id``. If it doesn't (file got
+            rewritten, hint stale, ...), scan from the top to find the
+            marker. If marker not found anywhere → start at 0.
+        """
+        if after_round_id is None:
+            return 0
+
+        if 0 < hint_line_offset <= len(lines):
+            if self._line_round_id(lines[hint_line_offset - 1]) == after_round_id:
+                return hint_line_offset
+
+        for i, line in enumerate(lines):
+            if self._line_round_id(line) == after_round_id:
+                return i + 1
+        # Marker not in the file at all — caller has a stale cursor.
+        # Treat as fresh ingest so the conflict-retry path kicks in.
+        return 0
+
+    @staticmethod
+    def _line_round_id(line: str) -> str | None:
+        line = line.strip()
+        if not line:
+            return None
+        try:
+            msg = json.loads(line)
+        except json.JSONDecodeError:
+            return None
+        return msg.get("uuid") or None
+
+    # ────────── per-message parsing ──────────
+
+    def _parse_message(self, msg: dict) -> RoundInput | None:
+        msg_type = msg.get("type")
+        if msg_type not in ("user", "assistant"):
+            return None
+        role = "human" if msg_type == "user" else "assistant"
+        speaker = "user" if msg_type == "user" else "assistant"
+        raw_content = msg.get("message", {}).get("content", [])
+        blocks = self._parse_content(raw_content)
+        if not blocks:
+            if isinstance(raw_content, str):
+                blocks = [{"type": "text", "text": raw_content}]
+            else:
+                return None
+        return RoundInput(
+            round_id=msg.get("uuid", ""),
+            parent_id=msg.get("parentUuid"),
+            timestamp=msg.get("timestamp"),
+            speaker=speaker,
+            role=role,
+            content=[ContentBlock(**b) for b in blocks],
+            is_sidechain=bool(msg.get("isSidechain")),
+            cwd=msg.get("cwd"),
+        )
+
+    def _parse_content(self, content_list) -> list[dict]:
+        if not isinstance(content_list, list):
+            return []
+        out: list[dict] = []
+        for block in content_list:
+            if not isinstance(block, dict):
+                continue
+            t = block.get("type")
+            if t == "text":
+                text = block.get("text") or ""
+                if text:
+                    out.append({"type": "text", "text": text})
+            elif t == "thinking":
+                thinking = block.get("thinking") or ""
+                if thinking:
+                    out.append({"type": "thinking", "thinking": thinking})
+            elif t == "tool_use":
+                name = block.get("name", "tool")
+                inp = block.get("input", "")
+                if isinstance(inp, dict):
+                    inp = json.dumps(inp, ensure_ascii=False)
+                # Keep typed; carry text for FTS.
+                out.append({"type": "tool_use", "name": name, "input": inp,
+                            "text": f"[{name}] {inp}"})
+            elif t == "tool_result":
+                c = block.get("content", "")
+                if isinstance(c, list):
+                    c = json.dumps(c, ensure_ascii=False)
+                out.append({"type": "tool_result", "text": str(c)})
+        return out

memorytalk-0.5.0/memorytalk/api/__init__.py

@@ -0,0 +1,140 @@
+"""FastAPI app factory for v3.
+
+Async startup via lifespan: open SQLite + LanceDB, probe the embedding
+provider, wire services into ``app.state``.
+
+Optional services that depend on optional providers (vector store) are
+constructed defensively — if a dep is unavailable the app still starts
+and the dependent endpoint returns 503 ``unavailable`` (rather than the
+whole server failing to come up).
+"""
+from __future__ import annotations
+import logging
+import os
+from contextlib import asynccontextmanager
+from pathlib import Path
+
+from fastapi import FastAPI
+
+from memorytalk.config import Config, ConfigValidationError
+
+
+_log = logging.getLogger("memorytalk.api")
+from memorytalk.provider.embedding import (
+    EmbedderValidationError, get_embedder, validate_embedder,
+)
+from memorytalk.provider.lancedb import LanceStore
+from memorytalk.provider.storage import LocalStorage
+from memorytalk.repository import SQLiteStore
+from memorytalk.repository.sync_checkpoint import SyncCheckpointStore
+from memorytalk.service import (
+    CardService, EventWriter, IngestService, ReadService,
+    RecallService, ReviewService,
+)
+from memorytalk.service.search import SearchService
+from memorytalk.service.sync import SyncWatcher
+
+
+def create_app(config: Config | None = None) -> FastAPI:
+    config = config or Config()
+    config.ensure_dirs()
+
+    @asynccontextmanager
+    async def lifespan(app: FastAPI):
+        storage = LocalStorage(config.data_root)
+        db = await SQLiteStore.create(config.db_path, storage)
+        sync_checkpoints = await SyncCheckpointStore.create(config.sync_db_path)
+
+        # LanceDB is optional at boot. If it can't open (missing pyarrow,
+        # bad dir perms, ...) we still want read/status to work.
+        vectors: LanceStore | None = None
+        try:
+            vectors = await LanceStore.create(
+                config.vectors_dir, dim=config.settings.embedding.dim,
+            )
+        except Exception:
+            _log.exception("lancedb init failed; vector-backed endpoints will 503")
+
+        embedder = get_embedder(config)
+
+        try:
+            await validate_embedder(config)
+        except EmbedderValidationError as e:
+            _log.exception("embedding startup check failed; aborting boot")
+            raise SystemExit(2) from e
+
+        events = EventWriter(db)
+        app.state.config = config
+        app.state.storage = storage
+        app.state.db = db
+        app.state.vectors = vectors
+        app.state.embedder = embedder
+        app.state.events = events
+        app.state.read = ReadService(db=db, events=events)
+        app.state.ingest = IngestService(
+            db=db, vectors=vectors, embedder=embedder, events=events,
+        )
+        app.state.sync_checkpoints = sync_checkpoints
+        app.state.sync = SyncWatcher(
+            config=config, ingest=app.state.ingest,
+            checkpoints=sync_checkpoints,
+        )
+        app.state.search = SearchService(
+            config=config, db=db, vectors=vectors, embedder=embedder,
+        )
+        app.state.cards = CardService(
+            db=db, vectors=vectors, embedder=embedder, events=events,
+        )
+        app.state.reviews = ReviewService(db=db, events=events)
+        app.state.recall = RecallService(
+            config=config, db=db, vectors=vectors, embedder=embedder,
+        )
+
+        # Spin up the watcher if settings says so. start() returns fast
+        # now — backfill runs as a background task; uvicorn's "startup
+        # complete" log is no longer gated on the initial ingest.
+        if config.settings.sync.enabled:
+            try:
+                await app.state.sync.start()
+            except Exception:
+                _log.exception("sync auto-start failed")
+
+        yield
+
+        # Pause (not stop) on shutdown — preserves the user's persisted
+        # enable choice so the next server start auto-resumes the watcher.
+        try:
+            await app.state.sync.pause()
+        except Exception:
+            pass
+        await db.close()
+        await sync_checkpoints.close()
+
+    app = FastAPI(title="memory.talk v3", lifespan=lifespan)
+    app.state.config = config
+
+    # Mount the v3 routers. Each module exports a ``router``; missing
+    # modules are silently skipped so partial-build environments still
+    # come up (useful during incremental implementation).
+    from memorytalk.api.status import router as status_router
+    app.include_router(status_router, prefix="/v3")
+    from memorytalk.api.read import router as read_router
+    app.include_router(read_router, prefix="/v3")
+    from memorytalk.api.sessions import router as sessions_router
+    app.include_router(sessions_router, prefix="/v3")
+
+    # Optional routers — lazy import so missing ones don't break boot.
+    for name in ("sync", "search", "cards", "reviews", "recall"):
+        try:
+            mod = __import__(f"memorytalk.api.{name}", fromlist=["router"])
+            app.include_router(mod.router, prefix="/v3")
+        except ImportError:
+            pass
+
+    return app
+
+
+# uvicorn entry point. The data_root env var is the same test hook
+# `Config.__init__` honors; user-facing CLI hardcodes ~/.memory-talk.
+_data_root = os.environ.get("MEMORY_TALK_DATA_ROOT")
+app = create_app(Config(_data_root) if _data_root else Config())

{memorytalk-0.4.2 → memorytalk-0.5.0}/memorytalk/api/cards.py

@@ -1,20 +1,24 @@
-"""POST /v2/cards."""
+"""POST /v3/cards."""
 from __future__ import annotations
 
 from fastapi import APIRouter, HTTPException, Request
 
 from memorytalk.schemas import CreateCardRequest, CreateCardResponse
-from memorytalk.service import CardConflictError, CardServiceError
+from memorytalk.service import CardConflict, CardServiceError
 
 
 router = APIRouter()
 
 
 @router.post("/cards", response_model=CreateCardResponse)
-async def post_cards(payload: CreateCardRequest, request: Request):
+async def post_cards(payload: CreateCardRequest, request: Request) -> CreateCardResponse:
+    svc = request.app.state.cards
+    if svc is None:
+        raise HTTPException(status_code=503, detail="cards service unavailable")
     try:
-        return await request.app.state.cards.create(payload)
-    except CardConflictError as e:
+        card_id = await svc.create(payload)
+    except CardConflict as e:
         raise HTTPException(status_code=409, detail=str(e))
     except CardServiceError as e:
         raise HTTPException(status_code=400, detail=str(e))
+    return CreateCardResponse(card_id=card_id)

memorytalk-0.5.0/memorytalk/api/read.py

@@ -0,0 +1,36 @@
+"""POST /v3/read — prefix-dispatched read of card or session."""
+from __future__ import annotations
+
+from fastapi import APIRouter, HTTPException, Request
+
+from memorytalk.schemas import ReadRequest
+from memorytalk.service import CardNotFound, SessionNotFound
+from memorytalk.util.ids import IdKind, InvalidIdError, parse_id
+
+
+router = APIRouter()
+
+
+@router.post("/read")
+async def post_read(payload: ReadRequest, request: Request):
+    try:
+        kind, _ = parse_id(payload.id)
+    except InvalidIdError:
+        raise HTTPException(status_code=400, detail="invalid id prefix")
+
+    svc = request.app.state.read
+
+    try:
+        if kind == IdKind.CARD:
+            card, read_at = await svc.read_card(payload.id)
+            return {"type": "card", "read_at": read_at, "card": card.model_dump()}
+        if kind == IdKind.SESSION:
+            session, read_at = await svc.read_session(payload.id)
+            return {"type": "session", "read_at": read_at,
+                    "session": session.model_dump()}
+    except CardNotFound as e:
+        raise HTTPException(status_code=404, detail=str(e))
+    except SessionNotFound as e:
+        raise HTTPException(status_code=404, detail=str(e))
+
+    raise HTTPException(status_code=400, detail="invalid id prefix")

memorytalk-0.5.0/memorytalk/api/recall.py

@@ -0,0 +1,26 @@
+"""POST /v3/recall."""
+from __future__ import annotations
+
+from fastapi import APIRouter, HTTPException, Request
+
+from memorytalk.schemas import RecallRequest, RecallResponse
+from memorytalk.service import RecallServiceError
+
+
+router = APIRouter()
+
+
+@router.post("/recall", response_model=RecallResponse)
+async def post_recall(payload: RecallRequest, request: Request) -> RecallResponse:
+    svc = request.app.state.recall
+    if svc is None:
+        raise HTTPException(status_code=503, detail="recall service unavailable")
+    try:
+        result = await svc.recall(
+            session_id=payload.session_id,
+            prompt=payload.prompt,
+            top_k=payload.top_k,
+        )
+    except RecallServiceError as e:
+        raise HTTPException(status_code=400, detail=str(e))
+    return RecallResponse(**result)

memorytalk-0.5.0/memorytalk/api/reviews.py

@@ -0,0 +1,26 @@
+"""POST /v3/reviews."""
+from __future__ import annotations
+
+from fastapi import APIRouter, HTTPException, Request
+
+from memorytalk.schemas import CreateReviewRequest, CreateReviewResponse
+from memorytalk.service import ReviewConflict, ReviewServiceError
+
+
+router = APIRouter()
+
+
+@router.post("/reviews", response_model=CreateReviewResponse)
+async def post_reviews(
+    payload: CreateReviewRequest, request: Request,
+) -> CreateReviewResponse:
+    svc = request.app.state.reviews
+    if svc is None:
+        raise HTTPException(status_code=503, detail="reviews service unavailable")
+    try:
+        result = await svc.create(payload)
+    except ReviewConflict as e:
+        raise HTTPException(status_code=409, detail=str(e))
+    except ReviewServiceError as e:
+        raise HTTPException(status_code=400, detail=str(e))
+    return CreateReviewResponse(**result)

memorytalk-0.5.0/memorytalk/api/search.py

@@ -0,0 +1,28 @@
+"""POST /v3/search."""
+from __future__ import annotations
+
+from fastapi import APIRouter, HTTPException, Request
+
+from memorytalk.schemas import SearchRequest, SearchResponse
+from memorytalk.util.dsl import DSLError
+from memorytalk.util.formula import FormulaError
+
+
+router = APIRouter()
+
+
+@router.post("/search", response_model=SearchResponse)
+async def post_search(payload: SearchRequest, request: Request) -> SearchResponse:
+    svc = request.app.state.search
+    if svc is None:
+        raise HTTPException(status_code=503, detail="search service unavailable")
+    try:
+        return await svc.search(
+            query=payload.query or "",
+            where=payload.where,
+            top_k=payload.top_k,
+        )
+    except DSLError as e:
+        raise HTTPException(status_code=400, detail=str(e))
+    except FormulaError as e:
+        raise HTTPException(status_code=500, detail=str(e))