memorytalk 0.9.2__tar.gz → 1.0.0__tar.gz

{memorytalk-0.9.2 → memorytalk-1.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: memorytalk
-Version: 0.9.2
+Version: 1.0.0
 Summary: Persistent cross-session memory for AI agents — Talk-Card architecture with forum-dynamics sinking/floating (v3)
 License-Expression: Apache-2.0
 Requires-Python: >=3.10

{memorytalk-0.9.2 → memorytalk-1.0.0}/memorytalk/api/__init__.py

@@ -20,10 +20,10 @@ from memorytalk.config import Config, ConfigValidationError
 
 
 _log = logging.getLogger("memorytalk.api")
+from memorytalk.migration import MigrationRunner
 from memorytalk.provider.embedding import (
-    EmbedderValidationError, get_embedder, validate_embedder,
+    EmbedderValidationError, validate_embedder,
 )
-from memorytalk.provider.lancedb import LanceStore
 from memorytalk.provider.storage import LocalStorage
 from memorytalk.repository import SQLiteStore
 from memorytalk.repository.sync_checkpoint import SyncCheckpointStore
@@ -32,8 +32,8 @@ from memorytalk.service import (
     RecallService, ReviewService,
 )
 from memorytalk.service.backfill import IndexBackfill
-from memorytalk.service.index_buffer import IndexWriteBuffer
 from memorytalk.service.search import SearchService
+from memorytalk.service.searchbase_schema import build_search_backend
 from memorytalk.service.sync import SyncWatcher
 
 
@@ -44,50 +44,76 @@ def create_app(config: Config | None = None) -> FastAPI:
     @asynccontextmanager
     async def lifespan(app: FastAPI):
         storage = LocalStorage(config.data_root)
-        db = await SQLiteStore.create(config.db_path, storage)
-        sync_checkpoints = await SyncCheckpointStore.create(config.sync_db_path)
-
-        # LanceDB is optional at boot. If it can't open (missing pyarrow,
-        # bad dir perms, ...) we still want read/status to work.
-        vectors: LanceStore | None = None
-        try:
-            vectors = await LanceStore.create(
-                config.vectors_dir, dim=config.settings.embedding.dim,
+        # Capture "is this a pre-existing install?" BEFORE we open any
+        # file or directory — opening the SQLite conn creates
+        # memory.db, and the searchbase backend populates vectors/, so
+        # the runner's own heuristic would fire false-positive on a
+        # fresh install if asked later.
+        existing_install = (
+            config.db_path.exists()
+            or (
+                config.vectors_dir.exists()
+                and any(config.vectors_dir.iterdir())
             )
-        except Exception:
-            _log.exception("lancedb init failed; vector-backed endpoints will 503")
-
-        embedder = get_embedder(config)
+        )
+        # Open the raw SQLite connection (no schema yet) so the
+        # migration runner can apply DDL against it before we wrap it
+        # in a SQLiteStore.
+        conn = await SQLiteStore.open_connection(config.db_path)
 
         try:
             await validate_embedder(config)
         except EmbedderValidationError as e:
+            await conn.close()
             _log.exception("embedding startup check failed; aborting boot")
             raise SystemExit(2) from e
 
+        # searchbase is optional at boot. If it can't open (missing
+        # pyarrow, bad dir perms, ...) we still want read/status to work;
+        # vector-backed endpoints then return 503 ``unavailable``. The
+        # migration runner skips the searchbase subsystem in that case
+        # and picks it up on the next boot.
+        searchbase = None
+        try:
+            searchbase = await build_search_backend(config)
+        except Exception:
+            _log.exception("searchbase init failed; vector-backed endpoints will 503")
+
+        # Bring persistent state up to v1 (creates tables on a fresh
+        # install, runs the 0.8.x → v1 deltas on an upgrade). Aborts
+        # boot on failure — partial schemas would let services start
+        # and corrupt data.
+        runner = MigrationRunner(
+            db_conn=conn,
+            admin=searchbase.admin() if searchbase is not None else None,
+            state_path=config.migrations_state_path,
+            data_root=config.data_root,
+            existing_install=existing_install,
+        )
+        try:
+            await runner.run()
+        except Exception as e:
+            _log.exception("migration failed; aborting boot")
+            await conn.close()
+            if searchbase is not None:
+                try:
+                    await searchbase.close()
+                except Exception:
+                    pass
+            raise SystemExit(3) from e
+
+        db = SQLiteStore(conn, config.db_path, storage)
+        sync_checkpoints = await SyncCheckpointStore.create(config.sync_db_path)
+
         events = EventWriter(db)
         app.state.config = config
         app.state.storage = storage
         app.state.db = db
-        app.state.vectors = vectors
-        app.state.embedder = embedder
+        app.state.searchbase = searchbase
         app.state.events = events
-        # IndexWriteBuffer aggregates LanceDB inserts across sessions so
-        # one ``table.add()`` carries many embedder batches' worth of
-        # rows. Without it the ingest path creates one fragment per
-        # embedder batch (10 with DashScope) → vector search eventually
-        # EMFILEs on fd ceiling. See service/index_buffer.py and
-        # docs/issue #4 §4.3.
-        app.state.index_buffer = IndexWriteBuffer(
-            vectors=vectors, db=db,
-            flush_rows=config.settings.index.lance_flush_rows,
-            flush_interval_seconds=config.settings.index.lance_flush_interval_seconds,
-        )
-        app.state.index_buffer.start()
         app.state.read = ReadService(db=db, events=events)
         app.state.ingest = IngestService(
-            db=db, vectors=vectors, embedder=embedder, events=events,
-            index_buffer=app.state.index_buffer,
+            db=db, search=searchbase, events=events,
         )
         app.state.sync_checkpoints = sync_checkpoints
         app.state.sync = SyncWatcher(
@@ -95,14 +121,14 @@ def create_app(config: Config | None = None) -> FastAPI:
             checkpoints=sync_checkpoints,
         )
         app.state.search = SearchService(
-            config=config, db=db, vectors=vectors, embedder=embedder,
+            config=config, db=db, search=searchbase,
         )
         app.state.cards = CardService(
-            db=db, vectors=vectors, embedder=embedder, events=events,
+            db=db, search=searchbase, events=events,
         )
         app.state.reviews = ReviewService(db=db, events=events)
         app.state.recall = RecallService(
-            config=config, db=db, vectors=vectors, embedder=embedder,
+            config=config, db=db, search=searchbase,
         )
 
         # Spin up the watcher if settings says so. start() returns fast
@@ -121,16 +147,12 @@ def create_app(config: Config | None = None) -> FastAPI:
         # when no embedder / lance is available; loop is cancelled on
         # lifespan shutdown.
         app.state.backfill = IndexBackfill(
-            db=db, vectors=vectors, embedder=embedder,
-            index_buffer=app.state.index_buffer,
+            db=db, search=searchbase,
         )
         app.state.backfill.start()
-        # Guaranteed one-shot compaction on every boot — grinds down the
-        # append-only fragment pile (cause of EMFILE in vector search)
-        # so a restart always makes progress. Side path off the re-embed
-        # loop: gated only on vectors, runs in the background, never
-        # blocks startup. See IndexBackfill.trigger_startup_compaction.
-        app.state.backfill.trigger_startup_compaction()
+        # Startup compaction is now owned by searchbase (compact_all runs
+        # in the background from the backend's create()), so backfill no
+        # longer triggers it here.
 
         yield
 
@@ -144,12 +166,12 @@ def create_app(config: Config | None = None) -> FastAPI:
             await app.state.backfill.stop()
         except Exception:
             pass
-        # Drain in-flight LanceDB writes before tearing down the DB
-        # — otherwise pending vectors are lost on shutdown.
-        try:
-            await app.state.index_buffer.stop()
-        except Exception:
-            pass
+        # Shut down the searchbase instance (stops its maintenance task).
+        if app.state.searchbase is not None:
+            try:
+                await app.state.searchbase.close()
+            except Exception:
+                pass
         await db.close()
         await sync_checkpoints.close()
 

{memorytalk-0.9.2 → memorytalk-1.0.0}/memorytalk/api/cards.py

@@ -12,11 +12,12 @@ import datetime as _dt
 from fastapi import APIRouter, HTTPException, Query, Request
 
 from memorytalk.schemas import (
-    CardListResponse, CardMeta, CardTagResponse,
+    CardDeleteResponse, CardListResponse, CardMeta, CardTagResponse,
     CreateCardRequest, CreateCardResponse,
     TagPatchRequest,
 )
 from memorytalk.service import CardConflict, CardServiceError
+from memorytalk.service.cards import CardNotFound
 from memorytalk.util.tag_filter import parse_tag_arg
 from memorytalk.util.tags import TagValidationError, apply_patch
 
@@ -142,3 +143,24 @@ async def patch_card_tags(
                 status_code=404, detail=f"card {card_id!r} not found",
             )
     return CardTagResponse(card_id=card_id, tags=merged)
+
+
+@router.delete("/cards/{card_id}", response_model=CardDeleteResponse)
+async def delete_card(card_id: str, request: Request) -> CardDeleteResponse:
+    """Hard-delete a card: SQLite row + reviews + outbound source_cards
+    + vector embedding + per-card filesystem dir.
+
+    Inbound source_cards (other cards that reference this one) are NOT
+    cascaded — they become dangling references, which the response
+    surfaces as ``inbound_refs_dangling`` so callers can warn the
+    user. recall_event rows that mention this card_id are NOT touched
+    — they're historical records, deleting the card doesn't rewrite
+    history."""
+    svc = request.app.state.cards
+    if svc is None:
+        raise HTTPException(status_code=503, detail="cards service unavailable")
+    try:
+        result = await svc.delete(card_id)
+    except CardNotFound as e:
+        raise HTTPException(status_code=404, detail=str(e))
+    return CardDeleteResponse(**result)

{memorytalk-0.9.2 → memorytalk-1.0.0}/memorytalk/api/sync.py

@@ -12,18 +12,22 @@ from fastapi import APIRouter, Query, Request
 router = APIRouter()
 
 
-def _gather_lance_health(state) -> dict:
-    """Collect LanceDB-layer observability for ``index.lance``.
+async def _gather_lance_health(state) -> dict:
+    """Collect vector-layer observability for ``index.lance``.
 
-    Pulls from three sources: the IndexWriteBuffer (write pipeline),
-    IndexBackfill (compaction cadence), and LanceStore (EMFILE
-    recovery count). All fields default to safe zeros / None when the
-    corresponding component is absent so a partially-disabled boot
-    still returns a well-shaped response.
+    Pulls compaction + EMFILE-recovery state from the searchbase
+    instance's ``health().detail``. The write path is now immediate
+    (no IndexWriteBuffer), so the legacy buffer fields report idle.
+    All fields default to safe zeros / None when searchbase is absent
+    so a partially-disabled boot still returns a well-shaped response.
     """
-    buf = getattr(state, "index_buffer", None)
-    backfill = getattr(state, "backfill", None)
-    vectors = getattr(state, "vectors", None)
+    searchbase = getattr(state, "searchbase", None)
+    detail: dict = {}
+    if searchbase is not None:
+        try:
+            detail = (await searchbase.health()).detail
+        except Exception:
+            detail = {}
 
     soft = hard = None
     try:
@@ -35,22 +39,15 @@ def _gather_lance_health(state) -> dict:
         pass
 
     return {
-        "pending_vector_rows": (buf.pending_rows if buf is not None else 0),
-        "last_flush_at": (buf.last_flush_at_iso if buf is not None else None),
-        "last_flush_error": (buf.last_flush_error if buf is not None else None),
-        "flush_count_since_boot": (buf.flush_count if buf is not None else 0),
-        "last_compaction_at": (
-            backfill.last_compact_at_iso if backfill is not None else None
-        ),
-        "last_compaction_error": (
-            backfill.last_compact_error if backfill is not None else None
-        ),
-        "emfile_recoveries_since_boot": (
-            vectors.emfile_recoveries if vectors is not None else 0
-        ),
-        "last_emfile_at": (
-            vectors.last_emfile_at_iso if vectors is not None else None
-        ),
+        # Writes are immediate now — no buffer to drain.
+        "pending_vector_rows": 0,
+        "last_flush_at": None,
+        "last_flush_error": None,
+        "flush_count_since_boot": 0,
+        "last_compaction_at": detail.get("last_compact_at_iso"),
+        "last_compaction_error": detail.get("last_compact_error"),
+        "emfile_recoveries_since_boot": detail.get("emfile_recoveries", 0),
+        "last_emfile_at": detail.get("last_emfile_at_iso"),
         "fd_soft_limit": soft,
         "fd_hard_limit": hard,
     }
@@ -75,7 +72,7 @@ async def get_sync_status(request: Request, limit: int = Query(5, ge=0, le=20)):
     index["last_index_error"] = (
         backfill.last_error if backfill is not None else None
     )
-    index["lance"] = _gather_lance_health(request.app.state)
+    index["lance"] = await _gather_lance_health(request.app.state)
 
     if not config.settings.sync.enabled:
         return {"status": "disabled", "index": index}

{memorytalk-0.9.2 → memorytalk-1.0.0}/memorytalk/cli/_format.py

@@ -858,6 +858,30 @@ def fmt_card_list(payload: dict, filter_summary: str = "") -> str:
     return "\n".join(lines).rstrip() + "\n"
 
 
+def fmt_card_delete(payload: dict) -> str:
+    """Render DELETE /v3/cards/<cid> response.
+
+    One-line confirmation + a hint about inbound-ref dangling when
+    applicable. We deliberately don't moan about every detail (vector
+    cleared / files cleared / etc.) — those are best-effort and
+    typically silent."""
+    cid = payload.get("card_id") or "?"
+    reviews = int(payload.get("reviews_deleted", 0) or 0)
+    dangling = int(payload.get("inbound_refs_dangling", 0) or 0)
+
+    bits = [f"deleted · `{cid}`"]
+    if reviews:
+        bits.append(
+            f"{reviews} review{'s' if reviews != 1 else ''} removed",
+        )
+    if dangling:
+        bits.append(
+            f"⚠ {dangling} inbound `source_cards` reference"
+            f"{'s' if dangling != 1 else ''} now dangling",
+        )
+    return " · ".join(bits) + "\n"
+
+
 def fmt_card_tag(payload: dict, *, is_query: bool) -> str:
     """Render PATCH /v3/cards/<cid>/tags response.
 

{memorytalk-0.9.2 → memorytalk-1.0.0}/memorytalk/cli/card.py

@@ -1,6 +1,6 @@
-"""CLI: memory.talk card {create, list, tag} — card write + maintenance.
+"""CLI: memory.talk card {create, list, tag, delete} — card write + maintenance.
 
-Three subcommands:
+Subcommands:
 
   create  write a new immutable card (was the bare ``card '<json>'`` form
           in 0.7.x; hard-renamed in 0.8.x because the top level now hosts
@@ -8,6 +8,7 @@ Three subcommands:
   list    multi-filter listing (tag / created_at) — no source/cwd: cards
           aren't from a source the way sessions are
   tag     query / set / unset kv tags on one card
+  delete  hard-delete a card (SQLite row + reviews + vector + files)
 
 The HTTP-call shape is identical to ``cli/session.py`` — both go through
 the same ``api()`` helper and share fmt_/parse_ helpers. See
@@ -20,7 +21,7 @@ import sys
 import click
 
 from memorytalk.cli._format import (
-    fmt_card_created, fmt_card_list, fmt_card_tag, fmt_error,
+    fmt_card_created, fmt_card_delete, fmt_card_list, fmt_card_tag, fmt_error,
 )
 from memorytalk.cli._http import ApiError, api, extract_error_message
 from memorytalk.cli._render import emit_json, emit_json_err, emit_md, emit_md_err
@@ -33,7 +34,7 @@ from memorytalk.util.tags import TagValidationError, parse_kv_args
 
 @click.group("card")
 def card() -> None:
-    """Card write + maintenance: create / list / tag."""
+    """Card write + maintenance: create / list / tag / delete."""
 
 
 # ────────── card create ──────────
@@ -159,6 +160,68 @@ def tag(card_id: str, kv_args: tuple[str, ...], json_out: bool) -> None:
         emit_md(fmt_card_tag(result, is_query=is_query))
 
 
+# ────────── card delete ──────────
+
+@card.command("delete")
+@click.argument("card_id")
+@click.option("--yes", "-y", is_flag=True, default=False,
+              help="Skip the interactive confirmation prompt.")
+@click.option("--json", "json_out", is_flag=True, default=False,
+              help="Emit JSON")
+def delete(card_id: str, yes: bool, json_out: bool) -> None:
+    """Hard-delete a card: SQLite row + reviews + outbound source_cards
+    + vector + per-card filesystem dir.
+
+    Other cards that reference this one via ``source_cards`` are NOT
+    cascaded — their references become dangling. ``recall_event``
+    history that mentions this card is NOT rewritten (history is
+    history). Pass ``--yes`` to skip the confirm prompt.
+    """
+    cfg = Config()
+
+    # Interactive path: pre-fetch the card to show the user WHAT they're
+    # about to delete (insight + created + reviews count). The DELETE
+    # response also surfaces those numbers, but only after the deed.
+    if not yes and not json_out:
+        try:
+            preview = api("POST", "/v3/read", cfg, json_body={"id": card_id})
+        except ApiError as e:
+            emit_md_err(fmt_error(extract_error_message(e.payload)))
+            sys.exit(1)
+        except Exception as e:
+            _emit_err(json_out, f"cannot reach server: {e}")
+            sys.exit(1)
+
+        card_doc = preview.get("card") or {}
+        reviews = card_doc.get("reviews") or []
+        click.echo("", err=True)
+        click.echo(f"card delete · {card_id}", err=True)
+        click.echo(f"  insight:  {card_doc.get('insight', '?')}", err=True)
+        click.echo(f"  created:  {card_doc.get('created_at', '?')}", err=True)
+        click.echo(f"  reviews:  {len(reviews)} (will be deleted with the card)", err=True)
+        click.echo("", err=True)
+        if not click.confirm("Delete this card?", default=False, err=True):
+            click.echo("aborted.", err=True)
+            sys.exit(1)
+
+    try:
+        result = api("DELETE", f"/v3/cards/{card_id}", cfg)
+    except ApiError as e:
+        if json_out:
+            emit_json_err(e.payload)
+        else:
+            emit_md_err(fmt_error(extract_error_message(e.payload)))
+        sys.exit(1)
+    except Exception as e:
+        _emit_err(json_out, f"cannot reach server: {e}")
+        sys.exit(1)
+
+    if json_out:
+        emit_json(result)
+    else:
+        emit_md(fmt_card_delete(result))
+
+
 # ────────── helpers ──────────
 
 def _summarize_filters(

{memorytalk-0.9.2 → memorytalk-1.0.0}/memorytalk/config.py

@@ -188,6 +188,15 @@ class Config:
     def vectors_dir(self) -> Path:
         return self.data_root / "vectors"
 
+    @property
+    def migrations_state_path(self) -> Path:
+        """Where the migration runner tracks which (version, subsystem)
+        pairs have been applied. JSON file — see
+        ``memorytalk.migration.state``. Empty / missing on a fresh
+        install or on the first 0.8.x → 0.9 boot; the runner picks the
+        mode from that + a heuristic on ``memory.db`` existence."""
+        return self.data_root / "migrations_state.json"
+
     @property
     def sessions_dir(self) -> Path:
         return self.data_root / "sessions"
@@ -220,6 +229,14 @@ class Config:
         chatty watcher doesn't crowd out request/error logs."""
         return self.sync_log_dir / "watch.log"
 
+    @property
+    def searchbase_log_dir(self) -> Path:
+        """Per-category searchbase logs (maintenance / query / index).
+        See ``searchbase/local/_logging.py`` — three files, daily
+        rotation, 14-day retention. Distinct from ``search_log_dir``
+        (business audit, JSONL) — this one is the backend-side trail."""
+        return self.logs_dir / "searchbase"
+
     @property
     def pid_path(self) -> Path:
         return self.data_root / "server.pid"
@@ -235,7 +252,7 @@ class Config:
         for d in [
             self.data_root, self.vectors_dir, self.sessions_dir,
             self.cards_dir, self.logs_dir, self.search_log_dir,
-            self.sync_log_dir,
+            self.sync_log_dir, self.searchbase_log_dir,
         ]:
             d.mkdir(parents=True, exist_ok=True)
 

memorytalk-1.0.0/memorytalk/migration/__init__.py

@@ -0,0 +1,23 @@
+"""memorytalk.migration — schema-evolution runner.
+
+The framework that brings persistent state (SQLite + searchbase) from
+whatever shape it has on disk up to what the current code declares.
+The version content lives in ``memorytalk.migrations`` (peer package);
+this module is the runner that discovers, picks a mode, and applies.
+
+Design: docs/works/v3/migration.md.
+
+Public surface:
+
+    runner = MigrationRunner(
+        db_conn=...,
+        admin=backend.admin(),
+        state_path=config.migrations_state_path,
+    )
+    summary = await runner.run()
+"""
+from memorytalk.migration._types import Mode, Summary
+from memorytalk.migration.runner import MigrationRunner
+from memorytalk.migration.state import MigrationState
+
+__all__ = ["MigrationRunner", "MigrationState", "Mode", "Summary"]

memorytalk-1.0.0/memorytalk/migration/_types.py

@@ -0,0 +1,27 @@
+"""Value types for the migration runner."""
+from __future__ import annotations
+
+from typing import Literal
+
+from pydantic import BaseModel, Field
+
+
+Mode = Literal["init_latest", "upgrade_from_zero", "catch_up"]
+Subsystem = Literal["database", "searchbase"]
+
+
+class Summary(BaseModel):
+    """What the runner did on this invocation. Returned by
+    :meth:`MigrationRunner.run`; useful for log lines + tests."""
+
+    mode: Mode
+    applied: list[tuple[str, Subsystem]] = Field(default_factory=list)
+    skipped: list[tuple[str, Subsystem]] = Field(default_factory=list)
+
+    @property
+    def applied_count(self) -> int:
+        return len(self.applied)
+
+    @property
+    def skipped_count(self) -> int:
+        return len(self.skipped)

memorytalk-1.0.0/memorytalk/migration/discover.py

@@ -0,0 +1,50 @@
+"""Discover migration version directories under ``memorytalk.migrations``.
+
+The runner doesn't import migrations eagerly — they're imported lazily
+when needed (``import_init`` / ``import_up``) so that a broken
+migration in version N doesn't keep the runner from listing and
+applying versions < N.
+"""
+from __future__ import annotations
+
+import importlib
+import pkgutil
+import re
+
+
+_VERSION_RE = re.compile(r"^v(\d+)$")
+
+
+def discover_versions(package: str = "memorytalk.migrations") -> list[str]:
+    """Return all ``vN`` subpackages under ``package``, sorted by N.
+
+    Returns an empty list if the package doesn't exist (no migrations
+    declared yet — fresh-install case before v1 lands)."""
+    try:
+        pkg = importlib.import_module(package)
+    except ImportError:
+        return []
+    versions: list[tuple[int, str]] = []
+    for info in pkgutil.iter_modules(pkg.__path__):
+        if not info.ispkg:
+            continue
+        m = _VERSION_RE.match(info.name)
+        if m:
+            versions.append((int(m.group(1)), info.name))
+    versions.sort()
+    return [name for _, name in versions]
+
+
+def import_migration_module(
+    version: str, subsystem: str, method: str,
+    *, package: str = "memorytalk.migrations",
+):
+    """Import e.g. ``memorytalk.migrations.v1.up_searchbase``. Returns
+    the module so the caller can call ``await module.run(handle)``.
+
+    ``method`` is ``"init"`` or ``"up"``.
+    ``subsystem`` is ``"database"`` or ``"searchbase"``.
+    """
+    return importlib.import_module(
+        f"{package}.{version}.{method}_{subsystem}",
+    )