basic-memory 0.22.1__py3-none-any.whl → 0.23.1.dev2__py3-none-any.whl

basic_memory/__init__.py

@@ -1,7 +1,7 @@
 """basic-memory - Local-first knowledge management combining Zettelkasten with knowledge graphs"""
 
 # Package version - updated by release automation
-__version__ = "0.22.1"
+__version__ = "0.23.0"
 
 # API version for FastAPI - independent of package version
-__api_version__ = "v0"
+__api_version__ = "v2"

basic_memory/alembic/env.py

@@ -2,37 +2,34 @@
 
 import asyncio
 import os
+import sys
 from contextlib import suppress
 from logging.config import fileConfig
 
 from loguru import logger
+from sqlalchemy import engine_from_config, pool
+from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine
 
-# Allow nested event loops (needed for pytest-asyncio and other async contexts)
-# Note: nest_asyncio doesn't work with uvloop or Python 3.14+, so we handle those cases separately
-import sys
+from alembic import context
+
+from basic_memory.config import ConfigManager
+from basic_memory.migration_loop import running_on_uvloop
 
-if sys.version_info < (3, 14):
+# Allow nested event loops (needed for pytest-asyncio and other async contexts).
+# nest_asyncio cannot patch a uvloop loop or Python 3.14+; in those cases we skip
+# it and rely on the thread-based fallback in run_migrations_online() instead
+# (see basic_memory.migration_loop for why uvloop must be detected up front).
+if sys.version_info < (3, 14) and not running_on_uvloop():
     try:
         import nest_asyncio
 
         nest_asyncio.apply()
     except (ImportError, ValueError) as exc:
         # Trigger: nest_asyncio is absent (ImportError) or refuses to patch the
-        # running loop (ValueError, e.g. the uvloop policy installed for the
-        # Postgres backend - #831/#877).
-        # Why: the uvloop ValueError is now an *expected* path on every Postgres
-        # startup, so swallowing it silently hides a routine branch.
+        # running loop (ValueError).
         # Outcome: log at DEBUG (observable, not noisy) and fall through to the
         # thread-based migration fallback.
         logger.debug(f"nest_asyncio not applied ({exc!r}); using thread-based migration fallback")
-# For Python 3.14+, we rely on the thread-based fallback in run_migrations_online()
-
-from sqlalchemy import engine_from_config, pool
-from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine
-
-from alembic import context
-
-from basic_memory.config import ConfigManager
 
 # Trigger: only set test env when actually running under pytest
 # Why: alembic/env.py is imported during normal operations (MCP server startup, migrations)
@@ -170,17 +167,30 @@ def _run_async_migrations_in_thread(connectable) -> None:
         future.result()  # Wait for completion and re-raise any exceptions
 
 
-def _run_async_engine_migrations(connectable) -> None:
-    """Run async-engine migrations with a running-loop fallback."""
+def _loop_is_running() -> bool:
+    """Check if an event loop is currently running."""
     try:
+        asyncio.get_running_loop()
+    except RuntimeError:
+        # Trigger: no running loop (get_running_loop raises RuntimeError)
+        # Outcome: return False to indicate no loop is running
+        return False
+    return True
+
+
+def _run_async_engine_migrations(connectable) -> None:
+    """Run async migrations, adapting to whether an event loop is already running."""
+
+    # Trigger: caller may be inside a running event loop (e.g. pytest-asyncio, uvicorn).
+    # Why: asyncio.run() raises RuntimeError when nested inside a running loop, so we
+    # detect the condition up front via _loop_is_running() instead of catching the error.
+    # Outcome: running-loop context -> thread fallback; no loop -> asyncio.run() directly.
+    if _loop_is_running():
+        # Can't nest asyncio.run() inside a running loop -> offload to a thread.
+        _run_async_migrations_in_thread(connectable)
+    else:
+        # No running loop: asyncio.run() is safe; let any unexpected errors propagate.
         _run_async_migrations_with_asyncio_run(connectable)
-    except RuntimeError as e:
-        if "cannot be called from a running event loop" in str(e):
-            # We're in a running event loop (likely uvloop or Python 3.14+ tests).
-            # Switch to a dedicated thread so Alembic can finish without nesting loops.
-            _run_async_migrations_in_thread(connectable)
-        else:
-            raise
 
 
 def run_migrations_online() -> None:

basic_memory/alembic/versions/2d26b287813b_dedupe_observations_and_purge_stale_.py

@@ -0,0 +1,69 @@
+"""Dedupe observations and purge stale observation search rows.
+
+Revision ID: 2d26b287813b
+Revises: r1m2n3o4p5q6
+Create Date: 2026-08-10 19:22:17.344875
+
+"""
+
+from typing import Sequence, Union
+
+from alembic import op
+from sqlalchemy import inspect
+
+
+# revision identifiers, used by Alembic.
+revision: str = "2d26b287813b"
+down_revision: Union[str, None] = "r1m2n3o4p5q6"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def _table_exists(connection, table_name: str) -> bool:
+    """Return whether a runtime-managed table is present for this database."""
+    return table_name in inspect(connection).get_table_names()
+
+
+def upgrade() -> None:
+    """Repair duplicate observation projections and their orphaned search rows."""
+    connection = op.get_bind()
+    if connection.dialect.name == "postgresql":
+        # PostgreSQL JSON has no equality operator, so its textual representation is the
+        # grouping key. GROUP BY treats NULL context and tags values consistently.
+        op.execute(
+            """
+            DELETE FROM observation
+            WHERE id NOT IN (
+                SELECT MIN(id)
+                FROM observation
+                GROUP BY entity_id, category, content, context, tags::text
+            )
+            """
+        )
+    else:
+        op.execute(
+            """
+            DELETE FROM observation
+            WHERE id NOT IN (
+                SELECT MIN(id)
+                FROM observation
+                GROUP BY entity_id, category, content, context, tags
+            )
+            """
+        )
+
+    # SQLite creates search_index at runtime rather than through Alembic. A fresh database
+    # therefore has no table or stale rows to purge during this migration.
+    if _table_exists(connection, "search_index"):
+        op.execute(
+            """
+            DELETE FROM search_index
+            WHERE type = 'observation'
+              AND id NOT IN (SELECT id FROM observation)
+            """
+        )
+
+
+def downgrade() -> None:
+    """No-op: removed duplicate and orphaned projection rows cannot be reconstructed."""
+    pass

basic_memory/alembic/versions/o8j9k0l1m2n3_add_note_file_vacate_table.py

@@ -0,0 +1,52 @@
+"""Add note_file_vacate table
+
+Revision ID: o8j9k0l1m2n3
+Revises: n7i8j9k0l1m2
+Create Date: 2026-07-26 12:00:00.000000
+
+"""
+
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+
+
+# revision identifiers, used by Alembic.
+revision: str = "o8j9k0l1m2n3"
+down_revision: Union[str, None] = "n7i8j9k0l1m2"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    """Create note_file_vacate: durable proof a source path was vacated by a move.
+
+    The indexer keys on (project_id, file_path) to tell a move's lingering source object (skip
+    re-import) from a legitimate byte-identical copy (index as new) — see
+    basic-memory-cloud#1601.
+    """
+    op.create_table(
+        "note_file_vacate",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("project_id", sa.Integer(), nullable=False),
+        sa.Column("entity_id", sa.Integer(), nullable=True),
+        sa.Column("file_path", sa.String(), nullable=False),
+        sa.Column("file_checksum", sa.String(), nullable=True),
+        sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
+        sa.ForeignKeyConstraint(["project_id"], ["project.id"], ondelete="CASCADE"),
+        sa.ForeignKeyConstraint(["entity_id"], ["entity.id"], ondelete="SET NULL"),
+        sa.PrimaryKeyConstraint("id"),
+        sa.UniqueConstraint(
+            "project_id", "file_path", name="uix_note_file_vacate_project_file_path"
+        ),
+    )
+    op.create_index(
+        "ix_note_file_vacate_project_id", "note_file_vacate", ["project_id"], unique=False
+    )
+
+
+def downgrade() -> None:
+    """Drop note_file_vacate and its supporting index."""
+    op.drop_index("ix_note_file_vacate_project_id", table_name="note_file_vacate")
+    op.drop_table("note_file_vacate")

basic_memory/alembic/versions/p9k0l1m2n3o4_add_vector_index_manifest_state.py

@@ -0,0 +1,132 @@
+"""Add vector index identity and readiness to the semantic manifest.
+
+Revision ID: p9k0l1m2n3o4
+Revises: o8j9k0l1m2n3
+Create Date: 2026-07-21 00:00:00.000000
+
+"""
+
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy import inspect
+
+
+revision: str = "p9k0l1m2n3o4"
+down_revision: Union[str, None] = "o8j9k0l1m2n3"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def _create_note_file_vacate_table() -> None:
+    """Backfill the canonical o8 schema when the duplicate vector revision ran."""
+    op.create_table(
+        "note_file_vacate",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("project_id", sa.Integer(), nullable=False),
+        sa.Column("entity_id", sa.Integer(), nullable=True),
+        sa.Column("file_path", sa.String(), nullable=False),
+        sa.Column("file_checksum", sa.String(), nullable=True),
+        sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
+        sa.ForeignKeyConstraint(["project_id"], ["project.id"], ondelete="CASCADE"),
+        sa.ForeignKeyConstraint(["entity_id"], ["entity.id"], ondelete="SET NULL"),
+        sa.PrimaryKeyConstraint("id"),
+        sa.UniqueConstraint(
+            "project_id",
+            "file_path",
+            name="uix_note_file_vacate_project_file_path",
+        ),
+    )
+    op.create_index(
+        "ix_note_file_vacate_project_id",
+        "note_file_vacate",
+        ["project_id"],
+        unique=False,
+    )
+
+
+def upgrade() -> None:
+    """Reconcile both historical o8 schemas, then add vector manifest state.
+
+    For a short period, note-file vacates and vector manifest state shared the
+    o8 revision ID. A database can therefore arrive here with either schema
+    while Alembic reports the same predecessor. Restore the canonical o8
+    note-file table first, then idempotently apply the PostgreSQL vector state.
+    SQLite creates its derived vector table lazily at runtime, so it only needs
+    the missing canonical table repaired.
+    """
+    connection = op.get_bind()
+    inspector = inspect(connection)
+    tables = set(inspector.get_table_names())
+
+    if "note_file_vacate" not in tables:
+        _create_note_file_vacate_table()
+
+    if connection.dialect.name != "postgresql":
+        return
+    if "search_vector_chunks" not in tables:
+        return
+
+    op.execute("ALTER TABLE search_vector_chunks ADD COLUMN IF NOT EXISTS vector_index TEXT")
+    op.execute("ALTER TABLE search_vector_chunks ADD COLUMN IF NOT EXISTS embedding_status TEXT")
+    op.execute(
+        """
+        UPDATE search_vector_chunks
+        SET vector_index = 'pgvector'
+        WHERE vector_index IS NULL
+        """
+    )
+
+    if "search_vector_embeddings" in tables:
+        op.execute(
+            """
+            UPDATE search_vector_chunks AS chunks
+            SET embedding_status = CASE
+                WHEN EXISTS (
+                    SELECT 1 FROM search_vector_embeddings AS embeddings
+                    WHERE embeddings.chunk_id = chunks.id
+                ) THEN 'ready'
+                ELSE 'pending'
+            END
+            WHERE embedding_status IS NULL
+            """
+        )
+    else:
+        op.execute(
+            """
+            UPDATE search_vector_chunks
+            SET embedding_status = 'pending'
+            WHERE embedding_status IS NULL
+            """
+        )
+
+    op.execute("ALTER TABLE search_vector_chunks ALTER COLUMN vector_index SET NOT NULL")
+    op.execute("ALTER TABLE search_vector_chunks ALTER COLUMN embedding_status SET NOT NULL")
+    constraint_names = {
+        constraint.get("name")
+        for constraint in inspector.get_check_constraints("search_vector_chunks")
+    }
+    if "ck_search_vector_chunks_embedding_status" not in constraint_names:
+        op.create_check_constraint(
+            "ck_search_vector_chunks_embedding_status",
+            "search_vector_chunks",
+            "embedding_status IN ('pending', 'ready')",
+        )
+
+
+def downgrade() -> None:
+    """Remove vector index manifest state from PostgreSQL."""
+    connection = op.get_bind()
+    if connection.dialect.name != "postgresql":
+        return
+    if "search_vector_chunks" not in inspect(connection).get_table_names():
+        return
+
+    op.drop_constraint(
+        "ck_search_vector_chunks_embedding_status",
+        "search_vector_chunks",
+        type_="check",
+    )
+    op.execute("ALTER TABLE search_vector_chunks DROP COLUMN IF EXISTS embedding_status")
+    op.execute("ALTER TABLE search_vector_chunks DROP COLUMN IF EXISTS vector_index")

basic_memory/alembic/versions/q0l1m2n3o4p5_add_relation_search_refresh_table.py

@@ -0,0 +1,57 @@
+"""Add durable relation search refresh work.
+
+Revision ID: q0l1m2n3o4p5
+Revises: p9k0l1m2n3o4
+Create Date: 2026-07-27 18:00:00.000000
+
+"""
+
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+
+
+revision: str = "q0l1m2n3o4p5"
+down_revision: Union[str, None] = "p9k0l1m2n3o4"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    """Create durable work items for relation-derived search refreshes."""
+    op.create_table(
+        "relation_search_refresh",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("project_id", sa.Integer(), nullable=False),
+        sa.Column("entity_id", sa.Integer(), nullable=False),
+        sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
+        sa.ForeignKeyConstraint(["project_id"], ["project.id"], ondelete="CASCADE"),
+        sa.ForeignKeyConstraint(["entity_id"], ["entity.id"], ondelete="CASCADE"),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_index(
+        "ix_relation_search_refresh_project_id",
+        "relation_search_refresh",
+        ["project_id"],
+        unique=False,
+    )
+    op.create_index(
+        "ix_relation_search_refresh_entity_id",
+        "relation_search_refresh",
+        ["entity_id"],
+        unique=False,
+    )
+
+
+def downgrade() -> None:
+    """Drop durable relation-search refresh work."""
+    op.drop_index(
+        "ix_relation_search_refresh_entity_id",
+        table_name="relation_search_refresh",
+    )
+    op.drop_index(
+        "ix_relation_search_refresh_project_id",
+        table_name="relation_search_refresh",
+    )
+    op.drop_table("relation_search_refresh")

basic_memory/alembic/versions/r1m2n3o4p5q6_add_relation_generation.py

@@ -0,0 +1,95 @@
+"""Add generation ownership to relation projections.
+
+Revision ID: r1m2n3o4p5q6
+Revises: q0l1m2n3o4p5
+Create Date: 2026-08-09 17:00:00.000000
+
+"""
+
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+
+
+revision: str = "r1m2n3o4p5q6"
+down_revision: Union[str, None] = "q0l1m2n3o4p5"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    """Backfill relation rows with their source note's accepted generation."""
+    op.add_column(
+        "relation_search_refresh",
+        sa.Column("publication_generation", sa.BigInteger(), nullable=True),
+    )
+    op.create_index(
+        "ix_relation_search_refresh_project_publication_generation",
+        "relation_search_refresh",
+        ["project_id", "publication_generation"],
+        unique=False,
+    )
+    op.add_column(
+        "relation",
+        sa.Column(
+            "generation",
+            sa.BigInteger(),
+            nullable=False,
+            server_default=sa.text("0"),
+        ),
+    )
+
+    connection = op.get_bind()
+    if connection.dialect.name == "postgresql":
+        op.execute(
+            """
+            UPDATE relation
+            SET generation = note_content.db_version
+            FROM note_content
+            WHERE relation.from_id = note_content.entity_id
+              AND relation.project_id = note_content.project_id
+            """
+        )
+    else:
+        op.execute(
+            """
+            UPDATE relation
+            SET generation = COALESCE(
+                (
+                    SELECT note_content.db_version
+                    FROM note_content
+                    WHERE note_content.entity_id = relation.from_id
+                      AND note_content.project_id = relation.project_id
+                ),
+                0
+            )
+            """
+        )
+
+    op.create_index(
+        "ix_relation_project_from_generation",
+        "relation",
+        ["project_id", "from_id", "generation"],
+        unique=False,
+    )
+
+
+def downgrade() -> None:
+    """Remove relation generation ownership."""
+    op.drop_index("ix_relation_project_from_generation", table_name="relation")
+    if op.get_bind().dialect.name == "postgresql":
+        op.drop_column("relation", "generation")
+    else:
+        with op.batch_alter_table("relation") as batch_op:
+            batch_op.drop_column("generation")
+
+    op.drop_index(
+        "ix_relation_search_refresh_project_publication_generation",
+        table_name="relation_search_refresh",
+    )
+    if op.get_bind().dialect.name == "postgresql":
+        op.drop_column("relation_search_refresh", "publication_generation")
+    else:
+        with op.batch_alter_table("relation_search_refresh") as batch_op:
+            batch_op.drop_column("publication_generation")

basic_memory/api/app.py

@@ -5,7 +5,6 @@ from contextlib import asynccontextmanager
 from fastapi import FastAPI, HTTPException, Request
 from fastapi.exception_handlers import http_exception_handler
 from fastapi.responses import JSONResponse
-from fastapi.routing import APIRouter
 from loguru import logger
 
 from basic_memory import __version__ as version
@@ -20,14 +19,12 @@ from basic_memory.api.v2.routers import (
     prompt_router as v2_prompt,
     importer_router as v2_importer,
     schema_router as v2_schema,
-)
-from basic_memory.api.v2.routers.project_router import (
-    add_project,
-    list_projects,
-    synchronize_projects,
+    inspect_router as v2_inspect,
 )
 import logfire
+from basic_memory.index.note_content_materialization import drain_pending_materializations
 from basic_memory.config import init_api_logging
+from basic_memory.index.local_schedulers import drain_background_tasks
 from basic_memory.services.exceptions import EntityAlreadyExistsError
 from basic_memory.services.initialization import initialize_app
 from basic_memory.workspace_context import (
@@ -67,10 +64,10 @@ async def lifespan(app: FastAPI):  # pragma: no cover
         app.state.session_maker = session_maker
         logger.info("Database connections cached in app state")
 
-        # Create and start sync coordinator (lifecycle centralized in coordinator)
-        sync_coordinator = container.create_sync_coordinator()
-        await sync_coordinator.start()
-        app.state.sync_coordinator = sync_coordinator
+        # Create and start local watch coordinator (lifecycle centralized in coordinator)
+        watch_coordinator = container.create_watch_coordinator()
+        await watch_coordinator.start()
+        app.state.watch_coordinator = watch_coordinator
 
     # Proceed with startup
     yield
@@ -82,7 +79,13 @@ async def lifespan(app: FastAPI):  # pragma: no cover
         mode=container.mode.name.lower(),
     ):
         logger.info("Shutting down Basic Memory API")
-        await sync_coordinator.stop()
+        await watch_coordinator.stop()
+        # A local note write returns 202 before its markdown file is written;
+        # SIGTERM can land while that materialization (and the vector sync /
+        # relation resolution it schedules) is still queued. Drain both queues
+        # before the engine closes so an accepted write is never lost.
+        await drain_pending_materializations()
+        await drain_background_tasks()
         await container.shutdown_database()
 
 
@@ -129,19 +132,12 @@ app.include_router(v2_directory, prefix="/v2/projects/{project_id}")
 app.include_router(v2_prompt, prefix="/v2/projects/{project_id}")
 app.include_router(v2_importer, prefix="/v2/projects/{project_id}")
 app.include_router(v2_schema, prefix="/v2/projects/{project_id}")
+app.include_router(v2_inspect, prefix="/v2/projects/{project_id}")
 app.include_router(v2_project, prefix="/v2")
 
 # Legacy web app proxy paths (compat with /proxy/projects/projects)
 app.include_router(v2_project, prefix="/proxy/projects")
 
-# Legacy v1 compat: older CLI versions (v0.18.0 and earlier) call /projects/...
-# Using router mount causes 307 redirect which proxy doesn't follow, so add explicit routes
-legacy_router = APIRouter(tags=["legacy"])
-legacy_router.add_api_route("/projects/projects", list_projects, methods=["GET"])
-legacy_router.add_api_route("/projects/projects", add_project, methods=["POST"])
-legacy_router.add_api_route("/projects/config/sync", synchronize_projects, methods=["POST"])
-app.include_router(legacy_router)
-
 # V2 routers are the only public API surface