basic-memory 0.7.0__py3-none-any.whl → 0.17.4__py3-none-any.whl

basic_memory/mcp/tools/write_note.py

@@ -0,0 +1,230 @@
+"""Write note tool for Basic Memory MCP server."""
+
+from typing import List, Union, Optional
+
+from loguru import logger
+
+from basic_memory.mcp.async_client import get_client
+from basic_memory.mcp.project_context import get_active_project, add_project_metadata
+from basic_memory.mcp.server import mcp
+from basic_memory.telemetry import track_mcp_tool
+from fastmcp import Context
+from basic_memory.schemas.base import Entity
+from basic_memory.utils import parse_tags, validate_project_path
+
+# Define TagType as a Union that can accept either a string or a list of strings or None
+TagType = Union[List[str], str, None]
+
+
+@mcp.tool(
+    description="Create or update a markdown note. Returns a markdown formatted summary of the semantic content.",
+)
+async def write_note(
+    title: str,
+    content: str,
+    folder: str,
+    project: Optional[str] = None,
+    tags: list[str] | str | None = None,
+    note_type: str = "note",
+    context: Context | None = None,
+) -> str:
+    """Write a markdown note to the knowledge base.
+
+    Creates or updates a markdown note with semantic observations and relations.
+
+    Project Resolution:
+    Server resolves projects in this order: Single Project Mode → project parameter → default project.
+    If project unknown, use list_memory_projects() or recent_activity() first.
+
+    The content can include semantic observations and relations using markdown syntax:
+
+    Observations format:
+        `- [category] Observation text #tag1 #tag2 (optional context)`
+
+        Examples:
+        `- [design] Files are the source of truth #architecture (All state comes from files)`
+        `- [tech] Using SQLite for storage #implementation`
+        `- [note] Need to add error handling #todo`
+
+    Relations format:
+        - Explicit: `- relation_type [[Entity]] (optional context)`
+        - Inline: Any `[[Entity]]` reference creates a relation
+
+        Examples:
+        `- depends_on [[Content Parser]] (Need for semantic extraction)`
+        `- implements [[Search Spec]] (Initial implementation)`
+        `- This feature extends [[Base Design]] and uses [[Core Utils]]`
+
+    Args:
+        title: The title of the note
+        content: Markdown content for the note, can include observations and relations
+        folder: Folder path relative to project root where the file should be saved.
+                Use forward slashes (/) as separators. Use "/" or "" to write to project root.
+                Examples: "notes", "projects/2025", "research/ml", "/" (root)
+        project: Project name to write to. Optional - server will resolve using the
+                hierarchy above. If unknown, use list_memory_projects() to discover
+                available projects.
+        tags: Tags to categorize the note. Can be a list of strings, a comma-separated string, or None.
+              Note: If passing from external MCP clients, use a string format (e.g. "tag1,tag2,tag3")
+        note_type: Type of note to create (stored in frontmatter). Defaults to "note".
+                   Can be "guide", "report", "config", "person", etc.
+        context: Optional FastMCP context for performance caching.
+
+    Returns:
+        A markdown formatted summary of the semantic content, including:
+        - Creation/update status with project name
+        - File path and checksum
+        - Observation counts by category
+        - Relation counts (resolved/unresolved)
+        - Tags if present
+        - Session tracking metadata for project awareness
+
+    Examples:
+        # Assistant flow when project is unknown
+        # 1. list_memory_projects() -> Ask user which project
+        # 2. User: "Use my-research"
+        # 3. write_note(...) and remember "my-research" for session
+
+        # Create a simple note
+        write_note(
+            project="my-research",
+            title="Meeting Notes",
+            folder="meetings",
+            content="# Weekly Standup\\n\\n- [decision] Use SQLite for storage #tech"
+        )
+
+        # Create a note with tags and note type
+        write_note(
+            project="work-project",
+            title="API Design",
+            folder="specs",
+            content="# REST API Specification\\n\\n- implements [[Authentication]]",
+            tags=["api", "design"],
+            note_type="guide"
+        )
+
+        # Update existing note (same title/folder)
+        write_note(
+            project="my-research",
+            title="Meeting Notes",
+            folder="meetings",
+            content="# Weekly Standup\\n\\n- [decision] Use PostgreSQL instead #tech"
+        )
+
+    Raises:
+        HTTPError: If project doesn't exist or is inaccessible
+        SecurityError: If folder path attempts path traversal
+    """
+    track_mcp_tool("write_note")
+    async with get_client() as client:
+        logger.info(
+            f"MCP tool call tool=write_note project={project} folder={folder}, title={title}, tags={tags}"
+        )
+
+        # Get and validate the project (supports optional project parameter)
+        active_project = await get_active_project(client, project, context)
+
+        # Normalize "/" to empty string for root folder (must happen before validation)
+        if folder == "/":
+            folder = ""
+
+        # Validate folder path to prevent path traversal attacks
+        project_path = active_project.home
+        if folder and not validate_project_path(folder, project_path):
+            logger.warning(
+                "Attempted path traversal attack blocked",
+                folder=folder,
+                project=active_project.name,
+            )
+            return f"# Error\n\nFolder path '{folder}' is not allowed - paths must stay within project boundaries"
+
+        # Process tags using the helper function
+        tag_list = parse_tags(tags)
+        # Create the entity request
+        metadata = {"tags": tag_list} if tag_list else None
+        entity = Entity(
+            title=title,
+            folder=folder,
+            entity_type=note_type,
+            content_type="text/markdown",
+            content=content,
+            entity_metadata=metadata,
+        )
+
+        # Import here to avoid circular import
+        from basic_memory.mcp.clients import KnowledgeClient
+
+        # Use typed KnowledgeClient for API calls
+        knowledge_client = KnowledgeClient(client, active_project.external_id)
+
+        # Try to create the entity first (optimistic create)
+        logger.debug(f"Attempting to create entity permalink={entity.permalink}")
+        action = "Created"  # Default to created
+        try:
+            result = await knowledge_client.create_entity(entity.model_dump())
+            action = "Created"
+        except Exception as e:
+            # If creation failed due to conflict (already exists), try to update
+            if (
+                "409" in str(e)
+                or "conflict" in str(e).lower()
+                or "already exists" in str(e).lower()
+            ):
+                logger.debug(f"Entity exists, updating instead permalink={entity.permalink}")
+                try:
+                    if not entity.permalink:
+                        raise ValueError("Entity permalink is required for updates")  # pragma: no cover
+                    entity_id = await knowledge_client.resolve_entity(entity.permalink)
+                    result = await knowledge_client.update_entity(entity_id, entity.model_dump())
+                    action = "Updated"
+                except Exception as update_error:  # pragma: no cover
+                    # Re-raise the original error if update also fails
+                    raise e from update_error  # pragma: no cover
+            else:
+                # Re-raise if it's not a conflict error
+                raise  # pragma: no cover
+        summary = [
+            f"# {action} note",
+            f"project: {active_project.name}",
+            f"file_path: {result.file_path}",
+            f"permalink: {result.permalink}",
+            f"checksum: {result.checksum[:8] if result.checksum else 'unknown'}",
+        ]
+
+        # Count observations by category
+        categories = {}
+        if result.observations:
+            for obs in result.observations:
+                categories[obs.category] = categories.get(obs.category, 0) + 1
+
+            summary.append("\n## Observations")
+            for category, count in sorted(categories.items()):
+                summary.append(f"- {category}: {count}")
+
+        # Count resolved/unresolved relations
+        unresolved = 0
+        resolved = 0
+        if result.relations:
+            unresolved = sum(1 for r in result.relations if not r.to_id)
+            resolved = len(result.relations) - unresolved
+
+            summary.append("\n## Relations")
+            summary.append(f"- Resolved: {resolved}")
+            if unresolved:
+                summary.append(f"- Unresolved: {unresolved}")
+                summary.append(
+                    "\nNote: Unresolved relations point to entities that don't exist yet."
+                )
+                summary.append(
+                    "They will be automatically resolved when target entities are created or during sync operations."
+                )
+
+        if tag_list:
+            summary.append(f"\n## Tags\n- {', '.join(tag_list)}")
+
+        # Log the response with structured data
+        logger.info(
+            f"MCP tool response: tool=write_note project={active_project.name} action={action} permalink={result.permalink} observations_count={len(result.observations)} relations_count={len(result.relations)} resolved_relations={resolved} unresolved_relations={unresolved}"
+        )
+        summary_result = "\n".join(summary)
+        return add_project_metadata(summary_result, active_project.name)

basic_memory/models/__init__.py

@@ -3,12 +3,13 @@
 import basic_memory
 from basic_memory.models.base import Base
 from basic_memory.models.knowledge import Entity, Observation, Relation
-
-SCHEMA_VERSION = basic_memory.__version__ + "-" + "003"
+from basic_memory.models.project import Project
 
 __all__ = [
     "Base",
     "Entity",
     "Observation",
     "Relation",
+    "Project",
+    "basic_memory",
 ]

basic_memory/models/knowledge.py

@@ -1,6 +1,8 @@
 """Knowledge graph models."""
 
+import uuid
 from datetime import datetime
+from basic_memory.utils import ensure_timezone_aware
 from typing import Optional
 
 from sqlalchemy import (
@@ -12,11 +14,12 @@ from sqlalchemy import (
     DateTime,
     Index,
     JSON,
+    Float,
+    text,
 )
 from sqlalchemy.orm import Mapped, mapped_column, relationship
 
 from basic_memory.models.base import Base
-
 from basic_memory.utils import generate_permalink
 
 
@@ -28,36 +31,73 @@ class Entity(Base):
     - Maps to a file on disk
     - Maintains a checksum for change detection
     - Tracks both source file and semantic properties
+    - Belongs to a specific project
     """
 
     __tablename__ = "entity"
     __table_args__ = (
-        UniqueConstraint("permalink", name="uix_entity_permalink"),  # Make permalink unique
+        # Regular indexes
         Index("ix_entity_type", "entity_type"),
         Index("ix_entity_title", "title"),
+        Index("ix_entity_external_id", "external_id", unique=True),
         Index("ix_entity_created_at", "created_at"),  # For timeline queries
         Index("ix_entity_updated_at", "updated_at"),  # For timeline queries
+        Index("ix_entity_project_id", "project_id"),  # For project filtering
+        # Project-specific uniqueness constraints
+        Index(
+            "uix_entity_permalink_project",
+            "permalink",
+            "project_id",
+            unique=True,
+            sqlite_where=text("content_type = 'text/markdown' AND permalink IS NOT NULL"),
+        ),
+        Index(
+            "uix_entity_file_path_project",
+            "file_path",
+            "project_id",
+            unique=True,
+        ),
     )
 
     # Core identity
     id: Mapped[int] = mapped_column(Integer, primary_key=True)
+    # External UUID for API references - stable identifier that won't change
+    external_id: Mapped[str] = mapped_column(
+        String, unique=True, default=lambda: str(uuid.uuid4())
+    )
     title: Mapped[str] = mapped_column(String)
     entity_type: Mapped[str] = mapped_column(String)
     entity_metadata: Mapped[Optional[dict]] = mapped_column(JSON, nullable=True)
     content_type: Mapped[str] = mapped_column(String)
 
-    # Normalized path for URIs
-    permalink: Mapped[str] = mapped_column(String, unique=True, index=True)
+    # Project reference
+    project_id: Mapped[int] = mapped_column(Integer, ForeignKey("project.id"), nullable=False)
+
+    # Normalized path for URIs - required for markdown files only
+    permalink: Mapped[Optional[str]] = mapped_column(String, nullable=True, index=True)
     # Actual filesystem relative path
-    file_path: Mapped[str] = mapped_column(String, unique=True, index=True)
+    file_path: Mapped[str] = mapped_column(String, index=True)
     # checksum of file
     checksum: Mapped[Optional[str]] = mapped_column(String, nullable=True)
 
+    # File metadata for sync
+    # mtime: file modification timestamp (Unix epoch float) for change detection
+    mtime: Mapped[Optional[float]] = mapped_column(Float, nullable=True)
+    # size: file size in bytes for quick change detection
+    size: Mapped[Optional[int]] = mapped_column(Integer, nullable=True)
+
     # Metadata and tracking
-    created_at: Mapped[datetime] = mapped_column(DateTime)
-    updated_at: Mapped[datetime] = mapped_column(DateTime)
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), default=lambda: datetime.now().astimezone()
+    )
+    updated_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True),
+        default=lambda: datetime.now().astimezone(),
+        onupdate=lambda: datetime.now().astimezone(),
+    )
 
     # Relationships
+    project = relationship("Project", back_populates="entities")
     observations = relationship(
         "Observation", back_populates="entity", cascade="all, delete-orphan"
     )
@@ -79,8 +119,23 @@ class Entity(Base):
         """Get all relations (incoming and outgoing) for this entity."""
         return self.incoming_relations + self.outgoing_relations
 
+    @property
+    def is_markdown(self):
+        """Check if the entity is a markdown file."""
+        return self.content_type == "text/markdown"
+
+    def __getattribute__(self, name):
+        """Override attribute access to ensure datetime fields are timezone-aware."""
+        value = super().__getattribute__(name)
+
+        # Ensure datetime fields are timezone-aware
+        if name in ("created_at", "updated_at") and isinstance(value, datetime):
+            return ensure_timezone_aware(value)
+
+        return value
+
     def __repr__(self) -> str:
-        return f"Entity(id={self.id}, name='{self.title}', type='{self.entity_type}'"
+        return f"Entity(id={self.id}, external_id='{self.external_id}', name='{self.title}', type='{self.entity_type}', checksum='{self.checksum}')"
 
 
 class Observation(Base):
@@ -96,6 +151,7 @@ class Observation(Base):
     )
 
     id: Mapped[int] = mapped_column(Integer, primary_key=True)
+    project_id: Mapped[int] = mapped_column(Integer, ForeignKey("project.id"), index=True)
     entity_id: Mapped[int] = mapped_column(Integer, ForeignKey("entity.id", ondelete="CASCADE"))
     content: Mapped[str] = mapped_column(Text)
     category: Mapped[str] = mapped_column(String, nullable=False, default="note")
@@ -113,9 +169,14 @@ class Observation(Base):
 
         We can construct these because observations are always defined in
         and owned by a single entity.
+
+        Content is truncated to 200 chars to stay under PostgreSQL's
+        btree index limit of 2704 bytes.
         """
+        # Truncate content to avoid exceeding PostgreSQL's btree index limit
+        content_for_permalink = self.content[:200] if len(self.content) > 200 else self.content
         return generate_permalink(
-            f"{self.entity.permalink}/observations/{self.category}/{self.content}"
+            f"{self.entity.permalink}/observations/{self.category}/{content_for_permalink}"
         )
 
     def __repr__(self) -> str:  # pragma: no cover
@@ -127,13 +188,17 @@ class Relation(Base):
 
     __tablename__ = "relation"
     __table_args__ = (
-        UniqueConstraint("from_id", "to_id", "relation_type", name="uix_relation"),
+        UniqueConstraint("from_id", "to_id", "relation_type", name="uix_relation_from_id_to_id"),
+        UniqueConstraint(
+            "from_id", "to_name", "relation_type", name="uix_relation_from_id_to_name"
+        ),
         Index("ix_relation_type", "relation_type"),
         Index("ix_relation_from_id", "from_id"),  # Add FK indexes
         Index("ix_relation_to_id", "to_id"),
     )
 
     id: Mapped[int] = mapped_column(Integer, primary_key=True)
+    project_id: Mapped[int] = mapped_column(Integer, ForeignKey("project.id"), index=True)
     from_id: Mapped[int] = mapped_column(Integer, ForeignKey("entity.id", ondelete="CASCADE"))
     to_id: Mapped[Optional[int]] = mapped_column(
         Integer, ForeignKey("entity.id", ondelete="CASCADE"), nullable=True
@@ -155,13 +220,13 @@ class Relation(Base):
         Format: source/relation_type/target
         Example: "specs/search/implements/features/search-ui"
         """
+        # Only create permalinks when both source and target have permalinks
+        from_permalink = self.from_entity.permalink or self.from_entity.file_path
+
         if self.to_entity:
-            return generate_permalink(
-                f"{self.from_entity.permalink}/{self.relation_type}/{self.to_entity.permalink}"
-            )
-        return generate_permalink(
-            f"{self.from_entity.permalink}/{self.relation_type}/{self.to_name}"
-        )
+            to_permalink = self.to_entity.permalink or self.to_entity.file_path
+            return generate_permalink(f"{from_permalink}/{self.relation_type}/{to_permalink}")
+        return generate_permalink(f"{from_permalink}/{self.relation_type}/{self.to_name}")
 
     def __repr__(self) -> str:
-        return f"Relation(id={self.id}, from_id={self.from_id}, to_id={self.to_id}, to_name={self.to_name}, type='{self.relation_type}')"
+        return f"Relation(id={self.id}, from_id={self.from_id}, to_id={self.to_id}, to_name={self.to_name}, type='{self.relation_type}')"  # pragma: no cover

basic_memory/models/project.py

@@ -0,0 +1,93 @@
+"""Project model for Basic Memory."""
+
+import uuid
+from datetime import datetime, UTC
+from typing import Optional
+
+from sqlalchemy import (
+    Integer,
+    String,
+    Text,
+    Boolean,
+    DateTime,
+    Float,
+    Index,
+    event,
+)
+from sqlalchemy.orm import Mapped, mapped_column, relationship
+
+from basic_memory.models.base import Base
+from basic_memory.utils import generate_permalink
+
+
+class Project(Base):
+    """Project model for Basic Memory.
+
+    A project represents a collection of knowledge entities that are grouped together.
+    Projects are stored in the app-level database and provide context for all knowledge
+    operations.
+    """
+
+    __tablename__ = "project"
+    __table_args__ = (
+        # Regular indexes
+        Index("ix_project_name", "name", unique=True),
+        Index("ix_project_permalink", "permalink", unique=True),
+        Index("ix_project_external_id", "external_id", unique=True),
+        Index("ix_project_path", "path"),
+        Index("ix_project_created_at", "created_at"),
+        Index("ix_project_updated_at", "updated_at"),
+    )
+
+    # Core identity
+    id: Mapped[int] = mapped_column(Integer, primary_key=True)
+    # External UUID for API references - stable identifier that won't change
+    external_id: Mapped[str] = mapped_column(
+        String, unique=True, default=lambda: str(uuid.uuid4())
+    )
+    name: Mapped[str] = mapped_column(String, unique=True)
+    description: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
+
+    # URL-friendly identifier generated from name
+    permalink: Mapped[str] = mapped_column(String, unique=True)
+
+    # Filesystem path to project directory
+    path: Mapped[str] = mapped_column(String)
+
+    # Status flags
+    is_active: Mapped[bool] = mapped_column(Boolean, default=True)
+    is_default: Mapped[Optional[bool]] = mapped_column(Boolean, default=None, nullable=True)
+
+    # Timestamps
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), default=lambda: datetime.now(UTC)
+    )
+    updated_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True),
+        default=lambda: datetime.now(UTC),
+        onupdate=lambda: datetime.now(UTC),
+    )
+
+    # Sync optimization - scan watermark tracking
+    last_scan_timestamp: Mapped[Optional[float]] = mapped_column(Float, nullable=True)
+    last_file_count: Mapped[Optional[int]] = mapped_column(Integer, nullable=True)
+
+    # Define relationships to entities, observations, and relations
+    # These relationships will be established once we add project_id to those models
+    entities = relationship("Entity", back_populates="project", cascade="all, delete-orphan")
+
+    def __repr__(self) -> str:  # pragma: no cover
+        return f"Project(id={self.id}, external_id='{self.external_id}', name='{self.name}', permalink='{self.permalink}', path='{self.path}')"
+
+
+@event.listens_for(Project, "before_insert")
+@event.listens_for(Project, "before_update")
+def set_project_permalink(mapper, connection, project):
+    """Generate URL-friendly permalink for the project if needed.
+
+    This event listener ensures the permalink is always derived from the name,
+    even if the name changes.
+    """
+    # If the name changed or permalink is empty, regenerate permalink
+    if not project.permalink or project.permalink != generate_permalink(project.name):
+        project.permalink = generate_permalink(project.name)

basic_memory/models/search.py

@@ -1,32 +1,92 @@
-"""Search models and tables."""
+"""Search DDL statements for SQLite and Postgres.
+
+The search_index table is created via raw DDL, not ORM models, because:
+- SQLite uses FTS5 virtual tables (cannot be represented as ORM)
+- Postgres uses composite primary keys and generated tsvector columns
+- Both backends use raw SQL for all search operations via SearchIndexRow dataclass
+"""
 
 from sqlalchemy import DDL
 
-# Define FTS5 virtual table creation
+
+# Define Postgres search_index table with composite primary key and tsvector
+# This DDL matches the Alembic migration schema (314f1ea54dc4)
+# Used by tests to create the table without running full migrations
+# NOTE: Split into separate DDL statements because asyncpg doesn't support
+# multiple statements in a single execute call.
+CREATE_POSTGRES_SEARCH_INDEX_TABLE = DDL("""
+CREATE TABLE IF NOT EXISTS search_index (
+    id INTEGER NOT NULL,
+    project_id INTEGER NOT NULL,
+    title TEXT,
+    content_stems TEXT,
+    content_snippet TEXT,
+    permalink VARCHAR,
+    file_path VARCHAR,
+    type VARCHAR,
+    from_id INTEGER,
+    to_id INTEGER,
+    relation_type VARCHAR,
+    entity_id INTEGER,
+    category VARCHAR,
+    metadata JSONB,
+    created_at TIMESTAMP WITH TIME ZONE,
+    updated_at TIMESTAMP WITH TIME ZONE,
+    textsearchable_index_col tsvector GENERATED ALWAYS AS (
+        to_tsvector('english', coalesce(title, '') || ' ' || coalesce(content_stems, ''))
+    ) STORED,
+    PRIMARY KEY (id, type, project_id),
+    FOREIGN KEY (project_id) REFERENCES project(id) ON DELETE CASCADE
+)
+""")
+
+CREATE_POSTGRES_SEARCH_INDEX_FTS = DDL("""
+CREATE INDEX IF NOT EXISTS idx_search_index_fts ON search_index USING gin(textsearchable_index_col)
+""")
+
+CREATE_POSTGRES_SEARCH_INDEX_METADATA = DDL("""
+CREATE INDEX IF NOT EXISTS idx_search_index_metadata_gin ON search_index USING gin(metadata jsonb_path_ops)
+""")
+
+# Partial unique index on (permalink, project_id) for non-null permalinks
+# This prevents duplicate permalinks per project and is used by upsert operations
+# in PostgresSearchRepository to handle race conditions during parallel indexing
+CREATE_POSTGRES_SEARCH_INDEX_PERMALINK = DDL("""
+CREATE UNIQUE INDEX IF NOT EXISTS uix_search_index_permalink_project
+ON search_index (permalink, project_id)
+WHERE permalink IS NOT NULL
+""")
+
+# Define FTS5 virtual table creation for SQLite only
+# This DDL is executed separately for SQLite databases
 CREATE_SEARCH_INDEX = DDL("""
 CREATE VIRTUAL TABLE IF NOT EXISTS search_index USING fts5(
     -- Core entity fields
     id UNINDEXED,          -- Row ID
     title,                 -- Title for searching
-    content,               -- Main searchable content
+    content_stems,         -- Main searchable content split into stems
+    content_snippet,       -- File content snippet for display
     permalink,             -- Stable identifier (now indexed for path search)
     file_path UNINDEXED,   -- Physical location
     type UNINDEXED,        -- entity/relation/observation
-    
-    -- Relation fields 
+
+    -- Project context
+    project_id UNINDEXED,  -- Project identifier
+
+    -- Relation fields
     from_id UNINDEXED,     -- Source entity
     to_id UNINDEXED,       -- Target entity
     relation_type UNINDEXED, -- Type of relation
-    
+
     -- Observation fields
     entity_id UNINDEXED,   -- Parent entity
     category UNINDEXED,    -- Observation category
-    
+
     -- Common fields
     metadata UNINDEXED,    -- JSON metadata
     created_at UNINDEXED,  -- Creation timestamp
     updated_at UNINDEXED,  -- Last update
-    
+
     -- Configuration
     tokenize='unicode61 tokenchars 0x2F',  -- Hex code for /
     prefix='1,2,3,4'                    -- Support longer prefixes for paths