basic-memory 0.0.0__py3-none-any.whl

basic_memory/models/knowledge.py

@@ -0,0 +1,204 @@
+"""Knowledge graph models."""
+
+import re
+from datetime import datetime
+from typing import Optional
+
+from sqlalchemy import (
+    Integer,
+    String,
+    Text,
+    ForeignKey,
+    UniqueConstraint,
+    DateTime,
+    Index,
+    JSON,
+)
+from sqlalchemy.orm import Mapped, mapped_column, relationship, validates
+
+from basic_memory.models.base import Base
+from enum import Enum
+
+from basic_memory.utils import generate_permalink
+
+
+class Entity(Base):
+    """
+    Core entity in the knowledge graph.
+
+    Entities represent semantic nodes maintained by the AI layer. Each entity:
+    - Has a unique numeric ID (database-generated)
+    - Maps to a file on disk
+    - Maintains a checksum for change detection
+    - Tracks both source file and semantic properties
+    """
+
+    __tablename__ = "entity"
+    __table_args__ = (
+        UniqueConstraint("permalink", name="uix_entity_permalink"),  # Make permalink unique
+        Index("ix_entity_type", "entity_type"),
+        Index("ix_entity_title", "title"),
+        Index("ix_entity_created_at", "created_at"),  # For timeline queries
+        Index("ix_entity_updated_at", "updated_at"),  # For timeline queries
+    )
+
+    # Core identity
+    id: Mapped[int] = mapped_column(Integer, primary_key=True)
+    title: Mapped[str] = mapped_column(String)
+    entity_type: Mapped[str] = mapped_column(String)
+    entity_metadata: Mapped[Optional[dict]] = mapped_column(JSON, nullable=True)
+    content_type: Mapped[str] = mapped_column(String)
+
+    # Normalized path for URIs
+    permalink: Mapped[str] = mapped_column(String, unique=True, index=True)
+    # Actual filesystem relative path
+    file_path: Mapped[str] = mapped_column(String, unique=True, index=True)
+    # checksum of file
+    checksum: Mapped[Optional[str]] = mapped_column(String, nullable=True)
+
+    # Metadata and tracking
+    created_at: Mapped[datetime] = mapped_column(DateTime)
+    updated_at: Mapped[datetime] = mapped_column(DateTime)
+
+    # Relationships
+    observations = relationship(
+        "Observation", back_populates="entity", cascade="all, delete-orphan"
+    )
+    outgoing_relations = relationship(
+        "Relation",
+        back_populates="from_entity",
+        foreign_keys="[Relation.from_id]",
+        cascade="all, delete-orphan",
+    )
+    incoming_relations = relationship(
+        "Relation",
+        back_populates="to_entity",
+        foreign_keys="[Relation.to_id]",
+        cascade="all, delete-orphan",
+    )
+
+    @property
+    def relations(self):
+        return self.incoming_relations + self.outgoing_relations
+
+    @validates("permalink")
+    def validate_permalink(self, key, value):
+        """Validate permalink format.
+
+        Requirements:
+        1. Must be valid URI path component
+        2. Only lowercase letters, numbers, and hyphens (no underscores)
+        3. Path segments separated by forward slashes
+        4. No leading/trailing hyphens in segments
+        """
+        if not value:
+            raise ValueError("Permalink must not be None")
+
+        if not re.match(r"^[a-z0-9][a-z0-9\-/]*[a-z0-9]$", value):
+            raise ValueError(
+                f"Invalid permalink format: {value}. "
+                "Use only lowercase letters, numbers, and hyphens."
+            )
+        return value
+
+    def __repr__(self) -> str:
+        return f"Entity(id={self.id}, name='{self.title}', type='{self.entity_type}'"
+
+
+class ObservationCategory(str, Enum):
+    TECH = "tech"
+    DESIGN = "design"
+    FEATURE = "feature"
+    NOTE = "note"
+    ISSUE = "issue"
+    TODO = "todo"
+
+
+class Observation(Base):
+    """
+    An observation about an entity.
+
+    Observations are atomic facts or notes about an entity.
+    """
+
+    __tablename__ = "observation"
+    __table_args__ = (
+        Index("ix_observation_entity_id", "entity_id"),  # Add FK index
+        Index("ix_observation_category", "category"),  # Add category index
+    )
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True)
+    entity_id: Mapped[int] = mapped_column(Integer, ForeignKey("entity.id", ondelete="CASCADE"))
+    content: Mapped[str] = mapped_column(Text)
+    category: Mapped[str] = mapped_column(
+        String,
+        nullable=False,
+        default=ObservationCategory.NOTE.value,
+        server_default=ObservationCategory.NOTE.value,
+    )
+    context: Mapped[str] = mapped_column(Text, nullable=True)
+    tags: Mapped[Optional[list[str]]] = mapped_column(
+        JSON, nullable=True, default=list, server_default="[]"
+    )
+
+    # Relationships
+    entity = relationship("Entity", back_populates="observations")
+
+    @property
+    def permalink(self) -> str:
+        """
+        Create synthetic permalink for the observation
+            We can construct these because observations are always
+            defined in and owned by a single entity
+        """
+        return generate_permalink(
+            f"{self.entity.permalink}/observations/{self.category}/{self.content}"
+        )
+
+    def __repr__(self) -> str:
+        return f"Observation(id={self.id}, entity_id={self.entity_id}, content='{self.content}')"
+
+
+class Relation(Base):
+    """
+    A directed relation between two entities.
+    """
+
+    __tablename__ = "relation"
+    __table_args__ = (
+        UniqueConstraint("from_id", "to_id", "relation_type", name="uix_relation"),
+        Index("ix_relation_type", "relation_type"),
+        Index("ix_relation_from_id", "from_id"),  # Add FK indexes
+        Index("ix_relation_to_id", "to_id"),
+    )
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True)
+    from_id: Mapped[int] = mapped_column(Integer, ForeignKey("entity.id", ondelete="CASCADE"))
+    to_id: Mapped[int] = mapped_column(
+        Integer, ForeignKey("entity.id", ondelete="CASCADE"), nullable=True
+    )
+    to_name: Mapped[str] = mapped_column(String)
+    relation_type: Mapped[str] = mapped_column(String)
+    context: Mapped[str] = mapped_column(Text, nullable=True)
+
+    # Relationships
+    from_entity = relationship(
+        "Entity", foreign_keys=[from_id], back_populates="outgoing_relations"
+    )
+    to_entity = relationship("Entity", foreign_keys=[to_id], back_populates="incoming_relations")
+
+    @property
+    def permalink(self) -> str:
+        """Create relation permalink showing the semantic connection:
+        source/relation_type/target
+        e.g., "specs/search/implements/features/search-ui"
+        """
+
+        return generate_permalink(
+            f"{self.from_entity.permalink}/{self.relation_type}/{self.to_entity.permalink}"
+            if self.to_entity
+            else f"{self.from_entity.permalink}/{self.relation_type}/{self.to_name}"
+        )
+
+    def __repr__(self) -> str:
+        return f"Relation(id={self.id}, from_id={self.from_id}, to_id={self.to_id}, to_name={self.to_name}, type='{self.relation_type}')"

basic_memory/models/search.py

@@ -0,0 +1,34 @@
+"""Search models and tables."""
+
+from sqlalchemy import DDL
+
+# Define FTS5 virtual table creation
+CREATE_SEARCH_INDEX = DDL("""
+CREATE VIRTUAL TABLE IF NOT EXISTS search_index USING fts5(
+    -- Core entity fields
+    id UNINDEXED,          -- Row ID
+    title,                 -- Title for searching
+    content,               -- Main searchable content
+    permalink,             -- Stable identifier (now indexed for path search)
+    file_path UNINDEXED,   -- Physical location
+    type UNINDEXED,        -- entity/relation/observation
+    
+    -- Relation fields 
+    from_id UNINDEXED,     -- Source entity
+    to_id UNINDEXED,       -- Target entity
+    relation_type UNINDEXED, -- Type of relation
+    
+    -- Observation fields
+    entity_id UNINDEXED,   -- Parent entity
+    category UNINDEXED,    -- Observation category
+    
+    -- Common fields
+    metadata UNINDEXED,    -- JSON metadata
+    created_at UNINDEXED,  -- Creation timestamp
+    updated_at UNINDEXED,  -- Last update
+    
+    -- Configuration
+    tokenize='unicode61 tokenchars 0x2F',  -- Hex code for /
+    prefix='1,2,3,4'                    -- Support longer prefixes for paths
+);
+""")

basic_memory/repository/__init__.py

@@ -0,0 +1,7 @@
+from .entity_repository import EntityRepository
+from .observation_repository import ObservationRepository
+from .relation_repository import RelationRepository
+
+__all__ = ["EntityRepository", "ObservationRepository", "RelationRepository", ]
+
+

basic_memory/repository/entity_repository.py

@@ -0,0 +1,156 @@
+"""Repository for managing entities in the knowledge graph."""
+
+from typing import List, Optional, Sequence
+
+from sqlalchemy import select, or_, asc
+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
+from sqlalchemy.orm import selectinload
+from sqlalchemy.orm.interfaces import LoaderOption
+
+from basic_memory.models.knowledge import Entity, Observation, Relation
+from basic_memory.repository.repository import Repository
+
+
+class EntityRepository(Repository[Entity]):
+    """Repository for Entity model."""
+
+    def __init__(self, session_maker: async_sessionmaker[AsyncSession]):
+        """Initialize with session maker."""
+        super().__init__(session_maker, Entity)
+
+    async def get_by_permalink(self, permalink: str) -> Optional[Entity]:
+        """Get entity by permalink."""
+        query = self.select().where(Entity.permalink == permalink).options(*self.get_load_options())
+        return await self.find_one(query)
+
+    async def get_by_title(self, title: str) -> Optional[Entity]:
+        """Get entity by title."""
+        query = self.select().where(Entity.title == title).options(*self.get_load_options())
+        return await self.find_one(query)
+
+    async def get_by_file_path(self, file_path: str) -> Optional[Entity]:
+        """Get entity by file_path."""
+        query = self.select().where(Entity.file_path == file_path).options(*self.get_load_options())
+        return await self.find_one(query)
+
+    async def list_entities(
+        self,
+        entity_type: Optional[str] = None,
+        sort_by: Optional[str] = "updated_at",
+        include_related: bool = False,
+    ) -> Sequence[Entity]:
+        """List all entities, optionally filtered by type and sorted."""
+        query = self.select()
+
+        # Always load base relations
+        query = query.options(*self.get_load_options())
+
+        # Apply filters
+        if entity_type:
+            # When include_related is True, get both:
+            # 1. Entities of the requested type
+            # 2. Entities that have relations with entities of the requested type
+            if include_related:
+                query = query.where(
+                    or_(
+                        Entity.entity_type == entity_type,
+                        Entity.outgoing_relations.any(
+                            Relation.to_entity.has(entity_type=entity_type)
+                        ),
+                        Entity.incoming_relations.any(
+                            Relation.from_entity.has(entity_type=entity_type)
+                        ),
+                    )
+                )
+            else:
+                query = query.where(Entity.entity_type == entity_type)
+
+        # Apply sorting
+        if sort_by:
+            sort_field = getattr(Entity, sort_by, Entity.updated_at)
+            query = query.order_by(asc(sort_field))
+
+        result = await self.execute_query(query)
+        return list(result.scalars().all())
+
+    async def get_entity_types(self) -> List[str]:
+        """Get list of distinct entity types."""
+        query = select(Entity.entity_type).distinct()
+
+        result = await self.execute_query(query, use_query_options=False)
+        return list(result.scalars().all())
+
+    async def search(self, query_str: str) -> List[Entity]:
+        """
+        Search for entities.
+
+        Searches across:
+        - Entity names
+        - Entity types
+        - Entity descriptions
+        - Associated Observations content
+        """
+        search_term = f"%{query_str}%"
+        query = (
+            self.select()
+            .where(
+                or_(
+                    Entity.title.ilike(search_term),
+                    Entity.entity_type.ilike(search_term),
+                    Entity.summary.ilike(search_term),
+                    Entity.observations.any(Observation.content.ilike(search_term)),
+                )
+            )
+            .options(*self.get_load_options())
+        )
+        result = await self.execute_query(query)
+        return list(result.scalars().all())
+
+    async def delete_entities_by_doc_id(self, doc_id: int) -> bool:
+        """Delete all entities associated with a document."""
+        return await self.delete_by_fields(doc_id=doc_id)
+
+    async def delete_by_file_path(self, file_path: str) -> bool:
+        """Delete entity with the provided file_path."""
+        return await self.delete_by_fields(file_path=file_path)
+
+    def get_load_options(self) -> List[LoaderOption]:
+        return [
+            selectinload(Entity.observations).selectinload(Observation.entity),
+            # Load from_relations and both entities for each relation
+            selectinload(Entity.outgoing_relations).selectinload(Relation.from_entity),
+            selectinload(Entity.outgoing_relations).selectinload(Relation.to_entity),
+            # Load to_relations and both entities for each relation
+            selectinload(Entity.incoming_relations).selectinload(Relation.from_entity),
+            selectinload(Entity.incoming_relations).selectinload(Relation.to_entity),
+        ]
+
+    async def find_by_permalinks(self, permalinks: List[str]) -> Sequence[Entity]:
+        """Find multiple entities by their permalink."""
+
+        # Handle empty input explicitly
+        if not permalinks:
+            return []
+
+        # Use existing select pattern
+        query = (
+            self.select().options(*self.get_load_options()).where(Entity.permalink.in_(permalinks))
+        )
+
+        result = await self.execute_query(query)
+        return list(result.scalars().all())
+
+    async def delete_by_permalinks(self, permalinks: List[str]) -> int:
+        """Delete multiple entities by permalink."""
+
+        # Handle empty input explicitly
+        if not permalinks:
+            return 0
+
+        # Find matching entities
+        entities = await self.find_by_permalinks(permalinks)
+        if not entities:
+            return 0
+
+        # Use existing delete_by_ids
+        return await self.delete_by_ids([entity.id for entity in entities])

basic_memory/repository/observation_repository.py

@@ -0,0 +1,40 @@
+"""Repository for managing Observation objects."""
+
+from typing import Sequence
+
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import async_sessionmaker
+
+from basic_memory.models import Observation
+from basic_memory.repository.repository import Repository
+
+
+class ObservationRepository(Repository[Observation]):
+    """Repository for Observation model with memory-specific operations."""
+
+    def __init__(self, session_maker: async_sessionmaker):
+        super().__init__(session_maker, Observation)
+
+    async def find_by_entity(self, entity_id: int) -> Sequence[Observation]:
+        """Find all observations for a specific entity."""
+        query = select(Observation).filter(Observation.entity_id == entity_id)
+        result = await self.execute_query(query)
+        return result.scalars().all()
+
+    async def find_by_context(self, context: str) -> Sequence[Observation]:
+        """Find observations with a specific context."""
+        query = select(Observation).filter(Observation.context == context)
+        result = await self.execute_query(query)
+        return result.scalars().all()
+
+    async def find_by_category(self, category: str) -> Sequence[Observation]:
+        """Find observations with a specific context."""
+        query = select(Observation).filter(Observation.category == category)
+        result = await self.execute_query(query)
+        return result.scalars().all()
+
+    async def observation_categories(self) -> Sequence[str]:
+        """Return a list of all observation categories."""
+        query = select(Observation.category).distinct()
+        result = await self.execute_query(query, use_query_options=False)
+        return result.scalars().all()

basic_memory/repository/relation_repository.py

@@ -0,0 +1,78 @@
+"""Repository for managing Relation objects."""
+
+from sqlalchemy import and_, delete
+from typing import Sequence, List, Optional
+
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import async_sessionmaker
+from sqlalchemy.orm import selectinload, aliased
+from sqlalchemy.orm.interfaces import LoaderOption
+
+from basic_memory import db
+from basic_memory.models import Relation, Entity
+from basic_memory.repository.repository import Repository
+
+
+class RelationRepository(Repository[Relation]):
+    """Repository for Relation model with memory-specific operations."""
+
+    def __init__(self, session_maker: async_sessionmaker):
+        super().__init__(session_maker, Relation)
+
+    async def find_relation(
+        self, from_permalink: str, to_permalink: str, relation_type: str
+    ) -> Optional[Relation]:
+        """Find a relation by its from and to path IDs."""
+        from_entity = aliased(Entity)
+        to_entity = aliased(Entity)
+
+        query = (
+            select(Relation)
+            .join(from_entity, Relation.from_id == from_entity.id)
+            .join(to_entity, Relation.to_id == to_entity.id)
+            .where(
+                and_(
+                    from_entity.permalink == from_permalink,
+                    to_entity.permalink == to_permalink,
+                    Relation.relation_type == relation_type,
+                )
+            )
+        )
+        return await self.find_one(query)
+
+    async def find_by_entity(self, from_entity_id: int) -> Sequence[Relation]:
+        """Find all relations from a specific entity."""
+        query = select(Relation).filter(Relation.from_id == from_entity_id)
+        result = await self.execute_query(query)
+        return result.scalars().all()
+
+    async def find_by_entities(self, from_id: int, to_id: int) -> Sequence[Relation]:
+        """Find all relations between two entities."""
+        query = select(Relation).where((Relation.from_id == from_id) & (Relation.to_id == to_id))
+        result = await self.execute_query(query)
+        return result.scalars().all()
+
+    async def find_by_type(self, relation_type: str) -> Sequence[Relation]:
+        """Find all relations of a specific type."""
+        query = select(Relation).filter(Relation.relation_type == relation_type)
+        result = await self.execute_query(query)
+        return result.scalars().all()
+
+    async def delete_outgoing_relations_from_entity(self, entity_id: int) -> None:
+        """Delete outgoing relations for an entity.
+
+        Only deletes relations where this entity is the source (from_id),
+        as these are the ones owned by this entity's markdown file.
+        """
+        async with db.scoped_session(self.session_maker) as session:
+            await session.execute(delete(Relation).where(Relation.from_id == entity_id))
+
+    async def find_unresolved_relations(self) -> Sequence[Relation]:
+        """Find all unresolved relations, where to_id is null."""
+        query = select(Relation).filter(Relation.to_id.is_(None))
+        result = await self.execute_query(query)
+        return result.scalars().all()
+
+
+    def get_load_options(self) -> List[LoaderOption]:
+        return [selectinload(Relation.from_entity), selectinload(Relation.to_entity)]