basic-memory 0.6.0__py3-none-any.whl → 0.8.0__py3-none-any.whl

basic_memory/mcp/tools/notes.py

@@ -7,6 +7,7 @@ while leveraging the underlying knowledge graph structure.
 from typing import Optional, List
 
 from loguru import logger
+import logfire
 
 from basic_memory.mcp.server import mcp
 from basic_memory.mcp.async_client import client
@@ -60,75 +61,61 @@ async def write_note(
         - Observation counts by category
         - Relation counts (resolved/unresolved)
         - Tags if present
-
-    Examples:
-        write_note(
-            title="Search Implementation",
-            content="# Search Component\\n\\n"
-                   "Implementation of the search feature, building on [[Core Search]].\\n\\n"
-                   "## Observations\\n"
-                   "- [tech] Using FTS5 for full-text search #implementation\\n"
-                   "- [design] Need pagination support #todo\\n\\n"
-                   "## Relations\\n"
-                   "- implements [[Search Spec]]\\n"
-                   "- depends_on [[Database Schema]]",
-            folder="docs/components"
-        )
     """
-    logger.info(f"Writing note folder:'{folder}' title: '{title}'")
-
-    # Create the entity request
-    metadata = {"tags": [f"#{tag}" for tag in tags]} if tags else None
-    entity = Entity(
-        title=title,
-        folder=folder,
-        entity_type="note",
-        content_type="text/markdown",
-        content=content,
-        entity_metadata=metadata,
-    )
-
-    # Create or update via knowledge API
-    logger.info(f"Creating {entity.permalink}")
-    url = f"/knowledge/entities/{entity.permalink}"
-    response = await call_put(client, url, json=entity.model_dump())
-    result = EntityResponse.model_validate(response.json())
-
-    # Format semantic summary based on status code
-    action = "Created" if response.status_code == 201 else "Updated"
-    assert result.checksum is not None
-    summary = [
-        f"# {action} {result.file_path} ({result.checksum[:8]})",
-        f"permalink: {result.permalink}",
-    ]
-
-    if result.observations:
-        categories = {}
-        for obs in result.observations:
-            categories[obs.category] = categories.get(obs.category, 0) + 1
-
-        summary.append("\n## Observations")
-        for category, count in sorted(categories.items()):
-            summary.append(f"- {category}: {count}")
-
-    if result.relations:
-        unresolved = sum(1 for r in result.relations if not r.to_id)
-        resolved = len(result.relations) - unresolved
-
-        summary.append("\n## Relations")
-        summary.append(f"- Resolved: {resolved}")
-        if unresolved:
-            summary.append(f"- Unresolved: {unresolved}")
-            summary.append("\nUnresolved relations will be retried on next sync.")
-
-    if tags:
-        summary.append(f"\n## Tags\n- {', '.join(tags)}")
-
-    return "\n".join(summary)
+    with logfire.span("Writing note", title=title, folder=folder):  # pyright: ignore [reportGeneralTypeIssues]
+        logger.info(f"Writing note folder:'{folder}' title: '{title}'")
+
+        # Create the entity request
+        metadata = {"tags": [f"#{tag}" for tag in tags]} if tags else None
+        entity = Entity(
+            title=title,
+            folder=folder,
+            entity_type="note",
+            content_type="text/markdown",
+            content=content,
+            entity_metadata=metadata,
+        )
+
+        # Create or update via knowledge API
+        logger.info(f"Creating {entity.permalink}")
+        url = f"/knowledge/entities/{entity.permalink}"
+        response = await call_put(client, url, json=entity.model_dump())
+        result = EntityResponse.model_validate(response.json())
+
+        # Format semantic summary based on status code
+        action = "Created" if response.status_code == 201 else "Updated"
+        summary = [
+            f"# {action} {result.file_path} ({result.checksum[:8] if result.checksum else 'unknown'})",
+            f"permalink: {result.permalink}",
+        ]
+
+        if result.observations:
+            categories = {}
+            for obs in result.observations:
+                categories[obs.category] = categories.get(obs.category, 0) + 1
+
+            summary.append("\n## Observations")
+            for category, count in sorted(categories.items()):
+                summary.append(f"- {category}: {count}")
+
+        if result.relations:
+            unresolved = sum(1 for r in result.relations if not r.to_id)
+            resolved = len(result.relations) - unresolved
+
+            summary.append("\n## Relations")
+            summary.append(f"- Resolved: {resolved}")
+            if unresolved:
+                summary.append(f"- Unresolved: {unresolved}")
+                summary.append("\nUnresolved relations will be retried on next sync.")
+
+        if tags:
+            summary.append(f"\n## Tags\n- {', '.join(tags)}")
+
+        return "\n".join(summary)
 
 
 @mcp.tool(description="Read note content by title, permalink, relation, or pattern")
-async def read_note(identifier: str) -> str:
+async def read_note(identifier: str, page: int = 1, page_size: int = 10) -> str:
     """Get note content in unified diff format.
 
     The content is returned in a unified diff inspired format:
@@ -146,6 +133,8 @@ async def read_note(identifier: str) -> str:
             - Note permalink ("docs/example")
             - Relation path ("docs/example/depends-on/other-doc")
             - Pattern match ("docs/*-architecture")
+        page: the page number of results to return (default 1)
+        page_size: the number of results to return per page (default 10)
 
     Returns:
         Document content in unified diff format. For single documents, returns
@@ -180,10 +169,13 @@ async def read_note(identifier: str) -> str:
     - Last modified timestamp
     - Content checksum
     """
-    logger.info(f"Reading note {identifier}")
-    url = memory_url_path(identifier)
-    response = await call_get(client, f"/resource/{url}")
-    return response.text
+    with logfire.span("Reading note", identifier=identifier):  # pyright: ignore [reportGeneralTypeIssues]
+        logger.info(f"Reading note {identifier}")
+        url = memory_url_path(identifier)
+        response = await call_get(
+            client, f"/resource/{url}", params={"page": page, "page_size": page_size}
+        )
+        return response.text
 
 
 @mcp.tool(description="Delete a note by title or permalink")
@@ -203,6 +195,7 @@ async def delete_note(identifier: str) -> bool:
         # Delete by permalink
         delete_note("notes/project-planning")
     """
-    response = await call_delete(client, f"/knowledge/entities/{identifier}")
-    result = DeleteEntitiesResponse.model_validate(response.json())
-    return result.deleted
+    with logfire.span("Deleting note", identifier=identifier):  # pyright: ignore [reportGeneralTypeIssues]
+        response = await call_delete(client, f"/knowledge/entities/{identifier}")
+        result = DeleteEntitiesResponse.model_validate(response.json())
+        return result.deleted

basic_memory/mcp/tools/resource.py

@@ -0,0 +1,192 @@
+from loguru import logger
+
+from basic_memory.mcp.server import mcp
+from basic_memory.mcp.async_client import client
+from basic_memory.mcp.tools.utils import call_get
+from basic_memory.schemas.memory import memory_url_path
+
+import base64
+import io
+from PIL import Image as PILImage
+
+
+def calculate_target_params(content_length):
+    """Calculate initial quality and size based on input file size"""
+    target_size = 350000  # Reduced target for more safety margin
+    ratio = content_length / target_size
+
+    logger.debug(
+        "Calculating target parameters",
+        content_length=content_length,
+        ratio=ratio,
+        target_size=target_size,
+    )
+
+    if ratio > 4:
+        # Very large images - start very aggressive
+        return 50, 600  # Lower initial quality and size
+    elif ratio > 2:
+        return 60, 800
+    else:
+        return 70, 1000
+
+
+def resize_image(img, max_size):
+    """Resize image maintaining aspect ratio"""
+    original_dimensions = {"width": img.width, "height": img.height}
+
+    if img.width > max_size or img.height > max_size:
+        ratio = min(max_size / img.width, max_size / img.height)
+        new_size = (int(img.width * ratio), int(img.height * ratio))
+        logger.debug("Resizing image", original=original_dimensions, target=new_size, ratio=ratio)
+        return img.resize(new_size, PILImage.Resampling.LANCZOS)
+
+    logger.debug("No resize needed", dimensions=original_dimensions)
+    return img
+
+
+def optimize_image(img, content_length, max_output_bytes=350000):
+    """Iteratively optimize image with aggressive size reduction"""
+    stats = {
+        "dimensions": {"width": img.width, "height": img.height},
+        "mode": img.mode,
+        "estimated_memory": (img.width * img.height * len(img.getbands())),
+    }
+
+    initial_quality, initial_size = calculate_target_params(content_length)
+
+    logger.debug(
+        "Starting optimization",
+        image_stats=stats,
+        content_length=content_length,
+        initial_quality=initial_quality,
+        initial_size=initial_size,
+        max_output_bytes=max_output_bytes,
+    )
+
+    quality = initial_quality
+    size = initial_size
+
+    # Convert to RGB if needed
+    if img.mode in ("RGBA", "LA") or (img.mode == "P" and "transparency" in img.info):
+        img = img.convert("RGB")
+        logger.debug("Converted to RGB mode")
+
+    iteration = 0
+    min_size = 300  # Absolute minimum size
+    min_quality = 20  # Absolute minimum quality
+
+    while True:
+        iteration += 1
+        buf = io.BytesIO()
+        resized = resize_image(img, size)
+
+        resized.save(
+            buf,
+            format="JPEG",
+            quality=quality,
+            optimize=True,
+            progressive=True,
+            subsampling="4:2:0",
+        )
+
+        output_size = buf.getbuffer().nbytes
+        reduction_ratio = output_size / content_length
+
+        logger.debug(
+            "Optimization attempt",
+            iteration=iteration,
+            quality=quality,
+            size=size,
+            output_bytes=output_size,
+            target_bytes=max_output_bytes,
+            reduction_ratio=f"{reduction_ratio:.2f}",
+        )
+
+        if output_size < max_output_bytes:
+            logger.info(
+                "Image optimization complete",
+                final_size=output_size,
+                quality=quality,
+                dimensions={"width": resized.width, "height": resized.height},
+                reduction_ratio=f"{reduction_ratio:.2f}",
+            )
+            return buf.getvalue()
+
+        # Very aggressive reduction for large files
+        if content_length > 2000000:  # 2MB+   # pragma: no cover
+            quality = max(min_quality, quality - 20)
+            size = max(min_size, int(size * 0.6))
+        elif content_length > 1000000:  # 1MB+ # pragma: no cover
+            quality = max(min_quality, quality - 15)
+            size = max(min_size, int(size * 0.7))
+        else:
+            quality = max(min_quality, quality - 10)  # pragma: no cover
+            size = max(min_size, int(size * 0.8))  # pragma: no cover
+
+        logger.debug("Reducing parameters", new_quality=quality, new_size=size)  # pragma: no cover
+
+        # If we've hit minimum values and still too big
+        if quality <= min_quality and size <= min_size:  # pragma: no cover
+            logger.warning(
+                "Reached minimum parameters",
+                final_size=output_size,
+                over_limit_by=output_size - max_output_bytes,
+            )
+            return buf.getvalue()
+
+
+@mcp.tool(description="Read a single file's content by path or permalink")
+async def read_resource(path: str) -> dict:
+    """Get a file's raw content."""
+    logger.info("Reading resource", path=path)
+
+    url = memory_url_path(path)
+    response = await call_get(client, f"/resource/{url}")
+    content_type = response.headers.get("content-type", "application/octet-stream")
+    content_length = int(response.headers.get("content-length", 0))
+
+    logger.debug("Resource metadata", content_type=content_type, size=content_length, path=path)
+
+    # Handle text or json
+    if content_type.startswith("text/") or content_type == "application/json":
+        logger.debug("Processing text resource")
+        return {
+            "type": "text",
+            "text": response.text,
+            "content_type": content_type,
+            "encoding": "utf-8",
+        }
+
+    # Handle images
+    elif content_type.startswith("image/"):
+        logger.debug("Processing image")
+        img = PILImage.open(io.BytesIO(response.content))
+        img_bytes = optimize_image(img, content_length)
+
+        return {
+            "type": "image",
+            "source": {
+                "type": "base64",
+                "media_type": "image/jpeg",
+                "data": base64.b64encode(img_bytes).decode("utf-8"),
+            },
+        }
+
+    # Handle other file types
+    else:
+        logger.debug(f"Processing binary resource content_type {content_type}")
+        if content_length > 350000:
+            logger.warning("Document too large for response", size=content_length)
+            return {
+                "type": "error",
+                "error": f"Document size {content_length} bytes exceeds maximum allowed size",
+            }
+        return {
+            "type": "document",
+            "source": {
+                "type": "base64",
+                "media_type": content_type,
+                "data": base64.b64encode(response.content).decode("utf-8"),
+            },
+        }

basic_memory/mcp/tools/search.py

@@ -1,5 +1,6 @@
 """Search tools for Basic Memory MCP server."""
 
+import logfire
 from loguru import logger
 
 from basic_memory.mcp.server import mcp
@@ -11,7 +12,7 @@ from basic_memory.mcp.async_client import client
 @mcp.tool(
     description="Search across all content in basic-memory, including documents and entities",
 )
-async def search(query: SearchQuery) -> SearchResponse:
+async def search(query: SearchQuery, page: int = 1, page_size: int = 10) -> SearchResponse:
     """Search across all content in basic-memory.
 
     Args:
@@ -20,10 +21,18 @@ async def search(query: SearchQuery) -> SearchResponse:
             - types: Optional list of content types to search ("document" or "entity")
             - entity_types: Optional list of entity types to filter by
             - after_date: Optional date filter for recent content
+        page: the page number of results to return (default 1)
+        page_size: the number of results to return per page (default 10)
 
     Returns:
         SearchResponse with search results and metadata
     """
-    logger.info(f"Searching for {query.text}")
-    response = await call_post(client, "/search/", json=query.model_dump())
-    return SearchResponse.model_validate(response.json())
+    with logfire.span("Searching for {query}", query=query):  # pyright: ignore [reportGeneralTypeIssues]
+        logger.info(f"Searching for {query}")
+        response = await call_post(
+            client,
+            "/search/",
+            json=query.model_dump(),
+            params={"page": page, "page_size": page_size},
+        )
+        return SearchResponse.model_validate(response.json())

basic_memory/mcp/tools/utils.py

@@ -44,7 +44,7 @@ async def call_get(
         response.raise_for_status()
         return response
     except HTTPStatusError as e:
-        logger.error(f"Error calling GET {url}: {e}")
+        logger.exception(f"Error calling GET {url}: {e}")
         raise ToolError(f"Error calling tool: {e}.") from e
 
 
@@ -79,6 +79,7 @@ async def call_put(
             timeout=timeout,
             extensions=extensions,
         )
+        logger.debug(response)
         response.raise_for_status()
         return response
     except HTTPStatusError as e:

basic_memory/models/knowledge.py

@@ -12,6 +12,7 @@ from sqlalchemy import (
     DateTime,
     Index,
     JSON,
+    text,
 )
 from sqlalchemy.orm import Mapped, mapped_column, relationship
 
@@ -32,11 +33,18 @@ class Entity(Base):
 
     __tablename__ = "entity"
     __table_args__ = (
-        UniqueConstraint("permalink", name="uix_entity_permalink"),  # Make permalink unique
+        # Regular indexes
         Index("ix_entity_type", "entity_type"),
         Index("ix_entity_title", "title"),
         Index("ix_entity_created_at", "created_at"),  # For timeline queries
         Index("ix_entity_updated_at", "updated_at"),  # For timeline queries
+        # Unique index only for markdown files with non-null permalinks
+        Index(
+            "uix_entity_permalink",
+            "permalink",
+            unique=True,
+            sqlite_where=text("content_type = 'text/markdown' AND permalink IS NOT NULL"),
+        ),
     )
 
     # Core identity
@@ -46,8 +54,8 @@ class Entity(Base):
     entity_metadata: Mapped[Optional[dict]] = mapped_column(JSON, nullable=True)
     content_type: Mapped[str] = mapped_column(String)
 
-    # Normalized path for URIs
-    permalink: Mapped[str] = mapped_column(String, unique=True, index=True)
+    # Normalized path for URIs - required for markdown files only
+    permalink: Mapped[Optional[str]] = mapped_column(String, nullable=True, index=True)
     # Actual filesystem relative path
     file_path: Mapped[str] = mapped_column(String, unique=True, index=True)
     # checksum of file
@@ -79,6 +87,11 @@ class Entity(Base):
         """Get all relations (incoming and outgoing) for this entity."""
         return self.incoming_relations + self.outgoing_relations
 
+    @property
+    def is_markdown(self):
+        """Check if the entity is a markdown file."""
+        return self.content_type == "text/markdown"
+
     def __repr__(self) -> str:
         return f"Entity(id={self.id}, name='{self.title}', type='{self.entity_type}'"
 
@@ -127,7 +140,10 @@ class Relation(Base):
 
     __tablename__ = "relation"
     __table_args__ = (
-        UniqueConstraint("from_id", "to_id", "relation_type", name="uix_relation"),
+        UniqueConstraint("from_id", "to_id", "relation_type", name="uix_relation_from_id_to_id"),
+        UniqueConstraint(
+            "from_id", "to_name", "relation_type", name="uix_relation_from_id_to_name"
+        ),
         Index("ix_relation_type", "relation_type"),
         Index("ix_relation_from_id", "from_id"),  # Add FK indexes
         Index("ix_relation_to_id", "to_id"),
@@ -155,13 +171,13 @@ class Relation(Base):
         Format: source/relation_type/target
         Example: "specs/search/implements/features/search-ui"
         """
+        # Only create permalinks when both source and target have permalinks
+        from_permalink = self.from_entity.permalink or self.from_entity.file_path
+
         if self.to_entity:
-            return generate_permalink(
-                f"{self.from_entity.permalink}/{self.relation_type}/{self.to_entity.permalink}"
-            )
-        return generate_permalink(
-            f"{self.from_entity.permalink}/{self.relation_type}/{self.to_name}"
-        )
+            to_permalink = self.to_entity.permalink or self.to_entity.file_path
+            return generate_permalink(f"{from_permalink}/{self.relation_type}/{to_permalink}")
+        return generate_permalink(f"{from_permalink}/{self.relation_type}/{self.to_name}")
 
     def __repr__(self) -> str:
-        return f"Relation(id={self.id}, from_id={self.from_id}, to_id={self.to_id}, to_name={self.to_name}, type='{self.relation_type}')"
+        return f"Relation(id={self.id}, from_id={self.from_id}, to_id={self.to_id}, to_name={self.to_name}, type='{self.relation_type}')"  # pragma: no cover

basic_memory/repository/repository.py

@@ -97,7 +97,7 @@ class Repository[T: Base]:
             entities = (self.Model,)
         return select(*entities)
 
-    async def find_all(self, skip: int = 0, limit: Optional[int] = 0) -> Sequence[T]:
+    async def find_all(self, skip: int = 0, limit: Optional[int] = None) -> Sequence[T]:
         """Fetch records from the database with pagination."""
         logger.debug(f"Finding all {self.Model.__name__} (skip={skip}, limit={limit})")
 

basic_memory/repository/search_repository.py

@@ -21,13 +21,14 @@ class SearchIndexRow:
 
     id: int
     type: str
-    permalink: str
     file_path: str
-    metadata: Optional[dict] = None
 
     # date values
-    created_at: Optional[datetime] = None
-    updated_at: Optional[datetime] = None
+    created_at: datetime
+    updated_at: datetime
+
+    permalink: Optional[str] = None
+    metadata: Optional[dict] = None
 
     # assigned in result
     score: Optional[float] = None
@@ -114,6 +115,7 @@ class SearchRepository:
         after_date: Optional[datetime] = None,
         entity_types: Optional[List[str]] = None,
         limit: int = 10,
+        offset: int = 0,
     ) -> List[SearchIndexRow]:
         """Search across all indexed content with fuzzy matching."""
         conditions = []
@@ -169,6 +171,7 @@ class SearchRepository:
 
         # set limit on search query
         params["limit"] = limit
+        params["offset"] = offset
 
         # Build WHERE clause
         where_clause = " AND ".join(conditions) if conditions else "1=1"
@@ -194,6 +197,7 @@ class SearchRepository:
             WHERE {where_clause}
             ORDER BY score ASC {order_by_clause}
             LIMIT :limit
+            OFFSET :offset
         """
 
         logger.debug(f"Search {sql} params: {params}")
@@ -262,6 +266,15 @@ class SearchRepository:
             logger.debug(f"indexed row {search_index_row}")
             await session.commit()
 
+    async def delete_by_entity_id(self, entity_id: int):
+        """Delete an item from the search index by entity_id."""
+        async with db.scoped_session(self.session_maker) as session:
+            await session.execute(
+                text("DELETE FROM search_index WHERE entity_id = :entity_id"),
+                {"entity_id": entity_id},
+            )
+            await session.commit()
+
     async def delete_by_permalink(self, permalink: str):
         """Delete an item from the search index."""
         async with db.scoped_session(self.session_maker) as session:

basic_memory/schemas/__init__.py

@@ -37,13 +37,6 @@ from basic_memory.schemas.response import (
     DeleteEntitiesResponse,
 )
 
-# Discovery and analytics models
-from basic_memory.schemas.discovery import (
-    EntityTypeList,
-    ObservationCategoryList,
-    TypedEntityList,
-)
-
 # For convenient imports, export all models
 __all__ = [
     # Base
@@ -66,8 +59,4 @@ __all__ = [
     "DeleteEntitiesResponse",
     # Delete Operations
     "DeleteEntitiesRequest",
-    # Discovery and Analytics
-    "EntityTypeList",
-    "ObservationCategoryList",
-    "TypedEntityList",
 ]

basic_memory/schemas/base.py

@@ -159,7 +159,10 @@ class Entity(BaseModel):
     @property
     def file_path(self):
         """Get the file path for this entity based on its permalink."""
-        return f"{self.folder}/{self.title}.md" if self.folder else f"{self.title}.md"
+        if self.content_type == "text/markdown":
+            return f"{self.folder}/{self.title}.md" if self.folder else f"{self.title}.md"
+        else:
+            return f"{self.folder}/{self.title}" if self.folder else self.title
 
     @property
     def permalink(self) -> Permalink:

basic_memory/schemas/memory.py

@@ -9,7 +9,7 @@ from pydantic import BaseModel, Field, BeforeValidator, TypeAdapter
 from basic_memory.schemas.search import SearchItemType
 
 
-def normalize_memory_url(url: str) -> str:
+def normalize_memory_url(url: str | None) -> str:
     """Normalize a MemoryUrl string.
 
     Args:
@@ -24,6 +24,9 @@ def normalize_memory_url(url: str) -> str:
         >>> normalize_memory_url("memory://specs/search")
         'memory://specs/search'
     """
+    if not url:
+        return ""
+
     clean_path = url.removeprefix("memory://")
     return f"memory://{clean_path}"
 
@@ -59,7 +62,7 @@ class EntitySummary(BaseModel):
     """Simplified entity representation."""
 
     type: str = "entity"
-    permalink: str
+    permalink: Optional[str]
     title: str
     file_path: str
     created_at: datetime
@@ -69,19 +72,25 @@ class RelationSummary(BaseModel):
     """Simplified relation representation."""
 
     type: str = "relation"
+    title: str
+    file_path: str
     permalink: str
     relation_type: str
     from_id: str
     to_id: Optional[str] = None
+    created_at: datetime
 
 
 class ObservationSummary(BaseModel):
     """Simplified observation representation."""
 
     type: str = "observation"
+    title: str
+    file_path: str
     permalink: str
     category: str
     content: str
+    created_at: datetime
 
 
 class MemoryMetadata(BaseModel):
@@ -111,3 +120,6 @@ class GraphContext(BaseModel):
 
     # Context metadata
     metadata: MemoryMetadata
+
+    page: int = 1
+    page_size: int = 1