basic-memory 0.6.0__py3-none-any.whl → 0.8.0__py3-none-any.whl

basic_memory/__init__.py

@@ -1,3 +1,3 @@
 """basic-memory - Local-first knowledge management combining Zettelkasten with knowledge graphs"""
 
-__version__ = "0.6.0"
+__version__ = "0.8.0"

basic_memory/alembic/alembic.ini

@@ -0,0 +1,119 @@
+# A generic, single database configuration.
+
+[alembic]
+# path to migration scripts
+# Use forward slashes (/) also on windows to provide an os agnostic path
+script_location = .
+
+# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s
+# Uncomment the line below if you want the files to be prepended with date and time
+# see https://alembic.sqlalchemy.org/en/latest/tutorial.html#editing-the-ini-file
+# for all available tokens
+# file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s
+
+# sys.path path, will be prepended to sys.path if present.
+# defaults to the current working directory.
+prepend_sys_path = .
+
+# timezone to use when rendering the date within the migration file
+# as well as the filename.
+# If specified, requires the python>=3.9 or backports.zoneinfo library and tzdata library.
+# Any required deps can installed by adding `alembic[tz]` to the pip requirements
+# string value is passed to ZoneInfo()
+# leave blank for localtime
+# timezone =
+
+# max length of characters to apply to the "slug" field
+# truncate_slug_length = 40
+
+# set to 'true' to run the environment during
+# the 'revision' command, regardless of autogenerate
+# revision_environment = false
+
+# set to 'true' to allow .pyc and .pyo files without
+# a source .py file to be detected as revisions in the
+# versions/ directory
+# sourceless = false
+
+# version location specification; This defaults
+# to migrations/versions.  When using multiple version
+# directories, initial revisions must be specified with --version-path.
+# The path separator used here should be the separator specified by "version_path_separator" below.
+# version_locations = %(here)s/bar:%(here)s/bat:migrations/versions
+
+# version path separator; As mentioned above, this is the character used to split
+# version_locations. The default within new alembic.ini files is "os", which uses os.pathsep.
+# If this key is omitted entirely, it falls back to the legacy behavior of splitting on spaces and/or commas.
+# Valid values for version_path_separator are:
+#
+# version_path_separator = :
+# version_path_separator = ;
+# version_path_separator = space
+# version_path_separator = newline
+#
+# Use os.pathsep. Default configuration used for new projects.
+version_path_separator = os
+
+# set to 'true' to search source files recursively
+# in each "version_locations" directory
+# new in Alembic version 1.10
+# recursive_version_locations = false
+
+# the output encoding used when revision files
+# are written from script.py.mako
+# output_encoding = utf-8
+
+sqlalchemy.url = driver://user:pass@localhost/dbname
+
+
+[post_write_hooks]
+# post_write_hooks defines scripts or Python functions that are run
+# on newly generated revision scripts.  See the documentation for further
+# detail and examples
+
+# format using "black" - use the console_scripts runner, against the "black" entrypoint
+# hooks = black
+# black.type = console_scripts
+# black.entrypoint = black
+# black.options = -l 79 REVISION_SCRIPT_FILENAME
+
+# lint with attempts to fix using "ruff" - use the exec runner, execute a binary
+# hooks = ruff
+# ruff.type = exec
+# ruff.executable = %(here)s/.venv/bin/ruff
+# ruff.options = --fix REVISION_SCRIPT_FILENAME
+
+# Logging configuration
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARNING
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARNING
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S

basic_memory/alembic/env.py

@@ -1,5 +1,6 @@
 """Alembic environment configuration."""
 
+import os
 from logging.config import fileConfig
 
 from sqlalchemy import engine_from_config
@@ -8,6 +9,10 @@ from sqlalchemy import pool
 from alembic import context
 
 from basic_memory.models import Base
+
+# set config.env to "test" for pytest to prevent logging to file in utils.setup_logging()
+os.environ["BASIC_MEMORY_ENV"] = "test"
+
 from basic_memory.config import config as app_config
 
 # this is the Alembic Config object, which provides
@@ -18,6 +23,8 @@ config = context.config
 sqlalchemy_url = f"sqlite:///{app_config.database_path}"
 config.set_main_option("sqlalchemy.url", sqlalchemy_url)
 
+# print(f"Using SQLAlchemy URL: {sqlalchemy_url}")
+
 # Interpret the config file for Python logging.
 if config.config_file_name is not None:
     fileConfig(config.config_file_name)
@@ -27,6 +34,14 @@ if config.config_file_name is not None:
 target_metadata = Base.metadata
 
 
+# Add this function to tell Alembic what to include/exclude
+def include_object(object, name, type_, reflected, compare_to):
+    # Ignore SQLite FTS tables
+    if type_ == "table" and name.startswith("search_index"):
+        return False
+    return True
+
+
 def run_migrations_offline() -> None:
     """Run migrations in 'offline' mode.
 
@@ -44,6 +59,8 @@ def run_migrations_offline() -> None:
         target_metadata=target_metadata,
         literal_binds=True,
         dialect_opts={"paramstyle": "named"},
+        include_object=include_object,
+        render_as_batch=True,
     )
 
     with context.begin_transaction():
@@ -63,7 +80,12 @@ def run_migrations_online() -> None:
     )
 
     with connectable.connect() as connection:
-        context.configure(connection=connection, target_metadata=target_metadata)
+        context.configure(
+            connection=connection,
+            target_metadata=target_metadata,
+            include_object=include_object,
+            render_as_batch=True,
+        )
 
         with context.begin_transaction():
             context.run_migrations()

basic_memory/alembic/versions/502b60eaa905_remove_required_from_entity_permalink.py

@@ -0,0 +1,51 @@
+"""remove required from entity.permalink
+
+Revision ID: 502b60eaa905
+Revises: b3c3938bacdb
+Create Date: 2025-02-24 13:33:09.790951
+
+"""
+
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = "502b60eaa905"
+down_revision: Union[str, None] = "b3c3938bacdb"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("entity", schema=None) as batch_op:
+        batch_op.alter_column("permalink", existing_type=sa.VARCHAR(), nullable=True)
+        batch_op.drop_index("ix_entity_permalink")
+        batch_op.create_index(batch_op.f("ix_entity_permalink"), ["permalink"], unique=False)
+        batch_op.drop_constraint("uix_entity_permalink", type_="unique")
+        batch_op.create_index(
+            "uix_entity_permalink",
+            ["permalink"],
+            unique=True,
+            sqlite_where=sa.text("content_type = 'text/markdown' AND permalink IS NOT NULL"),
+        )
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("entity", schema=None) as batch_op:
+        batch_op.drop_index(
+            "uix_entity_permalink",
+            sqlite_where=sa.text("content_type = 'text/markdown' AND permalink IS NOT NULL"),
+        )
+        batch_op.create_unique_constraint("uix_entity_permalink", ["permalink"])
+        batch_op.drop_index(batch_op.f("ix_entity_permalink"))
+        batch_op.create_index("ix_entity_permalink", ["permalink"], unique=1)
+        batch_op.alter_column("permalink", existing_type=sa.VARCHAR(), nullable=False)
+
+    # ### end Alembic commands ###

basic_memory/alembic/versions/b3c3938bacdb_relation_to_name_unique_index.py

@@ -0,0 +1,44 @@
+"""relation to_name unique index
+
+Revision ID: b3c3938bacdb
+Revises: 3dae7c7b1564
+Create Date: 2025-02-22 14:59:30.668466
+
+"""
+
+from typing import Sequence, Union
+
+from alembic import op
+
+
+# revision identifiers, used by Alembic.
+revision: str = "b3c3938bacdb"
+down_revision: Union[str, None] = "3dae7c7b1564"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # SQLite doesn't support constraint changes through ALTER
+    # Need to recreate table with desired constraints
+    with op.batch_alter_table("relation") as batch_op:
+        # Drop existing unique constraint
+        batch_op.drop_constraint("uix_relation", type_="unique")
+
+        # Add new constraints
+        batch_op.create_unique_constraint(
+            "uix_relation_from_id_to_id", ["from_id", "to_id", "relation_type"]
+        )
+        batch_op.create_unique_constraint(
+            "uix_relation_from_id_to_name", ["from_id", "to_name", "relation_type"]
+        )
+
+
+def downgrade() -> None:
+    with op.batch_alter_table("relation") as batch_op:
+        # Drop new constraints
+        batch_op.drop_constraint("uix_relation_from_id_to_name", type_="unique")
+        batch_op.drop_constraint("uix_relation_from_id_to_id", type_="unique")
+
+        # Restore original constraint
+        batch_op.create_unique_constraint("uix_relation", ["from_id", "to_id", "relation_type"])

basic_memory/api/app.py

@@ -7,18 +7,14 @@ from fastapi import FastAPI, HTTPException
 from fastapi.exception_handlers import http_exception_handler
 from loguru import logger
 
-import basic_memory
 from basic_memory import db
 from basic_memory.config import config as app_config
 from basic_memory.api.routers import knowledge, search, memory, resource
-from basic_memory.utils import setup_logging
 
 
 @asynccontextmanager
 async def lifespan(app: FastAPI):  # pragma: no cover
     """Lifecycle manager for the FastAPI app."""
-    setup_logging(log_file=".basic-memory/basic-memory.log")
-    logger.info(f"Starting Basic Memory API {basic_memory.__version__}")
     await db.run_migrations(app_config)
     yield
     logger.info("Shutting down Basic Memory API")

basic_memory/api/routers/knowledge_router.py

@@ -94,11 +94,8 @@ async def get_entity(
     try:
         entity = await entity_service.get_by_permalink(permalink)
         result = EntityResponse.model_validate(entity)
-
-        logger.info(f"response: get_entity with result={result}")
         return result
     except EntityNotFoundError:
-        logger.error(f"Error: Entity with {permalink} not found")
         raise HTTPException(status_code=404, detail=f"Entity with {permalink} not found")
 
 
@@ -114,8 +111,6 @@ async def get_entities(
     result = EntityListResponse(
         entities=[EntityResponse.model_validate(entity) for entity in entities]
     )
-
-    logger.info(f"response: get_entities with result={result}")
     return result
 
 
@@ -135,17 +130,15 @@ async def delete_entity(
 
     entity = await link_resolver.resolve_link(identifier)
     if entity is None:
-        logger.info("response: delete_entity with result=DeleteEntitiesResponse(deleted=False)")
         return DeleteEntitiesResponse(deleted=False)
 
     # Delete the entity
-    deleted = await entity_service.delete_entity(entity.permalink)
+    deleted = await entity_service.delete_entity(entity.permalink or entity.id)
 
     # Remove from search index
     background_tasks.add_task(search_service.delete_by_permalink, entity.permalink)
 
     result = DeleteEntitiesResponse(deleted=deleted)
-    logger.info(f"response: delete_entity with result={result}")
     return result
 
 
@@ -166,5 +159,4 @@ async def delete_entities(
         background_tasks.add_task(search_service.delete_by_permalink, permalink)
 
     result = DeleteEntitiesResponse(deleted=deleted)
-    logger.info(f"response: delete_entities with result={result}")
     return result

basic_memory/api/routers/memory_router.py

@@ -24,39 +24,37 @@ from basic_memory.services.context_service import ContextResultRow
 router = APIRouter(prefix="/memory", tags=["memory"])
 
 
-async def to_graph_context(context, entity_repository: EntityRepository):
+async def to_graph_context(context, entity_repository: EntityRepository, page: int, page_size: int):
     # return results
     async def to_summary(item: SearchIndexRow | ContextResultRow):
         match item.type:
             case SearchItemType.ENTITY:
-                assert item.title is not None
-                assert item.created_at is not None
-
                 return EntitySummary(
-                    title=item.title,
+                    title=item.title,  # pyright: ignore
                     permalink=item.permalink,
                     file_path=item.file_path,
                     created_at=item.created_at,
                 )
             case SearchItemType.OBSERVATION:
-                assert item.category is not None
-                assert item.content is not None
-
                 return ObservationSummary(
-                    category=item.category, content=item.content, permalink=item.permalink
+                    title=item.title,  # pyright: ignore
+                    file_path=item.file_path,
+                    category=item.category,  # pyright: ignore
+                    content=item.content,  # pyright: ignore
+                    permalink=item.permalink,  # pyright: ignore
+                    created_at=item.created_at,
                 )
             case SearchItemType.RELATION:
-                assert item.from_id is not None
-                from_entity = await entity_repository.find_by_id(item.from_id)
-                assert from_entity is not None
-
+                from_entity = await entity_repository.find_by_id(item.from_id)  # pyright: ignore
                 to_entity = await entity_repository.find_by_id(item.to_id) if item.to_id else None
-
                 return RelationSummary(
-                    permalink=item.permalink,
+                    title=item.title,  # pyright: ignore
+                    file_path=item.file_path,
+                    permalink=item.permalink,  # pyright: ignore
                     relation_type=item.type,
-                    from_id=from_entity.permalink,
+                    from_id=from_entity.permalink,  # pyright: ignore
                     to_id=to_entity.permalink if to_entity else None,
+                    created_at=item.created_at,
                 )
             case _:  # pragma: no cover
                 raise ValueError(f"Unexpected type: {item.type}")
@@ -66,7 +64,11 @@ async def to_graph_context(context, entity_repository: EntityRepository):
     metadata = MemoryMetadata.model_validate(context["metadata"])
     # Transform to GraphContext
     return GraphContext(
-        primary_results=primary_results, related_results=related_results, metadata=metadata
+        primary_results=primary_results,
+        related_results=related_results,
+        metadata=metadata,
+        page=page,
+        page_size=page_size,
     )
 
 
@@ -77,7 +79,9 @@ async def recent(
     type: Annotated[list[SearchItemType] | None, Query()] = None,
     depth: int = 1,
     timeframe: TimeFrame = "7d",
-    max_results: int = 10,
+    page: int = 1,
+    page_size: int = 10,
+    max_related: int = 10,
 ) -> GraphContext:
     # return all types by default
     types = (
@@ -87,16 +91,22 @@ async def recent(
     )
 
     logger.debug(
-        f"Getting recent context: `{types}` depth: `{depth}` timeframe: `{timeframe}` max_results: `{max_results}`"
+        f"Getting recent context: `{types}` depth: `{depth}` timeframe: `{timeframe}` page: `{page}` page_size: `{page_size}` max_related: `{max_related}`"
     )
     # Parse timeframe
     since = parse(timeframe)
+    limit = page_size
+    offset = (page - 1) * page_size
 
     # Build context
     context = await context_service.build_context(
-        types=types, depth=depth, since=since, max_results=max_results
+        types=types, depth=depth, since=since, limit=limit, offset=offset, max_related=max_related
     )
-    return await to_graph_context(context, entity_repository=entity_repository)
+    recent_context = await to_graph_context(
+        context, entity_repository=entity_repository, page=page, page_size=page_size
+    )
+    logger.debug(f"Recent context: {recent_context.model_dump_json()}")
+    return recent_context
 
 
 # get_memory_context needs to be declared last so other paths can match
@@ -109,21 +119,27 @@ async def get_memory_context(
     uri: str,
     depth: int = 1,
     timeframe: TimeFrame = "7d",
-    max_results: int = 10,
+    page: int = 1,
+    page_size: int = 10,
+    max_related: int = 10,
 ) -> GraphContext:
     """Get rich context from memory:// URI."""
     # add the project name from the config to the url as the "host
     # Parse URI
     logger.debug(
-        f"Getting context for URI: `{uri}` depth: `{depth}` timeframe: `{timeframe}` max_results: `{max_results}`"
+        f"Getting context for URI: `{uri}` depth: `{depth}` timeframe: `{timeframe}` page: `{page}` page_size: `{page_size}` max_related: `{max_related}`"
     )
     memory_url = normalize_memory_url(uri)
 
     # Parse timeframe
     since = parse(timeframe)
+    limit = page_size
+    offset = (page - 1) * page_size
 
     # Build context
     context = await context_service.build_context(
-        memory_url, depth=depth, since=since, max_results=max_results
+        memory_url, depth=depth, since=since, limit=limit, offset=offset, max_related=max_related
+    )
+    return await to_graph_context(
+        context, entity_repository=entity_repository, page=page, page_size=page_size
     )
-    return await to_graph_context(context, entity_repository=entity_repository)

basic_memory/api/routers/resource_router.py

@@ -2,9 +2,10 @@
 
 import tempfile
 from pathlib import Path
+from typing import Annotated
 
-from fastapi import APIRouter, HTTPException, BackgroundTasks
-from fastapi.responses import FileResponse
+from fastapi import APIRouter, HTTPException, BackgroundTasks, Body
+from fastapi.responses import FileResponse, JSONResponse
 from loguru import logger
 
 from basic_memory.deps import (
@@ -13,24 +14,27 @@ from basic_memory.deps import (
     SearchServiceDep,
     EntityServiceDep,
     FileServiceDep,
+    EntityRepositoryDep,
 )
 from basic_memory.repository.search_repository import SearchIndexRow
 from basic_memory.schemas.memory import normalize_memory_url
 from basic_memory.schemas.search import SearchQuery, SearchItemType
+from basic_memory.models.knowledge import Entity as EntityModel
+from datetime import datetime
 
 router = APIRouter(prefix="/resource", tags=["resources"])
 
 
-def get_entity_ids(item: SearchIndexRow) -> list[int]:
+def get_entity_ids(item: SearchIndexRow) -> set[int]:
     match item.type:
         case SearchItemType.ENTITY:
-            return [item.id]
+            return {item.id}
         case SearchItemType.OBSERVATION:
-            return [item.entity_id]  # pyright: ignore [reportReturnType]
+            return {item.entity_id}  # pyright: ignore [reportReturnType]
         case SearchItemType.RELATION:
             from_entity = item.from_id
             to_entity = item.to_id  # pyright: ignore [reportReturnType]
-            return [from_entity, to_entity] if to_entity else [from_entity]  # pyright: ignore [reportReturnType]
+            return {from_entity, to_entity} if to_entity else {from_entity}  # pyright: ignore [reportReturnType]
         case _:  # pragma: no cover
             raise ValueError(f"Unexpected type: {item.type}")
 
@@ -44,6 +48,8 @@ async def get_resource_content(
     file_service: FileServiceDep,
     background_tasks: BackgroundTasks,
     identifier: str,
+    page: int = 1,
+    page_size: int = 10,
 ) -> FileResponse:
     """Get resource content by identifier: name or permalink."""
     logger.debug(f"Getting content for: {identifier}")
@@ -52,6 +58,10 @@ async def get_resource_content(
     entity = await link_resolver.resolve_link(identifier)
     results = [entity] if entity else []
 
+    # pagination for multiple results
+    limit = page_size
+    offset = (page - 1) * page_size
+
     # search using the identifier as a permalink
     if not results:
         # if the identifier contains a wildcard, use GLOB search
@@ -60,13 +70,13 @@ async def get_resource_content(
             if "*" in identifier
             else SearchQuery(permalink=identifier)
         )
-        search_results = await search_service.search(query)
+        search_results = await search_service.search(query, limit, offset)
         if not search_results:
             raise HTTPException(status_code=404, detail=f"Resource not found: {identifier}")
 
-        # get the entities related to the search results
-        entity_ids = [id for result in search_results for id in get_entity_ids(result)]
-        results = await entity_service.get_entities_by_id(entity_ids)
+        # get the deduplicated entities related to the search results
+        entity_ids = {id for result in search_results for id in get_entity_ids(result)}
+        results = await entity_service.get_entities_by_id(list(entity_ids))
 
     # return single response
     if len(results) == 1:
@@ -88,8 +98,7 @@ async def get_resource_content(
             content = await file_service.read_entity_content(result)
             memory_url = normalize_memory_url(result.permalink)
             modified_date = result.updated_at.isoformat()
-            assert result.checksum
-            checksum = result.checksum[:8]
+            checksum = result.checksum[:8] if result.checksum else ""
 
             # Prepare the delimited content
             response_content = f"--- {memory_url} {modified_date} {checksum}\n"
@@ -116,3 +125,101 @@ def cleanup_temp_file(file_path: str):
         logger.debug(f"Temporary file deleted: {file_path}")
     except Exception as e:  # pragma: no cover
         logger.error(f"Error deleting temporary file {file_path}: {e}")
+
+
+@router.put("/{file_path:path}")
+async def write_resource(
+    config: ProjectConfigDep,
+    file_service: FileServiceDep,
+    entity_repository: EntityRepositoryDep,
+    search_service: SearchServiceDep,
+    file_path: str,
+    content: Annotated[str, Body()],
+) -> JSONResponse:
+    """Write content to a file in the project.
+
+    This endpoint allows writing content directly to a file in the project.
+    Also creates an entity record and indexes the file for search.
+
+    Args:
+        file_path: Path to write to, relative to project root
+        request: Contains the content to write
+
+    Returns:
+        JSON response with file information
+    """
+    try:
+        # Get content from request body
+
+        # Ensure it's UTF-8 string content
+        if isinstance(content, bytes):  # pragma: no cover
+            content_str = content.decode("utf-8")
+        else:
+            content_str = str(content)
+
+        # Get full file path
+        full_path = Path(f"{config.home}/{file_path}")
+
+        # Ensure parent directory exists
+        full_path.parent.mkdir(parents=True, exist_ok=True)
+
+        # Write content to file
+        checksum = await file_service.write_file(full_path, content_str)
+
+        # Get file info
+        file_stats = file_service.file_stats(full_path)
+
+        # Determine file details
+        file_name = Path(file_path).name
+        content_type = file_service.content_type(full_path)
+
+        entity_type = "canvas" if file_path.endswith(".canvas") else "file"
+
+        # Check if entity already exists
+        existing_entity = await entity_repository.get_by_file_path(file_path)
+
+        if existing_entity:
+            # Update existing entity
+            entity = await entity_repository.update(
+                existing_entity.id,
+                {
+                    "title": file_name,
+                    "entity_type": entity_type,
+                    "content_type": content_type,
+                    "file_path": file_path,
+                    "checksum": checksum,
+                    "updated_at": datetime.fromtimestamp(file_stats.st_mtime),
+                },
+            )
+            status_code = 200
+        else:
+            # Create a new entity model
+            entity = EntityModel(
+                title=file_name,
+                entity_type=entity_type,
+                content_type=content_type,
+                file_path=file_path,
+                checksum=checksum,
+                created_at=datetime.fromtimestamp(file_stats.st_ctime),
+                updated_at=datetime.fromtimestamp(file_stats.st_mtime),
+            )
+            entity = await entity_repository.add(entity)
+            status_code = 201
+
+        # Index the file for search
+        await search_service.index_entity(entity)  # pyright: ignore
+
+        # Return success response
+        return JSONResponse(
+            status_code=status_code,
+            content={
+                "file_path": file_path,
+                "checksum": checksum,
+                "size": file_stats.st_size,
+                "created_at": file_stats.st_ctime,
+                "modified_at": file_stats.st_mtime,
+            },
+        )
+    except Exception as e:  # pragma: no cover
+        logger.error(f"Error writing resource {file_path}: {e}")
+        raise HTTPException(status_code=500, detail=f"Failed to write resource: {str(e)}")