basic-memory 0.16.1__py3-none-any.whl → 0.17.4__py3-none-any.whl

basic_memory/services/context_service.py

@@ -4,11 +4,13 @@ from dataclasses import dataclass, field
 from datetime import datetime, timezone
 from typing import List, Optional, Tuple
 
+
 from loguru import logger
 from sqlalchemy import text
 
 from basic_memory.repository.entity_repository import EntityRepository
 from basic_memory.repository.observation_repository import ObservationRepository
+from basic_memory.repository.postgres_search_repository import PostgresSearchRepository
 from basic_memory.repository.search_repository import SearchRepository, SearchIndexRow
 from basic_memory.schemas.memory import MemoryUrl, memory_url_path
 from basic_memory.schemas.search import SearchItemType
@@ -252,9 +254,6 @@ class ContextService:
         # Build the VALUES clause for entity IDs
         entity_id_values = ", ".join([str(i) for i in entity_ids])
 
-        # For compatibility with the old query, we still need this for filtering
-        values = ", ".join([f"('{t}', {i})" for t, i in type_id_pairs])
-
         # Parameters for bindings - include project_id for security filtering
         params = {
             "max_depth": max_depth,
@@ -264,7 +263,14 @@ class ContextService:
 
         # Build date and timeframe filters conditionally based on since parameter
         if since:
-            params["since_date"] = since.isoformat()  # pyright: ignore
+            # SQLite accepts ISO strings, but Postgres/asyncpg requires datetime objects
+            if isinstance(self.search_repository, PostgresSearchRepository):  # pragma: no cover
+                # asyncpg expects timezone-NAIVE datetime in UTC for DateTime(timezone=True) columns
+                # even though the column stores timezone-aware values
+                since_utc = since.astimezone(timezone.utc) if since.tzinfo else since  # pragma: no cover
+                params["since_date"] = since_utc.replace(tzinfo=None)  # pyright: ignore  # pragma: no cover
+            else:
+                params["since_date"] = since.isoformat()  # pyright: ignore
             date_filter = "AND e.created_at >= :since_date"
             relation_date_filter = "AND e_from.created_at >= :since_date"
             timeframe_condition = "AND eg.relation_date >= :since_date"
@@ -279,13 +285,210 @@ class ContextService:
 
         # Use a CTE that operates directly on entity and relation tables
         # This avoids the overhead of the search_index virtual table
-        query = text(f"""
+        # Note: Postgres and SQLite have different CTE limitations:
+        # - Postgres: doesn't allow multiple UNION ALL branches referencing the CTE
+        # - SQLite: doesn't support LATERAL joins
+        # So we need different queries for each database backend
+
+        # Detect database backend
+        is_postgres = isinstance(self.search_repository, PostgresSearchRepository)
+
+        if is_postgres:  # pragma: no cover
+            query = self._build_postgres_query(
+                entity_id_values,
+                date_filter,
+                project_filter,
+                relation_date_filter,
+                relation_project_filter,
+                timeframe_condition,
+            )
+        else:
+            # SQLite needs VALUES clause for exclusion (not needed for Postgres)
+            values = ", ".join([f"('{t}', {i})" for t, i in type_id_pairs])
+            query = self._build_sqlite_query(
+                entity_id_values,
+                date_filter,
+                project_filter,
+                relation_date_filter,
+                relation_project_filter,
+                timeframe_condition,
+                values,
+            )
+
+        result = await self.search_repository.execute_query(query, params=params)
+        rows = result.all()
+
+        context_rows = [
+            ContextResultRow(
+                type=row.type,
+                id=row.id,
+                title=row.title,
+                permalink=row.permalink,
+                file_path=row.file_path,
+                from_id=row.from_id,
+                to_id=row.to_id,
+                relation_type=row.relation_type,
+                content=row.content,
+                category=row.category,
+                entity_id=row.entity_id,
+                depth=row.depth,
+                root_id=row.root_id,
+                created_at=row.created_at,
+            )
+            for row in rows
+        ]
+        return context_rows
+
+    def _build_postgres_query(  # pragma: no cover
+        self,
+        entity_id_values: str,
+        date_filter: str,
+        project_filter: str,
+        relation_date_filter: str,
+        relation_project_filter: str,
+        timeframe_condition: str,
+    ):
+        """Build Postgres-specific CTE query using LATERAL joins."""
+        return text(f"""
         WITH RECURSIVE entity_graph AS (
             -- Base case: seed entities
-            SELECT 
+            SELECT
                 e.id,
                 'entity' as type,
-                e.title, 
+                e.title,
+                e.permalink,
+                e.file_path,
+                CAST(NULL AS INTEGER) as from_id,
+                CAST(NULL AS INTEGER) as to_id,
+                CAST(NULL AS TEXT) as relation_type,
+                CAST(NULL AS TEXT) as content,
+                CAST(NULL AS TEXT) as category,
+                CAST(NULL AS INTEGER) as entity_id,
+                0 as depth,
+                e.id as root_id,
+                e.created_at,
+                e.created_at as relation_date
+            FROM entity e
+            WHERE e.id IN ({entity_id_values})
+            {date_filter}
+            {project_filter}
+
+            UNION ALL
+
+            -- Fetch BOTH relations AND connected entities in a single recursive step
+            -- Postgres only allows ONE reference to the recursive CTE in the recursive term
+            -- We use CROSS JOIN LATERAL to generate two rows (relation + entity) from each traversal
+            SELECT
+                CASE
+                    WHEN step_type = 1 THEN r.id
+                    ELSE e.id
+                END as id,
+                CASE
+                    WHEN step_type = 1 THEN 'relation'
+                    ELSE 'entity'
+                END as type,
+                CASE
+                    WHEN step_type = 1 THEN r.relation_type || ': ' || r.to_name
+                    ELSE e.title
+                END as title,
+                CASE
+                    WHEN step_type = 1 THEN ''
+                    ELSE COALESCE(e.permalink, '')
+                END as permalink,
+                CASE
+                    WHEN step_type = 1 THEN e_from.file_path
+                    ELSE e.file_path
+                END as file_path,
+                CASE
+                    WHEN step_type = 1 THEN r.from_id
+                    ELSE NULL
+                END as from_id,
+                CASE
+                    WHEN step_type = 1 THEN r.to_id
+                    ELSE NULL
+                END as to_id,
+                CASE
+                    WHEN step_type = 1 THEN r.relation_type
+                    ELSE NULL
+                END as relation_type,
+                CAST(NULL AS TEXT) as content,
+                CAST(NULL AS TEXT) as category,
+                CAST(NULL AS INTEGER) as entity_id,
+                eg.depth + step_type as depth,
+                eg.root_id,
+                CASE
+                    WHEN step_type = 1 THEN e_from.created_at
+                    ELSE e.created_at
+                END as created_at,
+                CASE
+                    WHEN step_type = 1 THEN e_from.created_at
+                    ELSE eg.relation_date
+                END as relation_date
+            FROM entity_graph eg
+            CROSS JOIN LATERAL (VALUES (1), (2)) AS steps(step_type)
+            JOIN relation r ON (
+                eg.type = 'entity' AND
+                (r.from_id = eg.id OR r.to_id = eg.id)
+            )
+            JOIN entity e_from ON (
+                r.from_id = e_from.id
+                {relation_project_filter}
+            )
+            LEFT JOIN entity e ON (
+                step_type = 2 AND
+                e.id = CASE
+                    WHEN r.from_id = eg.id THEN r.to_id
+                    ELSE r.from_id
+                END
+                {date_filter}
+                {project_filter}
+            )
+            WHERE eg.depth < :max_depth
+            AND (step_type = 1 OR (step_type = 2 AND e.id IS NOT NULL AND e.id != eg.id))
+            {timeframe_condition}
+        )
+        -- Materialize and filter
+        SELECT DISTINCT
+            type,
+            id,
+            title,
+            permalink,
+            file_path,
+            from_id,
+            to_id,
+            relation_type,
+            content,
+            category,
+            entity_id,
+            MIN(depth) as depth,
+            root_id,
+            created_at
+        FROM entity_graph
+        WHERE depth > 0
+        GROUP BY type, id, title, permalink, file_path, from_id, to_id,
+                 relation_type, content, category, entity_id, root_id, created_at
+        ORDER BY depth, type, id
+        LIMIT :max_results
+       """)
+
+    def _build_sqlite_query(
+        self,
+        entity_id_values: str,
+        date_filter: str,
+        project_filter: str,
+        relation_date_filter: str,
+        relation_project_filter: str,
+        timeframe_condition: str,
+        values: str,
+    ):
+        """Build SQLite-specific CTE query using multiple UNION ALL branches."""
+        return text(f"""
+        WITH RECURSIVE entity_graph AS (
+            -- Base case: seed entities
+            SELECT
+                e.id,
+                'entity' as type,
+                e.title,
                 e.permalink,
                 e.file_path,
                 NULL as from_id,
@@ -311,7 +514,6 @@ class ContextService:
                 r.id,
                 'relation' as type,
                 r.relation_type || ': ' || r.to_name as title,
-                -- Relation model doesn't have permalink column - we'll generate it at runtime
                 '' as permalink,
                 e_from.file_path,
                 r.from_id,
@@ -322,7 +524,7 @@ class ContextService:
                 NULL as entity_id,
                 eg.depth + 1,
                 eg.root_id,
-                e_from.created_at, -- Use the from_entity's created_at since relation has no timestamp
+                e_from.created_at,
                 e_from.created_at as relation_date,
                 CASE WHEN r.from_id = eg.id THEN 0 ELSE 1 END as is_incoming
             FROM entity_graph eg
@@ -337,7 +539,6 @@ class ContextService:
             )
             LEFT JOIN entity e_to ON (r.to_id = e_to.id)
             WHERE eg.depth < :max_depth
-            -- Ensure to_entity (if exists) also belongs to same project
             AND (r.to_id IS NULL OR e_to.project_id = :project_id)
 
             UNION ALL
@@ -347,9 +548,9 @@ class ContextService:
                 e.id,
                 'entity' as type,
                 e.title,
-                CASE 
-                    WHEN e.permalink IS NULL THEN '' 
-                    ELSE e.permalink 
+                CASE
+                    WHEN e.permalink IS NULL THEN ''
+                    ELSE e.permalink
                 END as permalink,
                 e.file_path,
                 NULL as from_id,
@@ -366,7 +567,7 @@ class ContextService:
             FROM entity_graph eg
             JOIN entity e ON (
                 eg.type = 'relation' AND
-                e.id = CASE 
+                e.id = CASE
                     WHEN eg.is_incoming = 0 THEN eg.to_id
                     ELSE eg.from_id
                 END
@@ -374,10 +575,9 @@ class ContextService:
                 {project_filter}
             )
             WHERE eg.depth < :max_depth
-            -- Only include entities connected by relations within timeframe if specified
             {timeframe_condition}
         )
-        SELECT DISTINCT 
+        SELECT DISTINCT
             type,
             id,
             title,
@@ -393,33 +593,9 @@ class ContextService:
             root_id,
             created_at
         FROM entity_graph
-        WHERE (type, id) NOT IN ({values})
-        GROUP BY
-            type, id
+        WHERE depth > 0
+        GROUP BY type, id, title, permalink, file_path, from_id, to_id,
+                 relation_type, content, category, entity_id, root_id, created_at
         ORDER BY depth, type, id
         LIMIT :max_results
        """)
-
-        result = await self.search_repository.execute_query(query, params=params)
-        rows = result.all()
-
-        context_rows = [
-            ContextResultRow(
-                type=row.type,
-                id=row.id,
-                title=row.title,
-                permalink=row.permalink,
-                file_path=row.file_path,
-                from_id=row.from_id,
-                to_id=row.to_id,
-                relation_type=row.relation_type,
-                content=row.content,
-                category=row.category,
-                entity_id=row.entity_id,
-                depth=row.depth,
-                root_id=row.root_id,
-                created_at=row.created_at,
-            )
-            for row in rows
-        ]
-        return context_rows

basic_memory/services/directory_service.py

@@ -3,8 +3,10 @@
 import fnmatch
 import logging
 import os
+from datetime import datetime
 from typing import Dict, List, Optional, Sequence
 
+
 from basic_memory.models import Entity
 from basic_memory.repository import EntityRepository
 from basic_memory.schemas.directory import DirectoryNode
@@ -12,6 +14,17 @@ from basic_memory.schemas.directory import DirectoryNode
 logger = logging.getLogger(__name__)
 
 
+def _mtime_to_datetime(entity: Entity) -> datetime:
+    """Convert entity mtime (file modification time) to datetime.
+
+    Returns the file's actual modification time, falling back to updated_at
+    if mtime is not available.
+    """
+    if entity.mtime:  # pragma: no cover
+        return datetime.fromtimestamp(entity.mtime).astimezone()  # pragma: no cover
+    return entity.updated_at
+
+
 class DirectoryService:
     """Service for working with directory trees."""
 
@@ -74,10 +87,11 @@ class DirectoryService:
                 type="file",
                 title=file.title,
                 permalink=file.permalink,
+                external_id=file.external_id,  # UUID for v2 API
                 entity_id=file.id,
                 entity_type=file.entity_type,
                 content_type=file.content_type,
-                updated_at=file.updated_at,
+                updated_at=_mtime_to_datetime(file),
             )
 
             # Add to parent directory's children
@@ -173,7 +187,7 @@ class DirectoryService:
         # Find the target directory node
         target_node = self._find_directory_node(root_tree, dir_name)
         if not target_node:
-            return []
+            return []  # pragma: no cover
 
         # Collect nodes with depth and glob filtering
         result = []
@@ -238,18 +252,19 @@ class DirectoryService:
                 type="file",
                 title=file.title,
                 permalink=file.permalink,
+                external_id=file.external_id,  # UUID for v2 API
                 entity_id=file.id,
                 entity_type=file.entity_type,
                 content_type=file.content_type,
-                updated_at=file.updated_at,
+                updated_at=_mtime_to_datetime(file),
             )
 
             # Add to parent directory's children
             if directory_path in dir_map:
                 dir_map[directory_path].children.append(file_node)
-            elif root_path in dir_map:
+            elif root_path in dir_map:  # pragma: no cover
                 # Fallback to root if parent not found
-                dir_map[root_path].children.append(file_node)
+                dir_map[root_path].children.append(file_node)  # pragma: no cover
 
         return root_node
 
@@ -260,13 +275,13 @@ class DirectoryService:
         if root.directory_path == target_path:
             return root
 
-        for child in root.children:
-            if child.type == "directory":
-                found = self._find_directory_node(child, target_path)
-                if found:
-                    return found
+        for child in root.children:  # pragma: no cover
+            if child.type == "directory":  # pragma: no cover
+                found = self._find_directory_node(child, target_path)  # pragma: no cover
+                if found:  # pragma: no cover
+                    return found  # pragma: no cover
 
-        return None
+        return None  # pragma: no cover
 
     def _collect_nodes_recursive(
         self,

basic_memory/services/entity_service.py

@@ -8,6 +8,7 @@ import yaml
 from loguru import logger
 from sqlalchemy.exc import IntegrityError
 
+
 from basic_memory.config import ProjectConfig, BasicMemoryConfig
 from basic_memory.file_utils import (
     has_frontmatter,
@@ -28,6 +29,7 @@ from basic_memory.schemas.base import Permalink
 from basic_memory.services import BaseService, FileService
 from basic_memory.services.exceptions import EntityCreationError, EntityNotFoundError
 from basic_memory.services.link_resolver import LinkResolver
+from basic_memory.services.search_service import SearchService
 from basic_memory.utils import generate_permalink
 
 
@@ -42,6 +44,7 @@ class EntityService(BaseService[EntityModel]):
         relation_repository: RelationRepository,
         file_service: FileService,
         link_resolver: LinkResolver,
+        search_service: Optional[SearchService] = None,
         app_config: Optional[BasicMemoryConfig] = None,
     ):
         super().__init__(entity_repository)
@@ -50,6 +53,7 @@ class EntityService(BaseService[EntityModel]):
         self.entity_parser = entity_parser
         self.file_service = file_service
         self.link_resolver = link_resolver
+        self.search_service = search_service
         self.app_config = app_config
 
     async def detect_file_path_conflicts(
@@ -106,6 +110,9 @@ class EntityService(BaseService[EntityModel]):
         4. Generate new unique permalink from file path
 
         Enhanced to detect and handle character-related conflicts.
+
+        Note: Uses lightweight repository methods that skip eager loading of
+        observations and relations for better performance during bulk operations.
         """
         file_path_str = Path(file_path).as_posix()
 
@@ -122,16 +129,20 @@ class EntityService(BaseService[EntityModel]):
         # If markdown has explicit permalink, try to validate it
         if markdown and markdown.frontmatter.permalink:
             desired_permalink = markdown.frontmatter.permalink
-            existing = await self.repository.get_by_permalink(desired_permalink)
+            # Use lightweight method - we only need to check file_path
+            existing_file_path = await self.repository.get_file_path_for_permalink(
+                desired_permalink
+            )
 
             # If no conflict or it's our own file, use as is
-            if not existing or existing.file_path == file_path_str:
+            if not existing_file_path or existing_file_path == file_path_str:
                 return desired_permalink
 
         # For existing files, try to find current permalink
-        existing = await self.repository.get_by_file_path(file_path_str)
-        if existing:
-            return existing.permalink
+        # Use lightweight method - we only need the permalink
+        existing_permalink = await self.repository.get_permalink_for_file_path(file_path_str)
+        if existing_permalink:
+            return existing_permalink
 
         # New file - generate permalink
         if markdown and markdown.frontmatter.permalink:
@@ -140,9 +151,10 @@ class EntityService(BaseService[EntityModel]):
             desired_permalink = generate_permalink(file_path_str)
 
         # Make unique if needed - enhanced to handle character conflicts
+        # Use lightweight existence check instead of loading full entity
         permalink = desired_permalink
         suffix = 1
-        while await self.repository.get_by_permalink(permalink):
+        while await self.repository.permalink_exists(permalink):
             permalink = f"{desired_permalink}-{suffix}"
             suffix += 1
             logger.debug(f"creating unique permalink: {permalink}")
@@ -224,8 +236,11 @@ class EntityService(BaseService[EntityModel]):
         final_content = dump_frontmatter(post)
         checksum = await self.file_service.write_file(file_path, final_content)
 
-        # parse entity from file
-        entity_markdown = await self.entity_parser.parse_file(file_path)
+        # parse entity from content we just wrote (avoids re-reading file for cloud compatibility)
+        entity_markdown = await self.entity_parser.parse_markdown_content(
+            file_path=file_path,
+            content=final_content,
+        )
 
         # create entity
         created = await self.create_entity_from_markdown(file_path, entity_markdown)
@@ -245,8 +260,12 @@ class EntityService(BaseService[EntityModel]):
         # Convert file path string to Path
         file_path = Path(entity.file_path)
 
-        # Read existing frontmatter from the file if it exists
-        existing_markdown = await self.entity_parser.parse_file(file_path)
+        # Read existing content via file_service (for cloud compatibility)
+        existing_content = await self.file_service.read_file_content(file_path)
+        existing_markdown = await self.entity_parser.parse_markdown_content(
+            file_path=file_path,
+            content=existing_content,
+        )
 
         # Parse content frontmatter to check for user-specified permalink and entity_type
         content_markdown = None
@@ -302,8 +321,11 @@ class EntityService(BaseService[EntityModel]):
         final_content = dump_frontmatter(merged_post)
         checksum = await self.file_service.write_file(file_path, final_content)
 
-        # parse entity from file
-        entity_markdown = await self.entity_parser.parse_file(file_path)
+        # parse entity from content we just wrote (avoids re-reading file for cloud compatibility)
+        entity_markdown = await self.entity_parser.parse_markdown_content(
+            file_path=file_path,
+            content=final_content,
+        )
 
         # update entity in db
         entity = await self.update_entity_and_observations(file_path, entity_markdown)
@@ -335,7 +357,11 @@ class EntityService(BaseService[EntityModel]):
                     )
                 entity = entities[0]
 
-            # Delete file first
+            # Delete from search index first (if search_service is available)
+            if self.search_service:
+                await self.search_service.handle_delete(entity)
+
+            # Delete file
             await self.file_service.delete_entity_file(entity)
 
             # Delete from DB (this will cascade to observations/relations)
@@ -378,7 +404,9 @@ class EntityService(BaseService[EntityModel]):
         Uses UPSERT approach to handle permalink/file_path conflicts cleanly.
         """
         logger.debug(f"Creating entity: {markdown.frontmatter.title} file_path: {file_path}")
-        model = entity_model_from_markdown(file_path, markdown)
+        model = entity_model_from_markdown(
+            file_path, markdown, project_id=self.repository.project_id
+        )
 
         # Mark as incomplete because we still need to add relations
         model.checksum = None
@@ -408,6 +436,7 @@ class EntityService(BaseService[EntityModel]):
         # add new observations
         observations = [
             Observation(
+                project_id=self.observation_repository.project_id,
                 entity_id=db_entity.id,
                 content=obs.content,
                 category=obs.category,
@@ -448,8 +477,11 @@ class EntityService(BaseService[EntityModel]):
             import asyncio
 
             # Create tasks for all relation lookups
+            # Use strict=True to disable fuzzy search - only exact matches should create resolved relations
+            # This ensures forward references (links to non-existent entities) remain unresolved (to_id=NULL)
             lookup_tasks = [
-                self.link_resolver.resolve_link(rel.target) for rel in markdown.relations
+                self.link_resolver.resolve_link(rel.target, strict=True)
+                for rel in markdown.relations
             ]
 
             # Execute all lookups in parallel
@@ -471,6 +503,7 @@ class EntityService(BaseService[EntityModel]):
 
                 # Create the relation
                 relation = Relation(
+                    project_id=self.relation_repository.project_id,
                     from_id=db_entity.id,
                     to_id=target_id,
                     to_name=target_name,
@@ -543,8 +576,11 @@ class EntityService(BaseService[EntityModel]):
         # Write the updated content back to the file
         checksum = await self.file_service.write_file(file_path, new_content)
 
-        # Parse the updated file to get new observations/relations
-        entity_markdown = await self.entity_parser.parse_file(file_path)
+        # Parse the content we just wrote (avoids re-reading file for cloud compatibility)
+        entity_markdown = await self.entity_parser.parse_markdown_content(
+            file_path=file_path,
+            content=new_content,
+        )
 
         # Update entity and its relationships
         entity = await self.update_entity_and_observations(file_path, entity_markdown)
@@ -763,23 +799,20 @@ class EntityService(BaseService[EntityModel]):
             raise ValueError(f"Invalid destination path: {destination_path}")
 
         # 3. Validate paths
-        source_file = project_config.home / current_path
-        destination_file = project_config.home / destination_path
-
-        # Validate source exists
-        if not source_file.exists():
+        # NOTE: In tenantless/cloud mode, we cannot rely on local filesystem paths.
+        # Use FileService for existence checks and moving.
+        if not await self.file_service.exists(current_path):
             raise ValueError(f"Source file not found: {current_path}")
 
-        # Check if destination already exists
-        if destination_file.exists():
+        if await self.file_service.exists(destination_path):
             raise ValueError(f"Destination already exists: {destination_path}")
 
         try:
-            # 4. Create destination directory if needed
-            destination_file.parent.mkdir(parents=True, exist_ok=True)
+            # 4. Ensure destination directory if needed (no-op for S3)
+            await self.file_service.ensure_directory(Path(destination_path).parent)
 
-            # 5. Move physical file
-            source_file.rename(destination_file)
+            # 5. Move physical file via FileService (filesystem rename or cloud move)
+            await self.file_service.move_file(current_path, destination_path)
             logger.info(f"Moved file: {current_path} -> {destination_path}")
 
             # 6. Prepare database updates
@@ -818,12 +851,14 @@ class EntityService(BaseService[EntityModel]):
 
         except Exception as e:
             # Rollback: try to restore original file location if move succeeded
-            if destination_file.exists() and not source_file.exists():
-                try:
-                    destination_file.rename(source_file)
+            try:
+                if await self.file_service.exists(
+                    destination_path
+                ) and not await self.file_service.exists(current_path):
+                    await self.file_service.move_file(destination_path, current_path)
                     logger.info(f"Rolled back file move: {destination_path} -> {current_path}")
-                except Exception as rollback_error:  # pragma: no cover
-                    logger.error(f"Failed to rollback file move: {rollback_error}")
+            except Exception as rollback_error:  # pragma: no cover
+                logger.error(f"Failed to rollback file move: {rollback_error}")
 
             # Re-raise the original error with context
             raise ValueError(f"Move failed: {str(e)}") from e