basic-memory 0.13.0b4__py3-none-any.whl → 0.13.0b6__py3-none-any.whl

basic_memory/services/migration_service.py

@@ -0,0 +1,168 @@
+"""Migration service for handling background migrations and status tracking."""
+
+import asyncio
+from dataclasses import dataclass
+from enum import Enum
+from pathlib import Path
+from typing import Optional
+
+from loguru import logger
+
+from basic_memory.config import BasicMemoryConfig
+
+
+class MigrationStatus(Enum):
+    """Status of migration operations."""
+
+    NOT_NEEDED = "not_needed"
+    PENDING = "pending"
+    IN_PROGRESS = "in_progress"
+    COMPLETED = "completed"
+    FAILED = "failed"
+
+
+@dataclass
+class MigrationState:
+    """Current state of migration operations."""
+
+    status: MigrationStatus
+    message: str
+    progress: Optional[str] = None
+    error: Optional[str] = None
+    projects_migrated: int = 0
+    projects_total: int = 0
+
+
+class MigrationManager:
+    """Manages background migration operations and status tracking."""
+
+    def __init__(self):
+        self._state = MigrationState(
+            status=MigrationStatus.NOT_NEEDED, message="No migration required"
+        )
+        self._migration_task: Optional[asyncio.Task] = None
+
+    @property
+    def state(self) -> MigrationState:
+        """Get current migration state."""
+        return self._state
+
+    @property
+    def is_ready(self) -> bool:
+        """Check if the system is ready for normal operations."""
+        return self._state.status in (MigrationStatus.NOT_NEEDED, MigrationStatus.COMPLETED)
+
+    @property
+    def status_message(self) -> str:
+        """Get a user-friendly status message."""
+        if self._state.status == MigrationStatus.IN_PROGRESS:
+            progress = (
+                f" ({self._state.projects_migrated}/{self._state.projects_total})"
+                if self._state.projects_total > 0
+                else ""
+            )
+            return f"🔄 File sync in progress{progress}: {self._state.message}. Use sync_status() tool for details."
+        elif self._state.status == MigrationStatus.FAILED:
+            return f"❌ File sync failed: {self._state.error or 'Unknown error'}. Use sync_status() tool for details."
+        elif self._state.status == MigrationStatus.COMPLETED:
+            return "✅ File sync completed successfully"
+        else:
+            return "✅ System ready"
+
+    async def check_migration_needed(self, app_config: BasicMemoryConfig) -> bool:
+        """Check if migration is needed without performing it."""
+        from basic_memory import db
+        from basic_memory.repository import ProjectRepository
+
+        try:
+            # Get database session
+            _, session_maker = await db.get_or_create_db(
+                db_path=app_config.database_path, db_type=db.DatabaseType.FILESYSTEM
+            )
+            project_repository = ProjectRepository(session_maker)
+
+            # Check for legacy projects
+            legacy_projects = []
+            for project_name, project_path in app_config.projects.items():
+                legacy_dir = Path(project_path) / ".basic-memory"
+                if legacy_dir.exists():
+                    project = await project_repository.get_by_name(project_name)
+                    if project:
+                        legacy_projects.append(project)
+
+            if legacy_projects:
+                self._state = MigrationState(
+                    status=MigrationStatus.PENDING,
+                    message="Legacy projects detected",
+                    projects_total=len(legacy_projects),
+                )
+                return True
+            else:
+                self._state = MigrationState(
+                    status=MigrationStatus.NOT_NEEDED, message="No migration required"
+                )
+                return False
+
+        except Exception as e:
+            logger.error(f"Error checking migration status: {e}")
+            self._state = MigrationState(
+                status=MigrationStatus.FAILED, message="Migration check failed", error=str(e)
+            )
+            return False
+
+    async def start_background_migration(self, app_config: BasicMemoryConfig) -> None:
+        """Start migration in background if needed."""
+        if not await self.check_migration_needed(app_config):
+            return
+
+        if self._migration_task and not self._migration_task.done():
+            logger.info("Migration already in progress")
+            return
+
+        logger.info("Starting background migration")
+        self._migration_task = asyncio.create_task(self._run_migration(app_config))
+
+    async def _run_migration(self, app_config: BasicMemoryConfig) -> None:
+        """Run the actual migration process."""
+        try:
+            self._state.status = MigrationStatus.IN_PROGRESS
+            self._state.message = "Migrating legacy projects"
+
+            # Import here to avoid circular imports
+            from basic_memory.services.initialization import migrate_legacy_projects
+
+            # Run the migration
+            await migrate_legacy_projects(app_config)
+
+            self._state = MigrationState(
+                status=MigrationStatus.COMPLETED, message="Migration completed successfully"
+            )
+            logger.info("Background migration completed successfully")
+
+        except Exception as e:
+            logger.error(f"Background migration failed: {e}")
+            self._state = MigrationState(
+                status=MigrationStatus.FAILED, message="Migration failed", error=str(e)
+            )
+
+    async def wait_for_completion(self, timeout: Optional[float] = None) -> bool:
+        """Wait for migration to complete."""
+        if self.is_ready:
+            return True
+
+        if not self._migration_task:
+            return False
+
+        try:
+            await asyncio.wait_for(self._migration_task, timeout=timeout)
+            return self.is_ready
+        except asyncio.TimeoutError:
+            return False
+
+    def mark_completed(self, message: str = "Migration completed") -> None:
+        """Mark migration as completed externally."""
+        self._state = MigrationState(status=MigrationStatus.COMPLETED, message=message)
+
+
+# Global migration manager instance
+migration_manager = MigrationManager()

basic_memory/services/project_service.py

@@ -159,7 +159,9 @@ class ProjectService:
         multiple projects might have is_default=True or no project is marked as default.
         """
         if not self.repository:
-            raise ValueError("Repository is required for _ensure_single_default_project") # pragma: no cover
+            raise ValueError(
+                "Repository is required for _ensure_single_default_project"
+            )  # pragma: no cover
 
         # Get all projects with is_default=True
         db_projects = await self.repository.find_all()
@@ -207,8 +209,29 @@ class ProjectService:
         db_projects = await self.repository.get_active_projects()
         db_projects_by_name = {p.name: p for p in db_projects}
 
-        # Get all projects from configuration
-        config_projects = config_manager.projects
+        # Get all projects from configuration and normalize names if needed
+        config_projects = config_manager.projects.copy()
+        updated_config = {}
+        config_updated = False
+
+        for name, path in config_projects.items():
+            # Generate normalized name (what the database expects)
+            normalized_name = generate_permalink(name)
+
+            if normalized_name != name:
+                logger.info(f"Normalizing project name in config: '{name}' -> '{normalized_name}'")
+                config_updated = True
+
+            updated_config[normalized_name] = path
+
+        # Update the configuration if any changes were made
+        if config_updated:
+            config_manager.config.projects = updated_config
+            config_manager.save_config(config_manager.config)
+            logger.info("Config updated with normalized project names")
+
+        # Use the normalized config for further processing
+        config_projects = updated_config
 
         # Add projects that exist in config but not in DB
         for name, path in config_projects.items():
@@ -217,7 +240,7 @@ class ProjectService:
                 project_data = {
                     "name": name,
                     "path": path,
-                    "permalink": name.lower().replace(" ", "-"),
+                    "permalink": generate_permalink(name),
                     "is_active": True,
                     # Don't set is_default here - let the enforcement logic handle it
                 }
@@ -309,8 +332,11 @@ class ProjectService:
                     f"Changed default project to '{new_default.name}' as '{name}' was deactivated"
                 )
 
-    async def get_project_info(self) -> ProjectInfoResponse:
-        """Get comprehensive information about the current Basic Memory project.
+    async def get_project_info(self, project_name: Optional[str] = None) -> ProjectInfoResponse:
+        """Get comprehensive information about the specified Basic Memory project.
+
+        Args:
+            project_name: Name of the project to get info for. If None, uses the current config project.
 
         Returns:
             Comprehensive project information and statistics
@@ -318,19 +344,27 @@ class ProjectService:
         if not self.repository:  # pragma: no cover
             raise ValueError("Repository is required for get_project_info")
 
-        # Get statistics
-        statistics = await self.get_statistics()
+        # Use specified project or fall back to config project
+        project_name = project_name or config.project
+        # Get project path from configuration
+        project_path = config_manager.projects.get(project_name)
+        if not project_path:  # pragma: no cover
+            raise ValueError(f"Project '{project_name}' not found in configuration")
 
-        # Get activity metrics
-        activity = await self.get_activity_metrics()
+        # Get project from database to get project_id
+        db_project = await self.repository.get_by_name(project_name)
+        if not db_project:  # pragma: no cover
+            raise ValueError(f"Project '{project_name}' not found in database")
+
+        # Get statistics for the specified project
+        statistics = await self.get_statistics(db_project.id)
+
+        # Get activity metrics for the specified project
+        activity = await self.get_activity_metrics(db_project.id)
 
         # Get system status
         system = self.get_system_status()
 
-        # Get current project information from config
-        project_name = config.project
-        project_path = str(config.home)
-
         # Get enhanced project information from database
         db_projects = await self.repository.get_active_projects()
         db_projects_by_name = {p.name: p for p in db_projects}
@@ -361,60 +395,85 @@ class ProjectService:
             system=system,
         )
 
-    async def get_statistics(self) -> ProjectStatistics:
-        """Get statistics about the current project."""
+    async def get_statistics(self, project_id: int) -> ProjectStatistics:
+        """Get statistics about the specified project.
+
+        Args:
+            project_id: ID of the project to get statistics for (required).
+        """
         if not self.repository:  # pragma: no cover
             raise ValueError("Repository is required for get_statistics")
 
         # Get basic counts
         entity_count_result = await self.repository.execute_query(
-            text("SELECT COUNT(*) FROM entity")
+            text("SELECT COUNT(*) FROM entity WHERE project_id = :project_id"),
+            {"project_id": project_id},
         )
         total_entities = entity_count_result.scalar() or 0
 
         observation_count_result = await self.repository.execute_query(
-            text("SELECT COUNT(*) FROM observation")
+            text(
+                "SELECT COUNT(*) FROM observation o JOIN entity e ON o.entity_id = e.id WHERE e.project_id = :project_id"
+            ),
+            {"project_id": project_id},
         )
         total_observations = observation_count_result.scalar() or 0
 
         relation_count_result = await self.repository.execute_query(
-            text("SELECT COUNT(*) FROM relation")
+            text(
+                "SELECT COUNT(*) FROM relation r JOIN entity e ON r.from_id = e.id WHERE e.project_id = :project_id"
+            ),
+            {"project_id": project_id},
         )
         total_relations = relation_count_result.scalar() or 0
 
         unresolved_count_result = await self.repository.execute_query(
-            text("SELECT COUNT(*) FROM relation WHERE to_id IS NULL")
+            text(
+                "SELECT COUNT(*) FROM relation r JOIN entity e ON r.from_id = e.id WHERE r.to_id IS NULL AND e.project_id = :project_id"
+            ),
+            {"project_id": project_id},
         )
         total_unresolved = unresolved_count_result.scalar() or 0
 
         # Get entity counts by type
         entity_types_result = await self.repository.execute_query(
-            text("SELECT entity_type, COUNT(*) FROM entity GROUP BY entity_type")
+            text(
+                "SELECT entity_type, COUNT(*) FROM entity WHERE project_id = :project_id GROUP BY entity_type"
+            ),
+            {"project_id": project_id},
         )
         entity_types = {row[0]: row[1] for row in entity_types_result.fetchall()}
 
         # Get observation counts by category
         category_result = await self.repository.execute_query(
-            text("SELECT category, COUNT(*) FROM observation GROUP BY category")
+            text(
+                "SELECT o.category, COUNT(*) FROM observation o JOIN entity e ON o.entity_id = e.id WHERE e.project_id = :project_id GROUP BY o.category"
+            ),
+            {"project_id": project_id},
         )
         observation_categories = {row[0]: row[1] for row in category_result.fetchall()}
 
         # Get relation counts by type
         relation_types_result = await self.repository.execute_query(
-            text("SELECT relation_type, COUNT(*) FROM relation GROUP BY relation_type")
+            text(
+                "SELECT r.relation_type, COUNT(*) FROM relation r JOIN entity e ON r.from_id = e.id WHERE e.project_id = :project_id GROUP BY r.relation_type"
+            ),
+            {"project_id": project_id},
         )
         relation_types = {row[0]: row[1] for row in relation_types_result.fetchall()}
 
-        # Find most connected entities (most outgoing relations)
+        # Find most connected entities (most outgoing relations) - project filtered
         connected_result = await self.repository.execute_query(
             text("""
-            SELECT e.id, e.title, e.permalink, COUNT(r.id) AS relation_count, file_path
+            SELECT e.id, e.title, e.permalink, COUNT(r.id) AS relation_count, e.file_path
             FROM entity e
             JOIN relation r ON e.id = r.from_id
+            WHERE e.project_id = :project_id
             GROUP BY e.id
             ORDER BY relation_count DESC
             LIMIT 10
-        """)
+        """),
+            {"project_id": project_id},
         )
         most_connected = [
             {
@@ -427,15 +486,16 @@ class ProjectService:
             for row in connected_result.fetchall()
         ]
 
-        # Count isolated entities (no relations)
+        # Count isolated entities (no relations) - project filtered
         isolated_result = await self.repository.execute_query(
             text("""
             SELECT COUNT(e.id)
             FROM entity e
             LEFT JOIN relation r1 ON e.id = r1.from_id
             LEFT JOIN relation r2 ON e.id = r2.to_id
-            WHERE r1.id IS NULL AND r2.id IS NULL
-        """)
+            WHERE e.project_id = :project_id AND r1.id IS NULL AND r2.id IS NULL
+        """),
+            {"project_id": project_id},
         )
         isolated_count = isolated_result.scalar() or 0
 
@@ -451,19 +511,25 @@ class ProjectService:
             isolated_entities=isolated_count,
         )
 
-    async def get_activity_metrics(self) -> ActivityMetrics:
-        """Get activity metrics for the current project."""
+    async def get_activity_metrics(self, project_id: int) -> ActivityMetrics:
+        """Get activity metrics for the specified project.
+
+        Args:
+            project_id: ID of the project to get activity metrics for (required).
+        """
         if not self.repository:  # pragma: no cover
             raise ValueError("Repository is required for get_activity_metrics")
 
-        # Get recently created entities
+        # Get recently created entities (project filtered)
         created_result = await self.repository.execute_query(
             text("""
             SELECT id, title, permalink, entity_type, created_at, file_path 
             FROM entity
+            WHERE project_id = :project_id
             ORDER BY created_at DESC
             LIMIT 10
-        """)
+        """),
+            {"project_id": project_id},
         )
         recently_created = [
             {
@@ -477,14 +543,16 @@ class ProjectService:
             for row in created_result.fetchall()
         ]
 
-        # Get recently updated entities
+        # Get recently updated entities (project filtered)
         updated_result = await self.repository.execute_query(
             text("""
             SELECT id, title, permalink, entity_type, updated_at, file_path 
             FROM entity
+            WHERE project_id = :project_id
             ORDER BY updated_at DESC
             LIMIT 10
-        """)
+        """),
+            {"project_id": project_id},
         )
         recently_updated = [
             {
@@ -505,47 +573,50 @@ class ProjectService:
             now.year - (1 if now.month <= 6 else 0), ((now.month - 6) % 12) or 12, 1
         )
 
-        # Query for monthly entity creation
+        # Query for monthly entity creation (project filtered)
         entity_growth_result = await self.repository.execute_query(
-            text(f"""
+            text("""
             SELECT 
                 strftime('%Y-%m', created_at) AS month,
                 COUNT(*) AS count
             FROM entity
-            WHERE created_at >= '{six_months_ago.isoformat()}'
+            WHERE created_at >= :six_months_ago AND project_id = :project_id
             GROUP BY month
             ORDER BY month
-        """)
+        """),
+            {"six_months_ago": six_months_ago.isoformat(), "project_id": project_id},
         )
         entity_growth = {row[0]: row[1] for row in entity_growth_result.fetchall()}
 
-        # Query for monthly observation creation
+        # Query for monthly observation creation (project filtered)
         observation_growth_result = await self.repository.execute_query(
-            text(f"""
+            text("""
             SELECT 
-                strftime('%Y-%m', created_at) AS month,
+                strftime('%Y-%m', entity.created_at) AS month,
                 COUNT(*) AS count
             FROM observation
             INNER JOIN entity ON observation.entity_id = entity.id
-            WHERE entity.created_at >= '{six_months_ago.isoformat()}'
+            WHERE entity.created_at >= :six_months_ago AND entity.project_id = :project_id
             GROUP BY month
             ORDER BY month
-        """)
+        """),
+            {"six_months_ago": six_months_ago.isoformat(), "project_id": project_id},
         )
         observation_growth = {row[0]: row[1] for row in observation_growth_result.fetchall()}
 
-        # Query for monthly relation creation
+        # Query for monthly relation creation (project filtered)
         relation_growth_result = await self.repository.execute_query(
-            text(f"""
+            text("""
             SELECT 
-                strftime('%Y-%m', created_at) AS month,
+                strftime('%Y-%m', entity.created_at) AS month,
                 COUNT(*) AS count
             FROM relation
             INNER JOIN entity ON relation.from_id = entity.id
-            WHERE entity.created_at >= '{six_months_ago.isoformat()}'
+            WHERE entity.created_at >= :six_months_ago AND entity.project_id = :project_id
             GROUP BY month
             ORDER BY month
-        """)
+        """),
+            {"six_months_ago": six_months_ago.isoformat(), "project_id": project_id},
         )
         relation_growth = {row[0]: row[1] for row in relation_growth_result.fetchall()}
 
@@ -597,4 +668,4 @@ class ProjectService:
             database_size=db_size_readable,
             watch_status=watch_status,
             timestamp=datetime.now(),
-        )
+        )

basic_memory/services/sync_status_service.py

@@ -0,0 +1,181 @@
+"""Simple sync status tracking service."""
+
+from dataclasses import dataclass
+from enum import Enum
+from typing import Dict, Optional
+
+
+class SyncStatus(Enum):
+    """Status of sync operations."""
+
+    IDLE = "idle"
+    SCANNING = "scanning"
+    SYNCING = "syncing"
+    COMPLETED = "completed"
+    FAILED = "failed"
+    WATCHING = "watching"
+
+
+@dataclass
+class ProjectSyncStatus:
+    """Sync status for a single project."""
+
+    project_name: str
+    status: SyncStatus
+    message: str = ""
+    files_total: int = 0
+    files_processed: int = 0
+    error: Optional[str] = None
+
+
+class SyncStatusTracker:
+    """Global tracker for all sync operations."""
+
+    def __init__(self):
+        self._project_statuses: Dict[str, ProjectSyncStatus] = {}
+        self._global_status: SyncStatus = SyncStatus.IDLE
+
+    def start_project_sync(self, project_name: str, files_total: int = 0) -> None:
+        """Start tracking sync for a project."""
+        self._project_statuses[project_name] = ProjectSyncStatus(
+            project_name=project_name,
+            status=SyncStatus.SCANNING,
+            message="Scanning files",
+            files_total=files_total,
+            files_processed=0,
+        )
+        self._update_global_status()
+
+    def update_project_progress(  # pragma: no cover
+        self,
+        project_name: str,
+        status: SyncStatus,
+        message: str = "",
+        files_processed: int = 0,
+        files_total: Optional[int] = None,
+    ) -> None:
+        """Update progress for a project."""
+        if project_name not in self._project_statuses:  # pragma: no cover
+            return
+
+        project_status = self._project_statuses[project_name]
+        project_status.status = status
+        project_status.message = message
+        project_status.files_processed = files_processed
+
+        if files_total is not None:
+            project_status.files_total = files_total
+
+        self._update_global_status()
+
+    def complete_project_sync(self, project_name: str) -> None:
+        """Mark project sync as completed."""
+        if project_name in self._project_statuses:
+            self._project_statuses[project_name].status = SyncStatus.COMPLETED
+            self._project_statuses[project_name].message = "Sync completed"
+            self._update_global_status()
+
+    def fail_project_sync(self, project_name: str, error: str) -> None:
+        """Mark project sync as failed."""
+        if project_name in self._project_statuses:
+            self._project_statuses[project_name].status = SyncStatus.FAILED
+            self._project_statuses[project_name].error = error
+            self._update_global_status()
+
+    def start_project_watch(self, project_name: str) -> None:
+        """Mark project as watching for changes (steady state after sync)."""
+        if project_name in self._project_statuses:
+            self._project_statuses[project_name].status = SyncStatus.WATCHING
+            self._project_statuses[project_name].message = "Watching for changes"
+            self._update_global_status()
+        else:
+            # Create new status if project isn't tracked yet
+            self._project_statuses[project_name] = ProjectSyncStatus(
+                project_name=project_name,
+                status=SyncStatus.WATCHING,
+                message="Watching for changes",
+                files_total=0,
+                files_processed=0,
+            )
+            self._update_global_status()
+
+    def _update_global_status(self) -> None:
+        """Update global status based on project statuses."""
+        if not self._project_statuses:  # pragma: no cover
+            self._global_status = SyncStatus.IDLE
+            return
+
+        statuses = [p.status for p in self._project_statuses.values()]
+
+        if any(s == SyncStatus.FAILED for s in statuses):
+            self._global_status = SyncStatus.FAILED
+        elif any(s in (SyncStatus.SCANNING, SyncStatus.SYNCING) for s in statuses):
+            self._global_status = SyncStatus.SYNCING
+        elif all(s in (SyncStatus.COMPLETED, SyncStatus.WATCHING) for s in statuses):
+            self._global_status = SyncStatus.COMPLETED
+        else:
+            self._global_status = SyncStatus.SYNCING
+
+    @property
+    def global_status(self) -> SyncStatus:
+        """Get overall sync status."""
+        return self._global_status
+
+    @property
+    def is_syncing(self) -> bool:
+        """Check if any sync operation is in progress."""
+        return self._global_status in (SyncStatus.SCANNING, SyncStatus.SYNCING)
+
+    @property
+    def is_ready(self) -> bool:  # pragma: no cover
+        """Check if system is ready (no sync in progress)."""
+        return self._global_status in (SyncStatus.IDLE, SyncStatus.COMPLETED)
+
+    def get_project_status(self, project_name: str) -> Optional[ProjectSyncStatus]:
+        """Get status for a specific project."""
+        return self._project_statuses.get(project_name)
+
+    def get_all_projects(self) -> Dict[str, ProjectSyncStatus]:
+        """Get all project statuses."""
+        return self._project_statuses.copy()
+
+    def get_summary(self) -> str:  # pragma: no cover
+        """Get a user-friendly summary of sync status."""
+        if self._global_status == SyncStatus.IDLE:
+            return "✅ System ready"
+        elif self._global_status == SyncStatus.COMPLETED:
+            return "✅ All projects synced successfully"
+        elif self._global_status == SyncStatus.FAILED:
+            failed_projects = [
+                p.project_name
+                for p in self._project_statuses.values()
+                if p.status == SyncStatus.FAILED
+            ]
+            return f"❌ Sync failed for: {', '.join(failed_projects)}"
+        else:
+            active_projects = [
+                p.project_name
+                for p in self._project_statuses.values()
+                if p.status in (SyncStatus.SCANNING, SyncStatus.SYNCING)
+            ]
+            total_files = sum(p.files_total for p in self._project_statuses.values())
+            processed_files = sum(p.files_processed for p in self._project_statuses.values())
+
+            if total_files > 0:
+                progress_pct = (processed_files / total_files) * 100
+                return f"🔄 Syncing {len(active_projects)} projects ({processed_files}/{total_files} files, {progress_pct:.0f}%)"
+            else:
+                return f"🔄 Syncing {len(active_projects)} projects"
+
+    def clear_completed(self) -> None:
+        """Remove completed project statuses to clean up memory."""
+        self._project_statuses = {
+            name: status
+            for name, status in self._project_statuses.items()
+            if status.status != SyncStatus.COMPLETED
+        }
+        self._update_global_status()
+
+
+# Global sync status tracker instance
+sync_status_tracker = SyncStatusTracker()