gobby 0.2.6__py3-none-any.whl → 0.2.8__py3-none-any.whl

gobby/memory/search/coordinator.py

@@ -0,0 +1,248 @@
+"""Search coordination for memory recall operations."""
+
+from __future__ import annotations
+
+import logging
+from typing import TYPE_CHECKING, Any
+
+from gobby.storage.memories import Memory
+
+if TYPE_CHECKING:
+    from gobby.config.persistence import MemoryConfig
+    from gobby.memory.search import SearchBackend
+    from gobby.storage.database import DatabaseProtocol
+    from gobby.storage.memories import LocalMemoryManager
+
+logger = logging.getLogger(__name__)
+
+
+class SearchCoordinator:
+    """
+    Coordinates search operations for memory recall.
+
+    Manages the search backend lifecycle, fitting, and query execution.
+    Extracts search-related logic from MemoryManager for focused responsibility.
+    """
+
+    def __init__(
+        self,
+        storage: LocalMemoryManager,
+        config: MemoryConfig,
+        db: DatabaseProtocol,
+    ):
+        """
+        Initialize the search coordinator.
+
+        Args:
+            storage: Memory storage manager for accessing memories
+            config: Memory configuration for search settings
+            db: Database connection for search backend initialization
+        """
+        self._storage = storage
+        self._config = config
+        self._db = db
+        self._search_backend: SearchBackend | None = None
+        self._search_backend_fitted = False
+
+    @property
+    def search_backend(self) -> SearchBackend:
+        """
+        Lazy-init search backend based on configuration.
+
+        The backend type is determined by config.search_backend:
+        - "tfidf" (default): Zero-dependency TF-IDF search
+        - "text": Simple text substring matching
+        """
+        if self._search_backend is None:
+            from gobby.memory.search import get_search_backend
+
+            backend_type = getattr(self._config, "search_backend", "tfidf")
+            logger.debug(f"Initializing search backend: {backend_type}")
+
+            try:
+                self._search_backend = get_search_backend(
+                    backend_type=backend_type,
+                    db=self._db,
+                )
+            except Exception as e:
+                logger.warning(
+                    f"Failed to initialize {backend_type} backend: {e}. Falling back to tfidf"
+                )
+                self._search_backend = get_search_backend("tfidf")
+
+        return self._search_backend
+
+    def ensure_fitted(self) -> None:
+        """Ensure the search backend is fitted with current memories."""
+        if self._search_backend_fitted:
+            return
+
+        backend = self.search_backend
+        if not backend.needs_refit():
+            self._search_backend_fitted = True
+            return
+
+        # Fit the backend with all memories
+        max_memories = getattr(self._config, "max_index_memories", 10000)
+        memories = self._storage.list_memories(limit=max_memories)
+        memory_tuples = [(m.id, m.content) for m in memories]
+
+        try:
+            backend.fit(memory_tuples)
+            self._search_backend_fitted = True
+            logger.info(f"Search backend fitted with {len(memory_tuples)} memories")
+        except Exception as e:
+            logger.error(f"Failed to fit search backend: {e}")
+            raise
+
+    def mark_refit_needed(self) -> None:
+        """Mark that the search backend needs to be refitted."""
+        self._search_backend_fitted = False
+
+    def reindex(self) -> dict[str, Any]:
+        """
+        Force rebuild of the search index.
+
+        This method explicitly rebuilds the TF-IDF (or other configured)
+        search index from all stored memories. Useful for:
+        - Initial index building
+        - Recovery after corruption
+        - After bulk memory operations
+
+        Returns:
+            Dict with index statistics including memory_count, backend_type, etc.
+        """
+        # Get all memories
+        memories = self._storage.list_memories(limit=10000)
+        memory_tuples = [(m.id, m.content) for m in memories]
+
+        # Force refit the backend
+        backend = self.search_backend
+        backend_type = getattr(self._config, "search_backend", "tfidf")
+
+        try:
+            backend.fit(memory_tuples)
+            self._search_backend_fitted = True
+
+            # Get backend stats
+            stats = backend.get_stats() if hasattr(backend, "get_stats") else {}
+
+            return {
+                "success": True,
+                "memory_count": len(memory_tuples),
+                "backend_type": backend_type,
+                "fitted": True,
+                **stats,
+            }
+        except Exception as e:
+            logger.error(f"Failed to reindex search backend: {e}")
+            return {
+                "success": False,
+                "error": str(e),
+                "memory_count": len(memory_tuples),
+                "backend_type": backend_type,
+            }
+
+    def search(
+        self,
+        query: str,
+        project_id: str | None = None,
+        limit: int = 10,
+        min_importance: float | None = None,
+        search_mode: str | None = None,
+        tags_all: list[str] | None = None,
+        tags_any: list[str] | None = None,
+        tags_none: list[str] | None = None,
+    ) -> list[Memory]:
+        """
+        Perform search using the configured search backend.
+
+        Uses the new search backend by default (TF-IDF),
+        falling back to legacy semantic search if configured.
+
+        Args:
+            query: Search query text
+            project_id: Filter by project
+            limit: Maximum results to return
+            min_importance: Minimum importance threshold
+            search_mode: Search mode (tfidf, text, etc.)
+            tags_all: Memory must have ALL of these tags
+            tags_any: Memory must have at least ONE of these tags
+            tags_none: Memory must have NONE of these tags
+
+        Returns:
+            List of matching Memory objects
+        """
+        # Determine search mode from config or parameters
+        if search_mode is None:
+            search_mode = getattr(self._config, "search_backend", "tfidf")
+
+        # Use the search backend
+        try:
+            self.ensure_fitted()
+            # Fetch more results to allow for filtering
+            fetch_multiplier = 3 if (tags_all or tags_any or tags_none) else 2
+            results = self.search_backend.search(query, top_k=limit * fetch_multiplier)
+
+            # Get the actual Memory objects
+            memory_ids = [mid for mid, _ in results]
+            memories = []
+            for mid in memory_ids:
+                memory = self._storage.get_memory(mid)
+                if memory:
+                    # Apply filters - allow global memories (project_id is None) to pass through
+                    if (
+                        project_id
+                        and memory.project_id is not None
+                        and memory.project_id != project_id
+                    ):
+                        continue
+                    if min_importance is not None and memory.importance < min_importance:
+                        continue
+                    # Apply tag filters
+                    if not self._passes_tag_filter(memory, tags_all, tags_any, tags_none):
+                        continue
+                    memories.append(memory)
+                    if len(memories) >= limit:
+                        break
+
+            return memories
+
+        except Exception as e:
+            logger.warning(f"Search backend failed, falling back to text search: {e}")
+            # Fall back to text search with tag filtering
+            memories = self._storage.search_memories(
+                query_text=query,
+                project_id=project_id,
+                limit=limit * 2,
+                tags_all=tags_all,
+                tags_any=tags_any,
+                tags_none=tags_none,
+            )
+            if min_importance:
+                memories = [m for m in memories if m.importance >= min_importance]
+            return memories[:limit]
+
+    def _passes_tag_filter(
+        self,
+        memory: Memory,
+        tags_all: list[str] | None = None,
+        tags_any: list[str] | None = None,
+        tags_none: list[str] | None = None,
+    ) -> bool:
+        """Check if a memory passes the tag filter criteria."""
+        memory_tags = set(memory.tags) if memory.tags else set()
+
+        # Check tags_all: memory must have ALL specified tags
+        if tags_all and not set(tags_all).issubset(memory_tags):
+            return False
+
+        # Check tags_any: memory must have at least ONE specified tag
+        if tags_any and not memory_tags.intersection(tags_any):
+            return False
+
+        # Check tags_none: memory must have NONE of the specified tags
+        if tags_none and memory_tags.intersection(tags_none):
+            return False
+
+        return True

gobby/memory/services/__init__.py

@@ -0,0 +1,5 @@
+"""Memory services for cross-referencing and linking."""
+
+from gobby.memory.services.crossref import CrossrefService
+
+__all__ = ["CrossrefService"]

gobby/memory/services/crossref.py

@@ -0,0 +1,142 @@
+"""Cross-reference service for linking related memories."""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+from collections.abc import Callable
+from typing import TYPE_CHECKING
+
+from gobby.storage.memories import Memory
+
+if TYPE_CHECKING:
+    from gobby.config.persistence import MemoryConfig
+    from gobby.memory.search import SearchBackend
+    from gobby.storage.memories import LocalMemoryManager
+
+logger = logging.getLogger(__name__)
+
+
+class CrossrefService:
+    """
+    Service for creating and managing cross-references between memories.
+
+    Cross-references link related memories based on content similarity,
+    enabling navigation between conceptually connected items.
+    """
+
+    def __init__(
+        self,
+        storage: LocalMemoryManager,
+        config: MemoryConfig,
+        search_backend_getter: Callable[[], SearchBackend],
+        ensure_fitted: Callable[[], None],
+    ):
+        """
+        Initialize the cross-reference service.
+
+        Args:
+            storage: Memory storage manager for persistence
+            config: Memory configuration for thresholds
+            search_backend_getter: Callable that returns the search backend
+            ensure_fitted: Callable that ensures search backend is fitted
+        """
+        self._storage = storage
+        self._config = config
+        self._get_search_backend = search_backend_getter
+        self._ensure_fitted = ensure_fitted
+
+    async def create_crossrefs(
+        self,
+        memory: Memory,
+        threshold: float | None = None,
+        max_links: int | None = None,
+    ) -> int:
+        """
+        Find and link similar memories.
+
+        Uses the search backend to find memories similar to the given one
+        and creates cross-references for those above the threshold.
+
+        Args:
+            memory: The memory to find links for
+            threshold: Minimum similarity to create link (default from config)
+            max_links: Maximum links to create (default from config)
+
+        Returns:
+            Number of cross-references created
+        """
+        # Get thresholds from config or use defaults
+        if threshold is None:
+            threshold = getattr(self._config, "crossref_threshold", None)
+            if threshold is None:
+                threshold = 0.3
+        if max_links is None:
+            max_links = getattr(self._config, "crossref_max_links", None)
+            if max_links is None:
+                max_links = 5
+
+        # Ensure search backend is fitted (sync check is fine - just checks a flag)
+        self._ensure_fitted()
+
+        # Search for similar memories (wrap sync I/O in to_thread)
+        search_backend = self._get_search_backend()
+        similar = await asyncio.to_thread(search_backend.search, memory.content, max_links + 1)
+
+        # Create cross-references
+        created = 0
+        for other_id, score in similar:
+            # Skip self-reference
+            if other_id == memory.id:
+                continue
+
+            # Skip below threshold
+            if score < threshold:
+                continue
+
+            # Create the crossref (wrap sync I/O in to_thread)
+            await asyncio.to_thread(self._storage.create_crossref, memory.id, other_id, score)
+            created += 1
+
+            if created >= max_links:
+                break
+
+        if created > 0:
+            logger.debug(f"Created {created} crossrefs for memory {memory.id}")
+
+        return created
+
+    async def get_related(
+        self,
+        memory_id: str,
+        limit: int = 5,
+        min_similarity: float = 0.0,
+    ) -> list[Memory]:
+        """
+        Get memories linked to this one via cross-references.
+
+        Args:
+            memory_id: The memory ID to find related memories for
+            limit: Maximum number of results
+            min_similarity: Minimum similarity threshold
+
+        Returns:
+            List of related Memory objects, sorted by similarity
+        """
+        crossrefs = await asyncio.to_thread(
+            self._storage.get_crossrefs,
+            memory_id,
+            limit=limit,
+            min_similarity=min_similarity,
+        )
+
+        # Get the actual Memory objects
+        memories = []
+        for ref in crossrefs:
+            # Get the "other" memory in the relationship
+            other_id = ref.target_id if ref.source_id == memory_id else ref.source_id
+            memory = await asyncio.to_thread(self._storage.get_memory, other_id)
+            if memory:
+                memories.append(memory)
+
+        return memories

gobby/prompts/loader.py

@@ -23,8 +23,11 @@ from .models import PromptTemplate
 
 logger = logging.getLogger(__name__)
 
-# Default location for bundled prompts
-DEFAULTS_DIR = Path(__file__).parent / "defaults"
+# Default location for bundled prompts (in install/shared/prompts for installation)
+# Falls back to src location for development if install location doesn't exist
+_INSTALL_PROMPTS_DIR = Path(__file__).parent.parent / "install" / "shared" / "prompts"
+_DEV_PROMPTS_DIR = Path(__file__).parent / "defaults"
+DEFAULTS_DIR = _INSTALL_PROMPTS_DIR if _INSTALL_PROMPTS_DIR.exists() else _DEV_PROMPTS_DIR
 
 
 class PromptLoader:

gobby/runner.py

@@ -471,6 +471,19 @@ class GobbyRunner:
                 except (asyncio.CancelledError, TimeoutError):
                     pass
 
+            # Export memories to JSONL backup on shutdown
+            if self.memory_sync_manager:
+                try:
+                    count = await asyncio.wait_for(
+                        self.memory_sync_manager.export_to_files(), timeout=5.0
+                    )
+                    if count > 0:
+                        logger.info(f"Shutdown memory backup: exported {count} memories")
+                except TimeoutError:
+                    logger.warning("Memory backup on shutdown timed out")
+                except Exception as e:
+                    logger.warning(f"Memory backup on shutdown failed: {e}")
+
             try:
                 await asyncio.wait_for(self.mcp_proxy.disconnect_all(), timeout=3.0)
             except TimeoutError:

gobby/servers/http.py

@@ -16,7 +16,7 @@ from fastapi import FastAPI, HTTPException, Request
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.responses import JSONResponse
 
-from gobby.adapters.codex import CodexAdapter
+from gobby.adapters.codex_impl.adapter import CodexAdapter
 from gobby.hooks.broadcaster import HookEventBroadcaster
 from gobby.hooks.hook_manager import HookManager
 from gobby.llm import LLMService, create_llm_service
@@ -25,9 +25,6 @@ from gobby.mcp_proxy.semantic_search import SemanticToolSearch
 from gobby.mcp_proxy.server import GobbyDaemonTools, create_mcp_server
 from gobby.mcp_proxy.services.tool_filter import ToolFilterService
 from gobby.memory.manager import MemoryManager
-
-# Re-export for backward compatibility
-from gobby.servers.models import SessionRegisterRequest  # noqa: F401
 from gobby.storage.sessions import LocalSessionManager
 from gobby.storage.tasks import LocalTaskManager
 from gobby.sync.tasks import TaskSyncManager

gobby/servers/routes/admin.py

@@ -175,6 +175,19 @@ def create_admin_router(server: "HTTPServer") -> APIRouter:
             except Exception as e:
                 logger.warning(f"Failed to get memory stats: {e}")
 
+        # Get skills statistics
+        skills_stats: dict[str, Any] = {"total": 0}
+        if server._internal_manager:
+            try:
+                for registry in server._internal_manager.get_all_registries():
+                    if registry.name == "gobby-skills":
+                        result = await registry.call("list_skills", {"limit": 10000})
+                        if result.get("success"):
+                            skills_stats["total"] = result.get("count", 0)
+                        break
+            except Exception as e:
+                logger.warning(f"Failed to get skills stats: {e}")
+
         # Get plugin status
         plugin_stats: dict[str, Any] = {"enabled": False, "loaded": 0, "handlers": 0}
         if hasattr(server, "_hook_manager") and server._hook_manager is not None:
@@ -218,6 +231,7 @@ def create_admin_router(server: "HTTPServer") -> APIRouter:
             "sessions": session_stats,
             "tasks": task_stats,
             "memory": memory_stats,
+            "skills": skills_stats,
             "plugins": plugin_stats,
             "response_time_ms": response_time_ms,
         }

gobby/servers/routes/mcp/endpoints/__init__.py

@@ -0,0 +1,61 @@
+"""
+MCP endpoint modules for the Gobby HTTP server.
+
+This package contains decomposed endpoint handlers extracted from tools.py
+using the Strangler Fig pattern. Each module handles a specific domain:
+
+- discovery: Tool and server listing endpoints
+- execution: Tool invocation endpoints
+- server: Server management (add/remove/import)
+- registry: Tool embedding, status, and refresh
+
+External modules should import `create_mcp_router` from the parent package:
+    from gobby.servers.routes.mcp import create_mcp_router
+
+For direct endpoint access (e.g., testing), import from submodules:
+    from gobby.servers.routes.mcp.endpoints.discovery import list_all_mcp_tools
+"""
+
+from gobby.servers.routes.mcp.endpoints.discovery import (
+    list_all_mcp_tools,
+    recommend_mcp_tools,
+    search_mcp_tools,
+)
+from gobby.servers.routes.mcp.endpoints.execution import (
+    call_mcp_tool,
+    get_tool_schema,
+    list_mcp_tools,
+    mcp_proxy,
+)
+from gobby.servers.routes.mcp.endpoints.registry import (
+    embed_mcp_tools,
+    get_mcp_status,
+    refresh_mcp_tools,
+)
+from gobby.servers.routes.mcp.endpoints.server import (
+    add_mcp_server,
+    import_mcp_server,
+    list_mcp_servers,
+    remove_mcp_server,
+)
+
+__all__ = [
+    # Discovery
+    "list_all_mcp_tools",
+    "recommend_mcp_tools",
+    "search_mcp_tools",
+    # Execution
+    "call_mcp_tool",
+    "get_tool_schema",
+    "list_mcp_tools",
+    "mcp_proxy",
+    # Registry
+    "embed_mcp_tools",
+    "get_mcp_status",
+    "refresh_mcp_tools",
+    # Server
+    "add_mcp_server",
+    "import_mcp_server",
+    "list_mcp_servers",
+    "remove_mcp_server",
+]