gobby 0.2.5__py3-none-any.whl → 0.2.7__py3-none-any.whl

gobby/memory/search/coordinator.py

@@ -0,0 +1,248 @@
+"""Search coordination for memory recall operations."""
+
+from __future__ import annotations
+
+import logging
+from typing import TYPE_CHECKING, Any
+
+from gobby.storage.memories import Memory
+
+if TYPE_CHECKING:
+    from gobby.config.persistence import MemoryConfig
+    from gobby.memory.search import SearchBackend
+    from gobby.storage.database import DatabaseProtocol
+    from gobby.storage.memories import LocalMemoryManager
+
+logger = logging.getLogger(__name__)
+
+
+class SearchCoordinator:
+    """
+    Coordinates search operations for memory recall.
+
+    Manages the search backend lifecycle, fitting, and query execution.
+    Extracts search-related logic from MemoryManager for focused responsibility.
+    """
+
+    def __init__(
+        self,
+        storage: LocalMemoryManager,
+        config: MemoryConfig,
+        db: DatabaseProtocol,
+    ):
+        """
+        Initialize the search coordinator.
+
+        Args:
+            storage: Memory storage manager for accessing memories
+            config: Memory configuration for search settings
+            db: Database connection for search backend initialization
+        """
+        self._storage = storage
+        self._config = config
+        self._db = db
+        self._search_backend: SearchBackend | None = None
+        self._search_backend_fitted = False
+
+    @property
+    def search_backend(self) -> SearchBackend:
+        """
+        Lazy-init search backend based on configuration.
+
+        The backend type is determined by config.search_backend:
+        - "tfidf" (default): Zero-dependency TF-IDF search
+        - "text": Simple text substring matching
+        """
+        if self._search_backend is None:
+            from gobby.memory.search import get_search_backend
+
+            backend_type = getattr(self._config, "search_backend", "tfidf")
+            logger.debug(f"Initializing search backend: {backend_type}")
+
+            try:
+                self._search_backend = get_search_backend(
+                    backend_type=backend_type,
+                    db=self._db,
+                )
+            except Exception as e:
+                logger.warning(
+                    f"Failed to initialize {backend_type} backend: {e}. Falling back to tfidf"
+                )
+                self._search_backend = get_search_backend("tfidf")
+
+        return self._search_backend
+
+    def ensure_fitted(self) -> None:
+        """Ensure the search backend is fitted with current memories."""
+        if self._search_backend_fitted:
+            return
+
+        backend = self.search_backend
+        if not backend.needs_refit():
+            self._search_backend_fitted = True
+            return
+
+        # Fit the backend with all memories
+        max_memories = getattr(self._config, "max_index_memories", 10000)
+        memories = self._storage.list_memories(limit=max_memories)
+        memory_tuples = [(m.id, m.content) for m in memories]
+
+        try:
+            backend.fit(memory_tuples)
+            self._search_backend_fitted = True
+            logger.info(f"Search backend fitted with {len(memory_tuples)} memories")
+        except Exception as e:
+            logger.error(f"Failed to fit search backend: {e}")
+            raise
+
+    def mark_refit_needed(self) -> None:
+        """Mark that the search backend needs to be refitted."""
+        self._search_backend_fitted = False
+
+    def reindex(self) -> dict[str, Any]:
+        """
+        Force rebuild of the search index.
+
+        This method explicitly rebuilds the TF-IDF (or other configured)
+        search index from all stored memories. Useful for:
+        - Initial index building
+        - Recovery after corruption
+        - After bulk memory operations
+
+        Returns:
+            Dict with index statistics including memory_count, backend_type, etc.
+        """
+        # Get all memories
+        memories = self._storage.list_memories(limit=10000)
+        memory_tuples = [(m.id, m.content) for m in memories]
+
+        # Force refit the backend
+        backend = self.search_backend
+        backend_type = getattr(self._config, "search_backend", "tfidf")
+
+        try:
+            backend.fit(memory_tuples)
+            self._search_backend_fitted = True
+
+            # Get backend stats
+            stats = backend.get_stats() if hasattr(backend, "get_stats") else {}
+
+            return {
+                "success": True,
+                "memory_count": len(memory_tuples),
+                "backend_type": backend_type,
+                "fitted": True,
+                **stats,
+            }
+        except Exception as e:
+            logger.error(f"Failed to reindex search backend: {e}")
+            return {
+                "success": False,
+                "error": str(e),
+                "memory_count": len(memory_tuples),
+                "backend_type": backend_type,
+            }
+
+    def search(
+        self,
+        query: str,
+        project_id: str | None = None,
+        limit: int = 10,
+        min_importance: float | None = None,
+        search_mode: str | None = None,
+        tags_all: list[str] | None = None,
+        tags_any: list[str] | None = None,
+        tags_none: list[str] | None = None,
+    ) -> list[Memory]:
+        """
+        Perform search using the configured search backend.
+
+        Uses the new search backend by default (TF-IDF),
+        falling back to legacy semantic search if configured.
+
+        Args:
+            query: Search query text
+            project_id: Filter by project
+            limit: Maximum results to return
+            min_importance: Minimum importance threshold
+            search_mode: Search mode (tfidf, text, etc.)
+            tags_all: Memory must have ALL of these tags
+            tags_any: Memory must have at least ONE of these tags
+            tags_none: Memory must have NONE of these tags
+
+        Returns:
+            List of matching Memory objects
+        """
+        # Determine search mode from config or parameters
+        if search_mode is None:
+            search_mode = getattr(self._config, "search_backend", "tfidf")
+
+        # Use the search backend
+        try:
+            self.ensure_fitted()
+            # Fetch more results to allow for filtering
+            fetch_multiplier = 3 if (tags_all or tags_any or tags_none) else 2
+            results = self.search_backend.search(query, top_k=limit * fetch_multiplier)
+
+            # Get the actual Memory objects
+            memory_ids = [mid for mid, _ in results]
+            memories = []
+            for mid in memory_ids:
+                memory = self._storage.get_memory(mid)
+                if memory:
+                    # Apply filters - allow global memories (project_id is None) to pass through
+                    if (
+                        project_id
+                        and memory.project_id is not None
+                        and memory.project_id != project_id
+                    ):
+                        continue
+                    if min_importance is not None and memory.importance < min_importance:
+                        continue
+                    # Apply tag filters
+                    if not self._passes_tag_filter(memory, tags_all, tags_any, tags_none):
+                        continue
+                    memories.append(memory)
+                    if len(memories) >= limit:
+                        break
+
+            return memories
+
+        except Exception as e:
+            logger.warning(f"Search backend failed, falling back to text search: {e}")
+            # Fall back to text search with tag filtering
+            memories = self._storage.search_memories(
+                query_text=query,
+                project_id=project_id,
+                limit=limit * 2,
+                tags_all=tags_all,
+                tags_any=tags_any,
+                tags_none=tags_none,
+            )
+            if min_importance:
+                memories = [m for m in memories if m.importance >= min_importance]
+            return memories[:limit]
+
+    def _passes_tag_filter(
+        self,
+        memory: Memory,
+        tags_all: list[str] | None = None,
+        tags_any: list[str] | None = None,
+        tags_none: list[str] | None = None,
+    ) -> bool:
+        """Check if a memory passes the tag filter criteria."""
+        memory_tags = set(memory.tags) if memory.tags else set()
+
+        # Check tags_all: memory must have ALL specified tags
+        if tags_all and not set(tags_all).issubset(memory_tags):
+            return False
+
+        # Check tags_any: memory must have at least ONE specified tag
+        if tags_any and not memory_tags.intersection(tags_any):
+            return False
+
+        # Check tags_none: memory must have NONE of the specified tags
+        if tags_none and memory_tags.intersection(tags_none):
+            return False
+
+        return True

gobby/memory/services/__init__.py

@@ -0,0 +1,5 @@
+"""Memory services for cross-referencing and linking."""
+
+from gobby.memory.services.crossref import CrossrefService
+
+__all__ = ["CrossrefService"]

gobby/memory/services/crossref.py

@@ -0,0 +1,142 @@
+"""Cross-reference service for linking related memories."""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+from collections.abc import Callable
+from typing import TYPE_CHECKING
+
+from gobby.storage.memories import Memory
+
+if TYPE_CHECKING:
+    from gobby.config.persistence import MemoryConfig
+    from gobby.memory.search import SearchBackend
+    from gobby.storage.memories import LocalMemoryManager
+
+logger = logging.getLogger(__name__)
+
+
+class CrossrefService:
+    """
+    Service for creating and managing cross-references between memories.
+
+    Cross-references link related memories based on content similarity,
+    enabling navigation between conceptually connected items.
+    """
+
+    def __init__(
+        self,
+        storage: LocalMemoryManager,
+        config: MemoryConfig,
+        search_backend_getter: Callable[[], SearchBackend],
+        ensure_fitted: Callable[[], None],
+    ):
+        """
+        Initialize the cross-reference service.
+
+        Args:
+            storage: Memory storage manager for persistence
+            config: Memory configuration for thresholds
+            search_backend_getter: Callable that returns the search backend
+            ensure_fitted: Callable that ensures search backend is fitted
+        """
+        self._storage = storage
+        self._config = config
+        self._get_search_backend = search_backend_getter
+        self._ensure_fitted = ensure_fitted
+
+    async def create_crossrefs(
+        self,
+        memory: Memory,
+        threshold: float | None = None,
+        max_links: int | None = None,
+    ) -> int:
+        """
+        Find and link similar memories.
+
+        Uses the search backend to find memories similar to the given one
+        and creates cross-references for those above the threshold.
+
+        Args:
+            memory: The memory to find links for
+            threshold: Minimum similarity to create link (default from config)
+            max_links: Maximum links to create (default from config)
+
+        Returns:
+            Number of cross-references created
+        """
+        # Get thresholds from config or use defaults
+        if threshold is None:
+            threshold = getattr(self._config, "crossref_threshold", None)
+            if threshold is None:
+                threshold = 0.3
+        if max_links is None:
+            max_links = getattr(self._config, "crossref_max_links", None)
+            if max_links is None:
+                max_links = 5
+
+        # Ensure search backend is fitted (sync check is fine - just checks a flag)
+        self._ensure_fitted()
+
+        # Search for similar memories (wrap sync I/O in to_thread)
+        search_backend = self._get_search_backend()
+        similar = await asyncio.to_thread(search_backend.search, memory.content, max_links + 1)
+
+        # Create cross-references
+        created = 0
+        for other_id, score in similar:
+            # Skip self-reference
+            if other_id == memory.id:
+                continue
+
+            # Skip below threshold
+            if score < threshold:
+                continue
+
+            # Create the crossref (wrap sync I/O in to_thread)
+            await asyncio.to_thread(self._storage.create_crossref, memory.id, other_id, score)
+            created += 1
+
+            if created >= max_links:
+                break
+
+        if created > 0:
+            logger.debug(f"Created {created} crossrefs for memory {memory.id}")
+
+        return created
+
+    async def get_related(
+        self,
+        memory_id: str,
+        limit: int = 5,
+        min_similarity: float = 0.0,
+    ) -> list[Memory]:
+        """
+        Get memories linked to this one via cross-references.
+
+        Args:
+            memory_id: The memory ID to find related memories for
+            limit: Maximum number of results
+            min_similarity: Minimum similarity threshold
+
+        Returns:
+            List of related Memory objects, sorted by similarity
+        """
+        crossrefs = await asyncio.to_thread(
+            self._storage.get_crossrefs,
+            memory_id,
+            limit=limit,
+            min_similarity=min_similarity,
+        )
+
+        # Get the actual Memory objects
+        memories = []
+        for ref in crossrefs:
+            # Get the "other" memory in the relationship
+            other_id = ref.target_id if ref.source_id == memory_id else ref.source_id
+            memory = await asyncio.to_thread(self._storage.get_memory, other_id)
+            if memory:
+                memories.append(memory)
+
+        return memories

gobby/prompts/loader.py

@@ -23,8 +23,11 @@ from .models import PromptTemplate
 
 logger = logging.getLogger(__name__)
 
-# Default location for bundled prompts
-DEFAULTS_DIR = Path(__file__).parent / "defaults"
+# Default location for bundled prompts (in install/shared/prompts for installation)
+# Falls back to src location for development if install location doesn't exist
+_INSTALL_PROMPTS_DIR = Path(__file__).parent.parent / "install" / "shared" / "prompts"
+_DEV_PROMPTS_DIR = Path(__file__).parent / "defaults"
+DEFAULTS_DIR = _INSTALL_PROMPTS_DIR if _INSTALL_PROMPTS_DIR.exists() else _DEV_PROMPTS_DIR
 
 
 class PromptLoader:

gobby/runner.py

@@ -18,6 +18,7 @@ from gobby.servers.http import HTTPServer
 from gobby.servers.websocket import WebSocketConfig, WebSocketServer
 from gobby.sessions.lifecycle import SessionLifecycleManager
 from gobby.sessions.processor import SessionMessageProcessor
+from gobby.storage.clones import LocalCloneManager
 from gobby.storage.database import DatabaseProtocol, LocalDatabase
 from gobby.storage.mcp import LocalMCPManager
 from gobby.storage.migrations import run_migrations
@@ -28,10 +29,10 @@ from gobby.storage.tasks import LocalTaskManager
 from gobby.storage.worktrees import LocalWorktreeManager
 from gobby.sync.memories import MemorySyncManager
 from gobby.sync.tasks import TaskSyncManager
-from gobby.tasks.expansion import TaskExpander
 from gobby.tasks.validation import TaskValidator
 from gobby.utils.logging import setup_file_logging
 from gobby.utils.machine_id import get_machine_id
+from gobby.worktrees.git import WorktreeGitManager
 
 os.environ["TOKENIZERS_PARALLELISM"] = "false"
 
@@ -43,7 +44,6 @@ class GobbyRunner:
 
     def __init__(self, config_path: Path | None = None, verbose: bool = False):
         setup_file_logging(verbose=verbose)
-        # setup_mcp_logging(verbose=verbose) # Removed as per instruction
 
         config_file = str(config_path) if config_path else None
         self.config = load_config(config_file)
@@ -59,6 +59,16 @@ class GobbyRunner:
         self.task_manager = LocalTaskManager(self.database)
         self.session_task_manager = SessionTaskManager(self.database)
 
+        # Sync bundled skills to database
+        from gobby.skills.sync import sync_bundled_skills
+
+        try:
+            skill_result = sync_bundled_skills(self.database)
+            if skill_result["synced"] > 0:
+                logger.info(f"Synced {skill_result['synced']} bundled skills to database")
+        except Exception as e:
+            logger.warning(f"Failed to sync bundled skills: {e}")
+
         # Initialize LLM Service
         self.llm_service: LLMService | None = None  # Added type hint
         try:
@@ -134,23 +144,11 @@ class GobbyRunner:
                 poll_interval=self.config.message_tracking.poll_interval,
             )
 
-        # Initialize Task Managers (Phase 7.1)
-        self.task_expander: TaskExpander | None = None
+        # Initialize Task Validator (Phase 7.1)
         self.task_validator: TaskValidator | None = None
 
         if self.llm_service:
             gobby_tasks_config = self.config.gobby_tasks
-            if gobby_tasks_config.expansion.enabled:
-                try:
-                    self.task_expander = TaskExpander(
-                        llm_service=self.llm_service,
-                        config=gobby_tasks_config.expansion,
-                        task_manager=self.task_manager,
-                        mcp_manager=self.mcp_proxy,
-                    )
-                except Exception as e:
-                    logger.error(f"Failed to initialize TaskExpander: {e}")
-
             if gobby_tasks_config.validation.enabled:
                 try:
                     self.task_validator = TaskValidator(
@@ -163,6 +161,26 @@ class GobbyRunner:
         # Initialize Worktree Storage (Phase 7 - Subagents)
         self.worktree_storage = LocalWorktreeManager(self.database)
 
+        # Initialize Clone Storage (local git clones for isolated development)
+        self.clone_storage = LocalCloneManager(self.database)
+
+        # Initialize Git Manager for current project (if in a git repo)
+        self.git_manager: WorktreeGitManager | None = None
+        self.project_id: str | None = None
+        try:
+            cwd = Path.cwd()
+            project_json = cwd / ".gobby" / "project.json"
+            if project_json.exists():
+                import json
+
+                project_data = json.loads(project_json.read_text())
+                repo_path = project_data.get("repo_path", str(cwd))
+                self.project_id = project_data.get("id")
+                self.git_manager = WorktreeGitManager(repo_path)
+                logger.info(f"Git manager initialized for project: {self.project_id}")
+        except Exception as e:
+            logger.debug(f"Could not initialize git manager: {e}")
+
         # Initialize Agent Runner (Phase 7 - Subagents)
         # Create executor registry for lazy executor creation
         self.executor_registry = ExecutorRegistry(config=self.config)
@@ -196,6 +214,7 @@ class GobbyRunner:
         # HTTP Server
         self.http_server = HTTPServer(
             port=self.config.daemon_port,
+            test_mode=self.config.test_mode,
             mcp_manager=self.mcp_proxy,
             mcp_db_manager=self.mcp_db_manager,
             config=self.config,
@@ -207,11 +226,13 @@ class GobbyRunner:
             llm_service=self.llm_service,
             message_processor=self.message_processor,
             memory_sync_manager=self.memory_sync_manager,
-            task_expander=self.task_expander,
             task_validator=self.task_validator,
             metrics_manager=self.metrics_manager,
             agent_runner=self.agent_runner,
             worktree_storage=self.worktree_storage,
+            clone_storage=self.clone_storage,
+            git_manager=self.git_manager,
+            project_id=self.project_id,
         )
 
         # Ensure message_processor property is set (redundant but explicit):

gobby/search/__init__.py

@@ -1,23 +1,65 @@
 """
-Shared search backend abstraction.
+Unified search backend abstraction.
 
-Provides pluggable search backends for semantic search:
-- TF-IDF (default) - Built-in local search using scikit-learn (sklearn)
+Provides a unified search layer with multiple backends:
+- TF-IDF (default) - Built-in local search using scikit-learn
+- Embedding - LiteLLM-based semantic search (OpenAI, Ollama, etc.)
+- Unified - Orchestrates between backends with automatic fallback
 
-Usage:
-    from gobby.search import SearchBackend, get_search_backend, TFIDFSearcher
+Basic usage (sync TF-IDF):
+    from gobby.search import TFIDFSearcher
 
-    backend = get_search_backend("tfidf")
+    backend = TFIDFSearcher()
     backend.fit([(id, content) for id, content in items])
     results = backend.search("query text", top_k=10)
+
+Unified search (async with fallback):
+    from gobby.search import UnifiedSearcher, SearchConfig
+
+    config = SearchConfig(mode="auto")  # auto, tfidf, embedding, hybrid
+    searcher = UnifiedSearcher(config)
+    await searcher.fit_async([(id, content) for id, content in items])
+    results = await searcher.search_async("query text", top_k=10)
+
+    if searcher.is_using_fallback():
+        print(f"Using fallback: {searcher.get_fallback_reason()}")
 """
 
+# Sync search backends (backwards compatibility)
+# Async backends
+from gobby.search.backends import AsyncSearchBackend, EmbeddingBackend, TFIDFBackend
+
+# Embedding utilities
+from gobby.search.embeddings import (
+    generate_embedding,
+    generate_embeddings,
+    is_embedding_available,
+)
+
+# Unified search (async with fallback)
+from gobby.search.models import FallbackEvent, SearchConfig, SearchMode
 from gobby.search.protocol import SearchBackend, SearchResult, get_search_backend
 from gobby.search.tfidf import TFIDFSearcher
+from gobby.search.unified import UnifiedSearcher
 
 __all__ = [
+    # Sync backends (backwards compatible)
     "SearchBackend",
     "SearchResult",
     "TFIDFSearcher",
     "get_search_backend",
+    # Models
+    "SearchConfig",
+    "SearchMode",
+    "FallbackEvent",
+    # Unified searcher
+    "UnifiedSearcher",
+    # Async backends
+    "AsyncSearchBackend",
+    "TFIDFBackend",
+    "EmbeddingBackend",
+    # Embedding utilities
+    "generate_embedding",
+    "generate_embeddings",
+    "is_embedding_available",
 ]