gobby 0.2.5__py3-none-any.whl → 0.2.6__py3-none-any.whl

gobby/runner.py

@@ -18,6 +18,7 @@ from gobby.servers.http import HTTPServer
 from gobby.servers.websocket import WebSocketConfig, WebSocketServer
 from gobby.sessions.lifecycle import SessionLifecycleManager
 from gobby.sessions.processor import SessionMessageProcessor
+from gobby.storage.clones import LocalCloneManager
 from gobby.storage.database import DatabaseProtocol, LocalDatabase
 from gobby.storage.mcp import LocalMCPManager
 from gobby.storage.migrations import run_migrations
@@ -28,10 +29,10 @@ from gobby.storage.tasks import LocalTaskManager
 from gobby.storage.worktrees import LocalWorktreeManager
 from gobby.sync.memories import MemorySyncManager
 from gobby.sync.tasks import TaskSyncManager
-from gobby.tasks.expansion import TaskExpander
 from gobby.tasks.validation import TaskValidator
 from gobby.utils.logging import setup_file_logging
 from gobby.utils.machine_id import get_machine_id
+from gobby.worktrees.git import WorktreeGitManager
 
 os.environ["TOKENIZERS_PARALLELISM"] = "false"
 
@@ -43,7 +44,6 @@ class GobbyRunner:
 
     def __init__(self, config_path: Path | None = None, verbose: bool = False):
         setup_file_logging(verbose=verbose)
-        # setup_mcp_logging(verbose=verbose) # Removed as per instruction
 
         config_file = str(config_path) if config_path else None
         self.config = load_config(config_file)
@@ -59,6 +59,16 @@ class GobbyRunner:
         self.task_manager = LocalTaskManager(self.database)
         self.session_task_manager = SessionTaskManager(self.database)
 
+        # Sync bundled skills to database
+        from gobby.skills.sync import sync_bundled_skills
+
+        try:
+            skill_result = sync_bundled_skills(self.database)
+            if skill_result["synced"] > 0:
+                logger.info(f"Synced {skill_result['synced']} bundled skills to database")
+        except Exception as e:
+            logger.warning(f"Failed to sync bundled skills: {e}")
+
         # Initialize LLM Service
         self.llm_service: LLMService | None = None  # Added type hint
         try:
@@ -134,23 +144,11 @@ class GobbyRunner:
                 poll_interval=self.config.message_tracking.poll_interval,
             )
 
-        # Initialize Task Managers (Phase 7.1)
-        self.task_expander: TaskExpander | None = None
+        # Initialize Task Validator (Phase 7.1)
         self.task_validator: TaskValidator | None = None
 
         if self.llm_service:
             gobby_tasks_config = self.config.gobby_tasks
-            if gobby_tasks_config.expansion.enabled:
-                try:
-                    self.task_expander = TaskExpander(
-                        llm_service=self.llm_service,
-                        config=gobby_tasks_config.expansion,
-                        task_manager=self.task_manager,
-                        mcp_manager=self.mcp_proxy,
-                    )
-                except Exception as e:
-                    logger.error(f"Failed to initialize TaskExpander: {e}")
-
             if gobby_tasks_config.validation.enabled:
                 try:
                     self.task_validator = TaskValidator(
@@ -163,6 +161,26 @@ class GobbyRunner:
         # Initialize Worktree Storage (Phase 7 - Subagents)
         self.worktree_storage = LocalWorktreeManager(self.database)
 
+        # Initialize Clone Storage (local git clones for isolated development)
+        self.clone_storage = LocalCloneManager(self.database)
+
+        # Initialize Git Manager for current project (if in a git repo)
+        self.git_manager: WorktreeGitManager | None = None
+        self.project_id: str | None = None
+        try:
+            cwd = Path.cwd()
+            project_json = cwd / ".gobby" / "project.json"
+            if project_json.exists():
+                import json
+
+                project_data = json.loads(project_json.read_text())
+                repo_path = project_data.get("repo_path", str(cwd))
+                self.project_id = project_data.get("id")
+                self.git_manager = WorktreeGitManager(repo_path)
+                logger.info(f"Git manager initialized for project: {self.project_id}")
+        except Exception as e:
+            logger.debug(f"Could not initialize git manager: {e}")
+
         # Initialize Agent Runner (Phase 7 - Subagents)
         # Create executor registry for lazy executor creation
         self.executor_registry = ExecutorRegistry(config=self.config)
@@ -196,6 +214,7 @@ class GobbyRunner:
         # HTTP Server
         self.http_server = HTTPServer(
             port=self.config.daemon_port,
+            test_mode=self.config.test_mode,
             mcp_manager=self.mcp_proxy,
             mcp_db_manager=self.mcp_db_manager,
             config=self.config,
@@ -207,11 +226,13 @@ class GobbyRunner:
             llm_service=self.llm_service,
             message_processor=self.message_processor,
             memory_sync_manager=self.memory_sync_manager,
-            task_expander=self.task_expander,
             task_validator=self.task_validator,
             metrics_manager=self.metrics_manager,
             agent_runner=self.agent_runner,
             worktree_storage=self.worktree_storage,
+            clone_storage=self.clone_storage,
+            git_manager=self.git_manager,
+            project_id=self.project_id,
         )
 
         # Ensure message_processor property is set (redundant but explicit):

gobby/search/__init__.py

@@ -1,23 +1,65 @@
 """
-Shared search backend abstraction.
+Unified search backend abstraction.
 
-Provides pluggable search backends for semantic search:
-- TF-IDF (default) - Built-in local search using scikit-learn (sklearn)
+Provides a unified search layer with multiple backends:
+- TF-IDF (default) - Built-in local search using scikit-learn
+- Embedding - LiteLLM-based semantic search (OpenAI, Ollama, etc.)
+- Unified - Orchestrates between backends with automatic fallback
 
-Usage:
-    from gobby.search import SearchBackend, get_search_backend, TFIDFSearcher
+Basic usage (sync TF-IDF):
+    from gobby.search import TFIDFSearcher
 
-    backend = get_search_backend("tfidf")
+    backend = TFIDFSearcher()
     backend.fit([(id, content) for id, content in items])
     results = backend.search("query text", top_k=10)
+
+Unified search (async with fallback):
+    from gobby.search import UnifiedSearcher, SearchConfig
+
+    config = SearchConfig(mode="auto")  # auto, tfidf, embedding, hybrid
+    searcher = UnifiedSearcher(config)
+    await searcher.fit_async([(id, content) for id, content in items])
+    results = await searcher.search_async("query text", top_k=10)
+
+    if searcher.is_using_fallback():
+        print(f"Using fallback: {searcher.get_fallback_reason()}")
 """
 
+# Sync search backends (backwards compatibility)
+# Async backends
+from gobby.search.backends import AsyncSearchBackend, EmbeddingBackend, TFIDFBackend
+
+# Embedding utilities
+from gobby.search.embeddings import (
+    generate_embedding,
+    generate_embeddings,
+    is_embedding_available,
+)
+
+# Unified search (async with fallback)
+from gobby.search.models import FallbackEvent, SearchConfig, SearchMode
 from gobby.search.protocol import SearchBackend, SearchResult, get_search_backend
 from gobby.search.tfidf import TFIDFSearcher
+from gobby.search.unified import UnifiedSearcher
 
 __all__ = [
+    # Sync backends (backwards compatible)
     "SearchBackend",
     "SearchResult",
     "TFIDFSearcher",
     "get_search_backend",
+    # Models
+    "SearchConfig",
+    "SearchMode",
+    "FallbackEvent",
+    # Unified searcher
+    "UnifiedSearcher",
+    # Async backends
+    "AsyncSearchBackend",
+    "TFIDFBackend",
+    "EmbeddingBackend",
+    # Embedding utilities
+    "generate_embedding",
+    "generate_embeddings",
+    "is_embedding_available",
 ]

gobby/search/backends/__init__.py

@@ -0,0 +1,159 @@
+"""Search backend abstractions.
+
+This module provides the protocol and implementations for search backends
+used by UnifiedSearcher:
+
+- AsyncSearchBackend: Protocol for async search backends
+- TFIDFBackend: TF-IDF based search (always available)
+- EmbeddingBackend: Embedding-based search (requires API)
+
+Usage:
+    from gobby.search.backends import AsyncSearchBackend, TFIDFBackend
+
+    backend: AsyncSearchBackend = TFIDFBackend()
+    await backend.fit_async([("id1", "content1"), ("id2", "content2")])
+    results = await backend.search_async("query", top_k=10)
+"""
+
+from __future__ import annotations
+
+from typing import Any, Protocol, runtime_checkable
+
+# Re-export sync TFIDFSearcher for backwards compatibility
+from gobby.search.tfidf import TFIDFSearcher
+
+__all__ = [
+    "AsyncSearchBackend",
+    "TFIDFBackend",
+    "EmbeddingBackend",
+    "TFIDFSearcher",
+]
+
+
+@runtime_checkable
+class AsyncSearchBackend(Protocol):
+    """Protocol for async search backends.
+
+    All search backends must implement this interface. The protocol
+    uses async methods to support embedding-based backends that need
+    to call external APIs.
+
+    Methods:
+        fit_async: Build/rebuild the search index
+        search_async: Find relevant items for a query
+        needs_refit: Check if index needs rebuilding
+        get_stats: Get backend statistics
+        clear: Clear the search index
+    """
+
+    async def fit_async(self, items: list[tuple[str, str]]) -> None:
+        """Build or rebuild the search index.
+
+        Args:
+            items: List of (item_id, content) tuples to index
+        """
+        ...
+
+    async def search_async(
+        self,
+        query: str,
+        top_k: int = 10,
+    ) -> list[tuple[str, float]]:
+        """Search for items matching the query.
+
+        Args:
+            query: Search query text
+            top_k: Maximum number of results to return
+
+        Returns:
+            List of (item_id, similarity_score) tuples, sorted by
+            relevance (highest similarity first).
+        """
+        ...
+
+    def needs_refit(self) -> bool:
+        """Check if the search index needs rebuilding.
+
+        Returns:
+            True if fit_async() should be called before search_async()
+        """
+        ...
+
+    def get_stats(self) -> dict[str, Any]:
+        """Get statistics about the search index.
+
+        Returns:
+            Dict with backend-specific statistics
+        """
+        ...
+
+    def clear(self) -> None:
+        """Clear the search index."""
+        ...
+
+
+class TFIDFBackend:
+    """Async wrapper around TFIDFSearcher.
+
+    Provides the AsyncSearchBackend interface for TF-IDF search.
+    This is a thin wrapper that delegates to the sync TFIDFSearcher.
+    """
+
+    def __init__(
+        self,
+        ngram_range: tuple[int, int] = (1, 2),
+        max_features: int = 10000,
+        min_df: int = 1,
+        stop_words: str | None = "english",
+        refit_threshold: int = 10,
+    ):
+        """Initialize TF-IDF backend.
+
+        Args:
+            ngram_range: Min/max n-gram sizes for tokenization
+            max_features: Maximum vocabulary size
+            min_df: Minimum document frequency for inclusion
+            stop_words: Language for stop words (None to disable)
+            refit_threshold: Number of updates before automatic refit
+        """
+        self._searcher = TFIDFSearcher(
+            ngram_range=ngram_range,
+            max_features=max_features,
+            min_df=min_df,
+            stop_words=stop_words,
+            refit_threshold=refit_threshold,
+        )
+
+    async def fit_async(self, items: list[tuple[str, str]]) -> None:
+        """Build or rebuild the search index."""
+        self._searcher.fit(items)
+
+    async def search_async(
+        self,
+        query: str,
+        top_k: int = 10,
+    ) -> list[tuple[str, float]]:
+        """Search for items matching the query."""
+        return self._searcher.search(query, top_k)
+
+    def needs_refit(self) -> bool:
+        """Check if the search index needs rebuilding."""
+        return self._searcher.needs_refit()
+
+    def get_stats(self) -> dict[str, Any]:
+        """Get statistics about the search index."""
+        stats = self._searcher.get_stats()
+        stats["backend_type"] = "tfidf"
+        return stats
+
+    def clear(self) -> None:
+        """Clear the search index."""
+        self._searcher.clear()
+
+    def mark_update(self) -> None:
+        """Mark that an item update occurred."""
+        self._searcher.mark_update()
+
+
+# Import EmbeddingBackend - needs to be at end to avoid circular imports
+from gobby.search.backends.embedding import EmbeddingBackend  # noqa: E402

gobby/search/backends/embedding.py

@@ -0,0 +1,225 @@
+"""Embedding-based search backend.
+
+This module provides embedding-based semantic search using cosine similarity.
+It stores embeddings in memory and uses LiteLLM for embedding generation.
+"""
+
+from __future__ import annotations
+
+import logging
+import math
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from gobby.search.models import SearchConfig
+
+logger = logging.getLogger(__name__)
+
+
+def _cosine_similarity(vec1: list[float], vec2: list[float]) -> float:
+    """Compute cosine similarity between two vectors.
+
+    Args:
+        vec1: First vector
+        vec2: Second vector
+
+    Returns:
+        Cosine similarity score between -1 and 1
+    """
+    if len(vec1) != len(vec2):
+        return 0.0
+
+    dot_product = sum(a * b for a, b in zip(vec1, vec2, strict=True))
+    norm1 = math.sqrt(sum(a * a for a in vec1))
+    norm2 = math.sqrt(sum(b * b for b in vec2))
+
+    if norm1 == 0 or norm2 == 0:
+        return 0.0
+
+    return dot_product / (norm1 * norm2)
+
+
+class EmbeddingBackend:
+    """Embedding-based search backend using LiteLLM.
+
+    This backend generates embeddings for indexed items and uses
+    cosine similarity for search. Embeddings are stored in memory.
+
+    Supports all providers supported by LiteLLM:
+    - OpenAI (text-embedding-3-small)
+    - Ollama (openai/nomic-embed-text with api_base)
+    - Azure, Gemini, Mistral, etc.
+
+    Example:
+        backend = EmbeddingBackend(
+            model="text-embedding-3-small",
+            api_key="sk-..."
+        )
+        await backend.fit_async([("id1", "hello"), ("id2", "world")])
+        results = await backend.search_async("greeting", top_k=5)
+    """
+
+    def __init__(
+        self,
+        model: str = "text-embedding-3-small",
+        api_base: str | None = None,
+        api_key: str | None = None,
+    ):
+        """Initialize embedding backend.
+
+        Args:
+            model: LiteLLM model string
+            api_base: Optional API base URL for custom endpoints
+            api_key: Optional API key (uses env var if not set)
+        """
+        self._model = model
+        self._api_base = api_base
+        self._api_key = api_key
+
+        # Item storage
+        self._item_ids: list[str] = []
+        self._item_embeddings: list[list[float]] = []
+        self._item_contents: dict[str, str] = {}  # For reindexing
+        self._fitted = False
+
+    @classmethod
+    def from_config(cls, config: SearchConfig) -> EmbeddingBackend:
+        """Create an EmbeddingBackend from a SearchConfig.
+
+        Args:
+            config: SearchConfig with model and API settings
+
+        Returns:
+            Configured EmbeddingBackend instance
+        """
+        return cls(
+            model=config.embedding_model,
+            api_base=config.embedding_api_base,
+            api_key=config.embedding_api_key,
+        )
+
+    async def fit_async(self, items: list[tuple[str, str]]) -> None:
+        """Build or rebuild the search index.
+
+        Generates embeddings for all items and stores them in memory.
+
+        Args:
+            items: List of (item_id, content) tuples to index
+
+        Raises:
+            RuntimeError: If embedding generation fails
+        """
+        if not items:
+            self._item_ids = []
+            self._item_embeddings = []
+            self._item_contents = {}
+            self._fitted = False
+            logger.debug("Embedding index cleared (no items)")
+            return
+
+        from gobby.search.embeddings import generate_embeddings
+
+        # Store contents for potential reindexing
+        self._item_ids = [item_id for item_id, _ in items]
+        self._item_contents = dict(items)
+        contents = [content for _, content in items]
+
+        # Generate embeddings in batch
+        try:
+            self._item_embeddings = await generate_embeddings(
+                texts=contents,
+                model=self._model,
+                api_base=self._api_base,
+                api_key=self._api_key,
+            )
+            self._fitted = True
+            logger.info(f"Embedding index built with {len(items)} items")
+        except Exception as e:
+            # Clear stale state to prevent inconsistent data
+            self._item_ids = []
+            self._item_contents = {}
+            self._item_embeddings = []
+            self._fitted = False
+            logger.error(f"Failed to build embedding index: {e}")
+            raise
+
+    async def search_async(
+        self,
+        query: str,
+        top_k: int = 10,
+    ) -> list[tuple[str, float]]:
+        """Search for items matching the query.
+
+        Generates an embedding for the query and finds items with
+        highest cosine similarity.
+
+        Args:
+            query: Search query text
+            top_k: Maximum number of results to return
+
+        Returns:
+            List of (item_id, similarity_score) tuples, sorted by
+            similarity descending.
+
+        Raises:
+            RuntimeError: If embedding generation fails
+        """
+        if not self._fitted or not self._item_embeddings:
+            return []
+
+        from gobby.search.embeddings import generate_embedding
+
+        # Generate query embedding
+        try:
+            query_embedding = await generate_embedding(
+                text=query,
+                model=self._model,
+                api_base=self._api_base,
+                api_key=self._api_key,
+            )
+        except Exception as e:
+            logger.error(f"Failed to embed query: {e}")
+            raise
+
+        # Compute similarities
+        similarities: list[tuple[str, float]] = []
+        for item_id, item_embedding in zip(self._item_ids, self._item_embeddings, strict=True):
+            similarity = _cosine_similarity(query_embedding, item_embedding)
+            if similarity > 0:
+                similarities.append((item_id, similarity))
+
+        # Sort by similarity descending
+        similarities.sort(key=lambda x: x[1], reverse=True)
+
+        return similarities[:top_k]
+
+    def needs_refit(self) -> bool:
+        """Check if the search index needs rebuilding."""
+        return not self._fitted
+
+    def get_stats(self) -> dict[str, Any]:
+        """Get statistics about the search index."""
+        return {
+            "backend_type": "embedding",
+            "fitted": self._fitted,
+            "item_count": len(self._item_ids),
+            "model": self._model,
+            "has_api_base": self._api_base is not None,
+        }
+
+    def clear(self) -> None:
+        """Clear the search index."""
+        self._item_ids = []
+        self._item_embeddings = []
+        self._item_contents = {}
+        self._fitted = False
+
+    def get_item_contents(self) -> dict[str, str]:
+        """Get stored item contents.
+
+        Useful for reindexing into a different backend (e.g., TF-IDF fallback).
+
+        Returns:
+            Dict mapping item_id to content
+        """
+        return self._item_contents.copy()