shotgun-sh 0.2.17__py3-none-any.whl → 0.4.0.dev1__py3-none-any.whl

shotgun/cli/spec/models.py

@@ -0,0 +1,48 @@
+"""Pydantic models for spec CLI commands."""
+
+from datetime import datetime
+from enum import StrEnum
+
+from pydantic import BaseModel, Field
+
+
+class PullSource(StrEnum):
+    """Source of spec pull operation for analytics."""
+
+    CLI = "cli"
+    TUI = "tui"
+
+
+class PullPhase(StrEnum):
+    """Phases during spec pull operation for analytics."""
+
+    STARTING = "starting"
+    FETCHING = "fetching"
+    BACKUP = "backup"
+    DOWNLOADING = "downloading"
+    FINALIZING = "finalizing"
+
+
+class SpecMeta(BaseModel):
+    """Metadata stored in .shotgun/meta.json after pulling a spec.
+
+    This file tracks the source of the local spec files and is used
+    by the TUI to display version information and enable future sync operations.
+    """
+
+    version_id: str = Field(description="Pulled version UUID")
+    spec_id: str = Field(description="Spec UUID")
+    spec_name: str = Field(description="Spec name at time of pull")
+    workspace_id: str = Field(description="Workspace UUID")
+    is_latest: bool = Field(
+        description="Whether this was the latest version when pulled"
+    )
+    pulled_at: datetime = Field(description="Timestamp when spec was pulled (UTC)")
+    backup_path: str | None = Field(
+        default=None,
+        description="Path where previous .shotgun/ files were backed up",
+    )
+    web_url: str | None = Field(
+        default=None,
+        description="URL to view this version in the web UI",
+    )

shotgun/cli/spec/pull_service.py

@@ -0,0 +1,219 @@
+"""Shared spec pull service for CLI and TUI."""
+
+import time
+from collections.abc import Callable
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from pathlib import Path
+
+from shotgun.logging_config import get_logger
+from shotgun.posthog_telemetry import track_event
+from shotgun.shotgun_web.specs_client import SpecsClient
+from shotgun.shotgun_web.supabase_client import download_file_from_url
+
+from .backup import clear_shotgun_dir, create_backup
+from .models import PullPhase, PullSource, SpecMeta
+
+logger = get_logger(__name__)
+
+
+@dataclass
+class PullProgress:
+    """Progress update during spec pull."""
+
+    phase: str
+    file_index: int | None = None
+    total_files: int | None = None
+    current_file: str | None = None
+
+
+@dataclass
+class PullResult:
+    """Result of a spec pull operation."""
+
+    success: bool
+    spec_name: str | None = None
+    file_count: int = 0
+    backup_path: str | None = None
+    web_url: str | None = None
+    error: str | None = None
+
+
+class CancelledError(Exception):
+    """Raised when pull is cancelled."""
+
+
+class SpecPullService:
+    """Service for pulling spec versions from cloud."""
+
+    def __init__(self) -> None:
+        self._client = SpecsClient()
+
+    async def pull_version(
+        self,
+        version_id: str,
+        shotgun_dir: Path,
+        on_progress: Callable[[PullProgress], None] | None = None,
+        is_cancelled: Callable[[], bool] | None = None,
+        source: PullSource = PullSource.CLI,
+    ) -> PullResult:
+        """Pull a spec version to the local directory.
+
+        Args:
+            version_id: The version UUID to pull
+            shotgun_dir: Target directory (typically .shotgun/)
+            on_progress: Optional callback for progress updates
+            is_cancelled: Optional callback to check if cancelled
+            source: Source of the pull request (CLI or TUI)
+
+        Returns:
+            PullResult with success status and details
+        """
+        start_time = time.time()
+        current_phase: PullPhase = PullPhase.STARTING
+        track_event("spec_pull_started", {"source": source.value})
+
+        def report(
+            phase: str,
+            file_index: int | None = None,
+            total_files: int | None = None,
+            current_file: str | None = None,
+        ) -> None:
+            if on_progress:
+                on_progress(
+                    PullProgress(
+                        phase=phase,
+                        file_index=file_index,
+                        total_files=total_files,
+                        current_file=current_file,
+                    )
+                )
+
+        def check_cancelled() -> None:
+            nonlocal current_phase
+            if is_cancelled and is_cancelled():
+                track_event(
+                    "spec_pull_cancelled",
+                    {"source": source.value, "phase": current_phase.value},
+                )
+                raise CancelledError()
+
+        try:
+            # Phase 1: Fetch version metadata
+            current_phase = PullPhase.FETCHING
+            report("Fetching version info...")
+            check_cancelled()
+
+            response = await self._client.get_version_with_files(version_id)
+            spec_name = response.spec_name
+            files = response.files
+
+            if not files:
+                track_event(
+                    "spec_pull_failed",
+                    {
+                        "source": source.value,
+                        "error_type": "EmptyVersion",
+                        "phase": current_phase.value,
+                    },
+                )
+                return PullResult(
+                    success=False,
+                    spec_name=spec_name,
+                    error="No files in this version.",
+                )
+
+            # Phase 2: Backup existing content
+            current_phase = PullPhase.BACKUP
+            backup_path: str | None = None
+            if shotgun_dir.exists():
+                report("Backing up existing files...")
+                check_cancelled()
+
+                backup_path = await create_backup(shotgun_dir)
+                if backup_path:
+                    clear_shotgun_dir(shotgun_dir)
+
+            # Ensure directory exists
+            shotgun_dir.mkdir(parents=True, exist_ok=True)
+
+            # Phase 3: Download files
+            current_phase = PullPhase.DOWNLOADING
+            total_files = len(files)
+            total_bytes = 0
+            for idx, file_info in enumerate(files):
+                check_cancelled()
+
+                report(
+                    f"Downloading files ({idx + 1}/{total_files})...",
+                    file_index=idx,
+                    total_files=total_files,
+                    current_file=file_info.relative_path,
+                )
+
+                if not file_info.download_url:
+                    logger.warning(
+                        "Skipping file without download URL: %s",
+                        file_info.relative_path,
+                    )
+                    continue
+
+                content = await download_file_from_url(file_info.download_url)
+                total_bytes += file_info.size_bytes
+
+                local_path = shotgun_dir / file_info.relative_path
+                local_path.parent.mkdir(parents=True, exist_ok=True)
+                local_path.write_bytes(content)
+
+            # Phase 4: Write meta.json
+            current_phase = PullPhase.FINALIZING
+            report("Finalizing...")
+            check_cancelled()
+
+            meta = SpecMeta(
+                version_id=response.version.id,
+                spec_id=response.spec_id,
+                spec_name=response.spec_name,
+                workspace_id=response.workspace_id,
+                is_latest=response.version.is_latest,
+                pulled_at=datetime.now(timezone.utc),
+                backup_path=backup_path,
+                web_url=response.web_url,
+            )
+            meta_path = shotgun_dir / "meta.json"
+            meta_path.write_text(meta.model_dump_json(indent=2))
+
+            # Track successful completion
+            duration = time.time() - start_time
+            track_event(
+                "spec_pull_completed",
+                {
+                    "source": source.value,
+                    "file_count": total_files,
+                    "total_bytes": total_bytes,
+                    "duration_seconds": round(duration, 2),
+                    "had_backup": backup_path is not None,
+                },
+            )
+
+            return PullResult(
+                success=True,
+                spec_name=spec_name,
+                file_count=total_files,
+                backup_path=backup_path,
+                web_url=response.web_url,
+            )
+
+        except CancelledError:
+            # Already tracked in check_cancelled()
+            raise
+        except Exception as e:
+            track_event(
+                "spec_pull_failed",
+                {
+                    "source": source.value,
+                    "error_type": type(e).__name__,
+                    "phase": current_phase.value,
+                },
+            )
+            raise

shotgun/cli/specify.py

@@ -11,6 +11,8 @@ from shotgun.agents.specify import (
     create_specify_agent,
     run_specify_agent,
 )
+from shotgun.cli.error_handler import print_agent_error
+from shotgun.exceptions import ErrorNotPickedUpBySentry
 from shotgun.logging_config import get_logger
 
 app = typer.Typer(
@@ -44,26 +46,25 @@ def specify(
 
     logger.info("📝 Specification Requirement: %s", requirement)
 
-    try:
-        # Create agent dependencies
-        agent_runtime_options = AgentRuntimeOptions(
-            interactive_mode=not non_interactive
-        )
+    # Create agent dependencies
+    agent_runtime_options = AgentRuntimeOptions(interactive_mode=not non_interactive)
 
-        # Create the specify agent with deps and provider
-        agent, deps = asyncio.run(create_specify_agent(agent_runtime_options, provider))
+    # Create the specify agent with deps and provider
+    agent, deps = asyncio.run(create_specify_agent(agent_runtime_options, provider))
 
-        # Start specification process
-        logger.info("📋 Starting specification generation...")
-        result = asyncio.run(run_specify_agent(agent, requirement, deps))
+    # Start specification process with error handling
+    logger.info("📋 Starting specification generation...")
 
-        # Display results
-        logger.info("✅ Specification Complete!")
-        logger.info("📋 Results:")
-        logger.info("%s", result.output)
+    async def async_specify() -> None:
+        try:
+            result = await run_specify_agent(agent, requirement, deps)
+            logger.info("✅ Specification Complete!")
+            logger.info("📋 Results:")
+            logger.info("%s", result.output)
+        except ErrorNotPickedUpBySentry as e:
+            print_agent_error(e)
+        except Exception as e:
+            logger.exception("Unexpected error in specify command")
+            print(f"⚠️  An unexpected error occurred: {str(e)}")
 
-    except Exception as e:
-        logger.error("❌ Error during specification: %s", str(e))
-        import traceback
-
-        logger.debug("Full traceback:\n%s", traceback.format_exc())
+    asyncio.run(async_specify())

shotgun/cli/tasks.py

@@ -11,7 +11,10 @@ from shotgun.agents.tasks import (
     create_tasks_agent,
     run_tasks_agent,
 )
+from shotgun.cli.error_handler import print_agent_error
+from shotgun.exceptions import ErrorNotPickedUpBySentry
 from shotgun.logging_config import get_logger
+from shotgun.posthog_telemetry import track_event
 
 app = typer.Typer(name="tasks", help="Generate task lists with agentic approach")
 logger = get_logger(__name__)
@@ -42,37 +45,34 @@ def tasks(
 
     logger.info("📋 Task Creation Instruction: %s", instruction)
 
-    try:
-        # Track tasks command usage
-        from shotgun.posthog_telemetry import track_event
-
-        track_event(
-            "tasks_command",
-            {
-                "non_interactive": non_interactive,
-                "provider": provider.value if provider else "default",
-            },
-        )
-
-        # Create agent dependencies
-        agent_runtime_options = AgentRuntimeOptions(
-            interactive_mode=not non_interactive
-        )
-
-        # Create the tasks agent with deps and provider
-        agent, deps = asyncio.run(create_tasks_agent(agent_runtime_options, provider))
-
-        # Start task creation process
-        logger.info("🎯 Starting task creation...")
-        result = asyncio.run(run_tasks_agent(agent, instruction, deps))
-
-        # Display results
-        logger.info("✅ Task Creation Complete!")
-        logger.info("📋 Results:")
-        logger.info("%s", result.output)
-
-    except Exception as e:
-        logger.error("❌ Error during task creation: %s", str(e))
-        import traceback
-
-        logger.debug("Full traceback:\n%s", traceback.format_exc())
+    # Track tasks command usage
+    track_event(
+        "tasks_command",
+        {
+            "non_interactive": non_interactive,
+            "provider": provider.value if provider else "default",
+        },
+    )
+
+    # Create agent dependencies
+    agent_runtime_options = AgentRuntimeOptions(interactive_mode=not non_interactive)
+
+    # Create the tasks agent with deps and provider
+    agent, deps = asyncio.run(create_tasks_agent(agent_runtime_options, provider))
+
+    # Start task creation process with error handling
+    logger.info("🎯 Starting task creation...")
+
+    async def async_tasks() -> None:
+        try:
+            result = await run_tasks_agent(agent, instruction, deps)
+            logger.info("✅ Task Creation Complete!")
+            logger.info("📋 Results:")
+            logger.info("%s", result.output)
+        except ErrorNotPickedUpBySentry as e:
+            print_agent_error(e)
+        except Exception as e:
+            logger.exception("Unexpected error in tasks command")
+            print(f"⚠️  An unexpected error occurred: {str(e)}")
+
+    asyncio.run(async_tasks())

shotgun/codebase/core/change_detector.py

@@ -7,7 +7,7 @@ from pathlib import Path
 from typing import Any, cast
 
 import aiofiles
-import kuzu
+import real_ladybug as kuzu
 
 from shotgun.logging_config import get_logger
 

shotgun/codebase/core/ingestor.py

@@ -6,11 +6,12 @@ import os
 import time
 import uuid
 from collections import defaultdict
+from collections.abc import Callable
 from pathlib import Path
 from typing import Any
 
 import aiofiles
-import kuzu
+import real_ladybug as kuzu
 from tree_sitter import Node, Parser, QueryCursor
 
 from shotgun.codebase.core.language_config import LANGUAGE_CONFIGS, get_language_config
@@ -198,11 +199,21 @@ class Ingestor:
                     return True
         return False
 
-    def flush_nodes(self) -> None:
-        """Flush pending node insertions to the database."""
+    def flush_nodes(
+        self,
+        progress_callback: Callable[[int, int], None] | None = None,
+    ) -> None:
+        """Flush pending node insertions to the database.
+
+        Args:
+            progress_callback: Optional callback(current, total) for progress reporting
+        """
         if not self.node_buffer:
             return
 
+        total_nodes = len(self.node_buffer)
+        processed = 0
+
         # Group nodes by label
         nodes_by_label: dict[str, list[dict[str, Any]]] = defaultdict(list)
         for label, properties in self.node_buffer:
@@ -239,9 +250,18 @@ class Ingestor:
                     params = dict(zip(prop_names, prop_values, strict=False))
                     self.conn.execute(query, params)
 
+                    # Report progress
+                    processed += 1
+                    if progress_callback and processed % 10 == 0:
+                        progress_callback(processed, total_nodes)
+
             except Exception as e:
                 logger.error(f"Failed to insert {label} nodes: {e}")
 
+        # Final progress report
+        if progress_callback:
+            progress_callback(total_nodes, total_nodes)
+
         # Log node counts by type
         node_type_counts: dict[str, int] = {}
         for label, _ in self.node_buffer:
@@ -280,11 +300,21 @@ class Ingestor:
 
         # Don't auto-flush relationships - wait for explicit flush_all() to ensure nodes exist first
 
-    def flush_relationships(self) -> None:
-        """Flush pending relationship insertions to the database."""
+    def flush_relationships(
+        self,
+        progress_callback: Callable[[int, int], None] | None = None,
+    ) -> None:
+        """Flush pending relationship insertions to the database.
+
+        Args:
+            progress_callback: Optional callback(current, total) for progress reporting
+        """
         if not self.relationship_buffer:
             return
 
+        total_rels = len(self.relationship_buffer)
+        processed = 0
+
         # Group relationships by type
         rels_by_type: dict[
             str, list[tuple[str, str, Any, str, str, str, Any, dict[str, Any] | None]]
@@ -299,7 +329,7 @@ class Ingestor:
                 to_label,
                 to_key,
                 to_value,
-                properties,
+                _properties,
             ) = rel_data
 
             # Determine actual table name
@@ -323,7 +353,7 @@ class Ingestor:
                         to_label,
                         to_key,
                         to_value,
-                        properties,
+                        _properties,
                     ) = rel_data
 
                     # Build MATCH and MERGE query (use MERGE to avoid duplicate relationships)
@@ -337,6 +367,11 @@ class Ingestor:
                     try:
                         self.conn.execute(query, params)
                         success_count += 1
+
+                        # Report progress
+                        processed += 1
+                        if progress_callback and processed % 10 == 0:
+                            progress_callback(processed, total_rels)
                     except Exception as e:
                         logger.error(
                             f"Failed to create single relationship {table_name}: {from_label}({from_value}) -> {to_label}({to_value})"
@@ -360,6 +395,10 @@ class Ingestor:
                 # Don't swallow the exception - let it propagate
                 raise
 
+        # Final progress report
+        if progress_callback:
+            progress_callback(total_rels, total_rels)
+
         # Log summary of flushed relationships
         logger.info(
             f"Flushed {len(self.relationship_buffer)} relationships: {relationship_counts}"
@@ -586,6 +625,9 @@ class SimpleGraphBuilder:
             self.ignore_dirs = self.ignore_dirs.union(set(exclude_patterns))
         self.progress_callback = progress_callback
 
+        # Generate unique session ID for correlating timing events in PostHog
+        self._index_session_id = str(uuid.uuid4())[:8]
+
         # Caches
         self.structural_elements: dict[Path, str | None] = {}
         self.ast_cache: dict[Path, tuple[Node, str]] = {}
@@ -621,25 +663,129 @@ class SimpleGraphBuilder:
             # Don't let progress callback errors crash the build
             logger.debug(f"Progress callback error: {e}")
 
+    def _log_timing(
+        self,
+        phase: str,
+        duration: float,
+        items: int,
+        extra_props: dict[str, Any] | None = None,
+    ) -> None:
+        """Log timing data to PostHog for analysis."""
+        from shotgun.posthog_telemetry import track_event
+
+        properties: dict[str, Any] = {
+            "session_id": self._index_session_id,
+            "phase": phase,
+            "duration_seconds": round(duration, 3),
+            "item_count": items,
+        }
+        if extra_props:
+            properties.update(extra_props)
+
+        track_event("codebase_index_phase_completed", properties)
+
+    def _log_summary(
+        self,
+        total_duration: float,
+        total_files: int,
+        total_nodes: int,
+        total_relationships: int,
+    ) -> None:
+        """Log indexing summary event to PostHog."""
+        from shotgun.posthog_telemetry import track_event
+
+        track_event(
+            "codebase_index_completed",
+            {
+                "session_id": self._index_session_id,
+                "total_duration_seconds": round(total_duration, 3),
+                "total_files": total_files,
+                "total_nodes": total_nodes,
+                "total_relationships": total_relationships,
+            },
+        )
+
     async def run(self) -> None:
         """Run the three-pass graph building process."""
         logger.info(f"Building graph for project: {self.project_name}")
 
         # Pass 1: Structure
         logger.info("Pass 1: Identifying packages and folders...")
+        t0 = time.time()
         self._identify_structure()
+        t1 = time.time()
+        self._log_timing("structure", t1 - t0, len(self.structural_elements))
 
         # Pass 2: Definitions
         logger.info("Pass 2: Processing files and extracting definitions...")
+        t2 = time.time()
         await self._process_files()
+        t3 = time.time()
+        self._log_timing(
+            "definitions",
+            t3 - t2,
+            len(self.ast_cache),
+            {"file_count": len(self.ast_cache)},
+        )
 
         # Pass 3: Relationships
         logger.info("Pass 3: Processing relationships (calls, imports)...")
+        t4 = time.time()
         self._process_relationships()
+        t5 = time.time()
+        self._log_timing("relationships", t5 - t4, len(self.ast_cache))
 
         # Flush all pending operations
         logger.info("Flushing all data to database...")
-        self.ingestor.flush_all()
+        t6 = time.time()
+        node_count = len(self.ingestor.node_buffer)
+
+        # Create progress callback for flush_nodes
+        def node_progress(current: int, total: int) -> None:
+            self._report_progress(
+                "flush_nodes", "Flushing nodes to database", current, total
+            )
+
+        self.ingestor.flush_nodes(progress_callback=node_progress)
+        self._report_progress(
+            "flush_nodes", "Flushing nodes to database", node_count, node_count, True
+        )
+        t7 = time.time()
+        self._log_timing("flush_nodes", t7 - t6, node_count, {"node_count": node_count})
+
+        rel_count = len(self.ingestor.relationship_buffer)
+
+        # Create progress callback for flush_relationships
+        def rel_progress(current: int, total: int) -> None:
+            self._report_progress(
+                "flush_relationships",
+                "Flushing relationships to database",
+                current,
+                total,
+            )
+
+        self.ingestor.flush_relationships(progress_callback=rel_progress)
+        self._report_progress(
+            "flush_relationships",
+            "Flushing relationships to database",
+            rel_count,
+            rel_count,
+            True,
+        )
+        t8 = time.time()
+        self._log_timing(
+            "flush_relationships", t8 - t7, rel_count, {"relationship_count": rel_count}
+        )
+
+        # Track summary event with totals (no PII - only numeric metadata)
+        total_duration = t8 - t0
+        self._log_summary(
+            total_duration=total_duration,
+            total_files=len(self.ast_cache),
+            total_nodes=node_count,
+            total_relationships=rel_count,
+        )
+
         logger.info("Graph building complete!")
 
     def _identify_structure(self) -> None:

shotgun/codebase/core/manager.py

@@ -12,7 +12,7 @@ from pathlib import Path
 from typing import Any, ClassVar
 
 import anyio
-import kuzu
+import real_ladybug as kuzu
 from watchdog.events import FileSystemEvent, FileSystemEventHandler
 from watchdog.observers import Observer
 

shotgun/codebase/models.py

@@ -29,6 +29,8 @@ class ProgressPhase(StrEnum):
     STRUCTURE = "structure"  # Identifying packages and folders
     DEFINITIONS = "definitions"  # Processing files and extracting definitions
     RELATIONSHIPS = "relationships"  # Processing relationships (calls, imports)
+    FLUSH_NODES = "flush_nodes"  # Flushing nodes to database
+    FLUSH_RELATIONSHIPS = "flush_relationships"  # Flushing relationships to database
 
 
 class IndexProgress(BaseModel):