shotgun-sh 0.4.0.dev1__py3-none-any.whl → 0.6.2__py3-none-any.whl

shotgun/codebase/benchmarks/formatters/markdown.py

@@ -0,0 +1,136 @@
+"""Markdown formatter for benchmark results.
+
+This module provides the MarkdownFormatter class for displaying benchmark results
+as GitHub-compatible markdown.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from shotgun.codebase.benchmarks.models import (
+        BenchmarkResults,
+        MetricsDisplayOptions,
+    )
+
+
+class MarkdownFormatter:
+    """Format benchmark results as GitHub-compatible markdown."""
+
+    def format_results(
+        self,
+        results: BenchmarkResults,
+        options: MetricsDisplayOptions,
+    ) -> str:
+        """Format benchmark results as markdown.
+
+        Args:
+            results: Benchmark results to format
+            options: Display options
+
+        Returns:
+            Markdown string
+        """
+        lines = []
+
+        # Header
+        lines.append(f"# Indexing Benchmark: {results.codebase_name}")
+        lines.append("")
+        lines.append(f"**Path:** `{results.codebase_path}`")
+
+        mode = results.config.mode.capitalize()
+        worker_info = ""
+        if results.config.mode == "parallel":
+            worker_count = results.config.worker_count or "auto"
+            worker_info = f" ({worker_count} workers)"
+        lines.append(f"**Mode:** {mode}{worker_info}")
+        lines.append(
+            f"**Iterations:** {results.config.iterations} ({results.config.warmup_iterations} warmup)"
+        )
+        lines.append("")
+
+        # Summary statistics
+        lines.append("## Summary")
+        lines.append("")
+        lines.append("| Metric | Value |")
+        lines.append("|--------|-------|")
+
+        if results.config.iterations > 1:
+            lines.append(f"| Duration (avg) | {results.avg_duration_seconds:.2f}s |")
+            lines.append(f"| Duration (min) | {results.min_duration_seconds:.2f}s |")
+            lines.append(f"| Duration (max) | {results.max_duration_seconds:.2f}s |")
+            lines.append(f"| Duration (std dev) | {results.std_dev_seconds:.2f}s |")
+        else:
+            lines.append(f"| Duration | {results.avg_duration_seconds:.2f}s |")
+
+        lines.append(f"| Throughput | {results.avg_throughput:.1f} files/s |")
+        lines.append(f"| Peak Memory | {results.avg_memory_mb:.1f} MB |")
+
+        metrics = results.get_last_metrics()
+        if metrics:
+            lines.append(f"| Files Processed | {metrics.total_files:,} |")
+            lines.append(f"| Nodes Created | {metrics.total_nodes:,} |")
+            lines.append(f"| Relationships | {metrics.total_relationships:,} |")
+
+        if results.efficiency:
+            lines.append(
+                f"| Parallelism Efficiency | {results.efficiency * 100:.0f}% |"
+            )
+        if results.speedup_factor:
+            lines.append(f"| Speedup | {results.speedup_factor:.2f}x |")
+
+        lines.append("")
+
+        # Phase breakdown
+        if metrics and options.show_phase_metrics and not options.show_summary_only:
+            lines.append("## Phase Breakdown")
+            lines.append("")
+            lines.append("| Phase | Duration | Items | Throughput | Memory |")
+            lines.append("|-------|----------|-------|------------|--------|")
+
+            for name, phase in metrics.phase_metrics.items():
+                lines.append(
+                    f"| {name} | {phase.duration_seconds:.2f}s | "
+                    f"{phase.items_processed} | {phase.throughput:.1f}/s | "
+                    f"{phase.memory_mb:.1f} MB |"
+                )
+
+            lines.append("")
+
+        # File metrics
+        if (
+            metrics
+            and options.show_file_metrics
+            and metrics.file_metrics
+            and not options.show_summary_only
+        ):
+            file_metrics = sorted(
+                metrics.file_metrics,
+                key=lambda f: f.parse_time_ms,
+                reverse=True,
+            )
+            if options.top_n_files:
+                file_metrics = file_metrics[: options.top_n_files]
+
+            if file_metrics:
+                title = "File Metrics"
+                if options.top_n_files:
+                    title = f"Top {len(file_metrics)} Slowest Files"
+
+                lines.append(f"## {title}")
+                lines.append("")
+                lines.append("| File | Language | Size | Duration | Definitions |")
+                lines.append("|------|----------|------|----------|-------------|")
+
+                for f in file_metrics:
+                    size_kb = f.file_size_bytes / 1024
+                    lines.append(
+                        f"| `{f.file_path}` | {f.language} | "
+                        f"{size_kb:.1f} KB | {f.parse_time_ms:.1f}ms | "
+                        f"{f.definitions_extracted} |"
+                    )
+
+                lines.append("")
+
+        return "\n".join(lines)

shotgun/codebase/benchmarks/models.py

@@ -0,0 +1,129 @@
+"""Pydantic models for benchmark system.
+
+This module contains all data models used by the benchmark system.
+"""
+
+from __future__ import annotations
+
+import statistics
+from enum import StrEnum
+
+from pydantic import BaseModel, Field
+
+from shotgun.codebase.core.metrics_types import IndexingMetrics
+
+
+class BenchmarkMode(StrEnum):
+    """Execution mode for benchmarks."""
+
+    PARALLEL = "parallel"
+    SEQUENTIAL = "sequential"
+
+
+class OutputFormat(StrEnum):
+    """Output format for benchmark results."""
+
+    JSON = "json"
+    MARKDOWN = "markdown"
+
+
+class BenchmarkConfig(BaseModel):
+    """Configuration for benchmark execution."""
+
+    mode: BenchmarkMode = BenchmarkMode.PARALLEL
+    worker_count: int | None = None
+    iterations: int = 1
+    warmup_iterations: int = 0
+    collect_file_metrics: bool = True
+    collect_worker_metrics: bool = True
+
+
+class BenchmarkRun(BaseModel):
+    """Results from a single benchmark run."""
+
+    run_id: int
+    is_warmup: bool
+    metrics: IndexingMetrics
+
+
+class BenchmarkResults(BaseModel):
+    """Complete results from benchmark execution."""
+
+    codebase_name: str
+    codebase_path: str
+    config: BenchmarkConfig
+    warmup_runs: list[BenchmarkRun] = Field(default_factory=list)
+    measured_runs: list[BenchmarkRun] = Field(default_factory=list)
+
+    # Aggregate statistics (calculated after runs)
+    avg_duration_seconds: float = 0.0
+    min_duration_seconds: float = 0.0
+    max_duration_seconds: float = 0.0
+    std_dev_seconds: float = 0.0
+    avg_throughput: float = 0.0
+    avg_memory_mb: float = 0.0
+
+    # Comparison data
+    baseline_duration: float | None = None
+    speedup_factor: float | None = None
+    efficiency: float | None = None
+
+    def add_run(self, run: BenchmarkRun) -> None:
+        """Add a benchmark run to results.
+
+        Args:
+            run: Benchmark run to add
+        """
+        if run.is_warmup:
+            self.warmup_runs.append(run)
+        else:
+            self.measured_runs.append(run)
+
+    def calculate_statistics(self) -> None:
+        """Calculate aggregate statistics from measured runs."""
+        if not self.measured_runs:
+            return
+
+        durations = [r.metrics.total_duration_seconds for r in self.measured_runs]
+        throughputs = [r.metrics.avg_throughput for r in self.measured_runs]
+        memories = [r.metrics.peak_memory_mb for r in self.measured_runs]
+
+        self.avg_duration_seconds = statistics.mean(durations)
+        self.min_duration_seconds = min(durations)
+        self.max_duration_seconds = max(durations)
+        self.std_dev_seconds = (
+            statistics.stdev(durations) if len(durations) > 1 else 0.0
+        )
+        self.avg_throughput = statistics.mean(throughputs)
+        self.avg_memory_mb = statistics.mean(memories)
+
+        # Calculate efficiency if parallel mode with known worker count
+        if (
+            self.config.mode == BenchmarkMode.PARALLEL
+            and self.config.worker_count
+            and self.baseline_duration
+        ):
+            speedup = self.baseline_duration / self.avg_duration_seconds
+            self.speedup_factor = speedup
+            self.efficiency = speedup / self.config.worker_count
+
+    def get_last_metrics(self) -> IndexingMetrics | None:
+        """Get metrics from the last measured run.
+
+        Returns:
+            IndexingMetrics from last run, or None if no runs
+        """
+        if self.measured_runs:
+            return self.measured_runs[-1].metrics
+        return None
+
+
+class MetricsDisplayOptions(BaseModel):
+    """Options for controlling metrics display."""
+
+    show_phase_metrics: bool = True
+    show_worker_metrics: bool = False
+    show_file_metrics: bool = False
+    show_summary_only: bool = False
+    top_n_files: int | None = None
+    min_file_duration_ms: float | None = None

shotgun/codebase/core/__init__.py

@@ -5,6 +5,7 @@ from shotgun.codebase.core.code_retrieval import (
     retrieve_code_by_cypher,
     retrieve_code_by_qualified_name,
 )
+from shotgun.codebase.core.gitignore import GitignoreManager, load_gitignore_for_repo
 from shotgun.codebase.core.ingestor import (
     CodebaseIngestor,
     Ingestor,
@@ -29,6 +30,9 @@ __all__ = [
     "Ingestor",
     "SimpleGraphBuilder",
     "CodebaseGraphManager",
+    # Gitignore support
+    "GitignoreManager",
+    "load_gitignore_for_repo",
     # Language configuration
     "LanguageConfig",
     "LANGUAGE_CONFIGS",

shotgun/codebase/core/call_resolution.py

@@ -0,0 +1,91 @@
+"""Call resolution utilities for function/method call graph building.
+
+This module provides shared utilities for resolving function calls
+and calculating confidence scores for potential callee matches.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Collection, Mapping
+
+
+def calculate_callee_confidence(
+    caller_qn: str,
+    callee_qn: str,
+    module_qn: str,
+    object_name: str | None,
+    simple_name_lookup: Mapping[str, Collection[str]],
+) -> float:
+    """Calculate confidence score for a potential callee match.
+
+    Uses multiple heuristics to determine how likely a given callee
+    is the correct target of a function call:
+    1. Module locality - functions in the same module are most likely
+    2. Package locality - functions in the same package hierarchy
+    3. Object/class match for method calls
+    4. Standard library boost
+    5. Name uniqueness boost
+
+    Args:
+        caller_qn: Qualified name of the calling function
+        callee_qn: Qualified name of the potential callee
+        module_qn: Qualified name of the current module
+        object_name: Object name for method calls (e.g., 'obj' in obj.method())
+        simple_name_lookup: Mapping from simple names to qualified names
+            (supports both set[str] and list[str] values)
+
+    Returns:
+        Confidence score between 0.0 and 1.0
+    """
+    score = 0.0
+
+    # 1. Module locality - functions in the same module are most likely
+    if callee_qn.startswith(module_qn + "."):
+        score += 0.5
+
+        # Even higher if in the same class
+        caller_parts = caller_qn.split(".")
+        callee_parts = callee_qn.split(".")
+        if len(caller_parts) >= 3 and len(callee_parts) >= 3:
+            if caller_parts[:-1] == callee_parts[:-1]:  # Same class
+                score += 0.2
+
+    # 2. Package locality - functions in the same package hierarchy
+    elif "." in module_qn:
+        package = module_qn.rsplit(".", 1)[0]
+        if callee_qn.startswith(package + "."):
+            score += 0.3
+
+    # 3. Object/class match for method calls
+    if object_name:
+        # Check if callee is a method of a class matching the object name
+        callee_parts = callee_qn.split(".")
+        if len(callee_parts) >= 2:
+            # Simple heuristic: check if class name matches object name
+            # (In reality, we'd need type inference for accuracy)
+            class_name = callee_parts[-2]
+            if class_name.lower() == object_name.lower():
+                score += 0.3
+            elif object_name == "self" and callee_qn.startswith(
+                caller_qn.rsplit(".", 1)[0]
+            ):
+                # 'self' refers to the same class
+                score += 0.4
+
+    # 4. Standard library boost
+    # Give a small boost to standard library functions
+    if callee_qn.startswith(("builtins.", "typing.", "collections.")):
+        score += 0.1
+
+    # 5. Name uniqueness boost
+    # If function names are unique enough, boost confidence
+    callee_simple_name = callee_qn.split(".")[-1]
+    possible_matches = simple_name_lookup.get(callee_simple_name, [])
+    possible_count = len(possible_matches)
+    if possible_count == 1:
+        score += 0.2
+    elif possible_count <= 3:
+        score += 0.1
+
+    # Normalize to [0, 1]
+    return min(score, 1.0)

shotgun/codebase/core/change_detector.py

@@ -1,16 +1,21 @@
 """Change detection for incremental graph updates."""
 
+from __future__ import annotations
+
 import hashlib
 import os
 from enum import Enum
 from pathlib import Path
-from typing import Any, cast
+from typing import TYPE_CHECKING, Any, cast
 
 import aiofiles
-import real_ladybug as kuzu
 
+from shotgun.codebase.models import NodeLabel, RelationshipType
 from shotgun.logging_config import get_logger
 
+if TYPE_CHECKING:
+    import real_ladybug as kuzu
+
 logger = get_logger(__name__)
 
 
@@ -332,10 +337,10 @@ class ChangeDetector:
 
         # Query each TRACKS relationship type
         for node_type, rel_type in [
-            ("Module", "TRACKS_Module"),
-            ("Class", "TRACKS_Class"),
-            ("Function", "TRACKS_Function"),
-            ("Method", "TRACKS_Method"),
+            (NodeLabel.MODULE, RelationshipType.TRACKS_MODULE),
+            (NodeLabel.CLASS, RelationshipType.TRACKS_CLASS),
+            (NodeLabel.FUNCTION, RelationshipType.TRACKS_FUNCTION),
+            (NodeLabel.METHOD, RelationshipType.TRACKS_METHOD),
         ]:
             try:
                 result = self.conn.execute(

shotgun/codebase/core/errors.py

@@ -0,0 +1,159 @@
+"""Error classification for Kuzu database operations.
+
+This module provides error classification for Kuzu database errors,
+allowing the application to distinguish between different failure modes
+(lock contention, corruption, permissions, etc.) and handle each appropriately.
+"""
+
+from enum import StrEnum
+from pathlib import Path
+
+from pydantic import BaseModel
+
+
+class KuzuErrorType(StrEnum):
+    """Classification of Kuzu database errors."""
+
+    LOCKED = "locked"  # Another process has DB open
+    CORRUPTION = "corruption"  # Database file is invalid/corrupted
+    PERMISSION = "permission"  # Permission denied (transient)
+    MISSING = "missing"  # File not found
+    SCHEMA = "schema"  # Table doesn't exist (incomplete build)
+    TIMEOUT = "timeout"  # Operation timed out
+    UNKNOWN = "unknown"  # Unrecognized error
+
+
+def classify_kuzu_error(exception: Exception) -> KuzuErrorType:
+    """Classify a Kuzu RuntimeError by its message pattern.
+
+    Note: Kuzu only throws generic RuntimeError exceptions with no error codes
+    or custom exception types. String matching on the error message is the only
+    way to distinguish between different failure modes.
+
+    Args:
+        exception: The exception to classify
+
+    Returns:
+        KuzuErrorType indicating the category of error
+    """
+    error_str = str(exception)
+
+    # Lock contention - another process has the database open
+    if "Could not set lock" in error_str:
+        return KuzuErrorType.LOCKED
+
+    # True corruption - database file is invalid
+    if "Unable to open database" in error_str:
+        return KuzuErrorType.CORRUPTION
+    if "Reading past the end of the file" in error_str:
+        return KuzuErrorType.CORRUPTION
+    if "not a valid" in error_str.lower() and "database" in error_str.lower():
+        return KuzuErrorType.CORRUPTION
+
+    # C++ internal errors - likely corruption
+    if "unordered_map" in error_str:
+        return KuzuErrorType.CORRUPTION
+    if "key not found" in error_str.lower():
+        return KuzuErrorType.CORRUPTION
+    if "std::exception" in error_str:
+        return KuzuErrorType.CORRUPTION
+
+    # Permission errors - transient, may resolve on retry
+    if "Permission denied" in error_str:
+        return KuzuErrorType.PERMISSION
+
+    # Missing file - nothing to delete
+    if "No such file or directory" in error_str:
+        return KuzuErrorType.MISSING
+
+    # Schema errors - incomplete build, table doesn't exist
+    if "Table" in error_str and "does not exist" in error_str:
+        return KuzuErrorType.SCHEMA
+    if "Binder exception" in error_str and "does not exist" in error_str:
+        return KuzuErrorType.SCHEMA
+
+    return KuzuErrorType.UNKNOWN
+
+
+class DatabaseIssue(BaseModel):
+    """Structured information about a database issue.
+
+    Attributes:
+        graph_id: The ID of the affected graph
+        graph_path: Path to the database file
+        error_type: Classification of the error
+        message: Human-readable error message
+    """
+
+    model_config = {"arbitrary_types_allowed": True}
+
+    graph_id: str
+    graph_path: Path
+    error_type: KuzuErrorType
+    message: str
+
+
+class KuzuDatabaseError(Exception):
+    """Base exception for Kuzu database errors with classification."""
+
+    def __init__(
+        self, message: str, graph_id: str, graph_path: str, error_type: KuzuErrorType
+    ) -> None:
+        super().__init__(message)
+        self.graph_id = graph_id
+        self.graph_path = graph_path
+        self.error_type = error_type
+
+
+class DatabaseLockedError(KuzuDatabaseError):
+    """Raised when the database is locked by another process."""
+
+    def __init__(self, graph_id: str, graph_path: str) -> None:
+        super().__init__(
+            f"Database '{graph_id}' is locked by another process. "
+            "Only one shotgun instance can access a codebase at a time.",
+            graph_id=graph_id,
+            graph_path=graph_path,
+            error_type=KuzuErrorType.LOCKED,
+        )
+
+
+class DatabaseCorruptedError(KuzuDatabaseError):
+    """Raised when the database is corrupted."""
+
+    def __init__(self, graph_id: str, graph_path: str, details: str = "") -> None:
+        message = f"Database '{graph_id}' is corrupted"
+        if details:
+            message += f": {details}"
+        super().__init__(
+            message,
+            graph_id=graph_id,
+            graph_path=graph_path,
+            error_type=KuzuErrorType.CORRUPTION,
+        )
+
+
+class DatabaseSchemaError(KuzuDatabaseError):
+    """Raised when the database schema is incomplete (interrupted build)."""
+
+    def __init__(self, graph_id: str, graph_path: str) -> None:
+        super().__init__(
+            f"Database '{graph_id}' has incomplete schema (build was interrupted)",
+            graph_id=graph_id,
+            graph_path=graph_path,
+            error_type=KuzuErrorType.SCHEMA,
+        )
+
+
+class DatabaseTimeoutError(KuzuDatabaseError):
+    """Raised when database operation times out."""
+
+    def __init__(self, graph_id: str, graph_path: str, timeout_seconds: float) -> None:
+        super().__init__(
+            f"Database '{graph_id}' operation timed out after {timeout_seconds}s. "
+            "This can happen with large codebases.",
+            graph_id=graph_id,
+            graph_path=graph_path,
+            error_type=KuzuErrorType.TIMEOUT,
+        )
+        self.timeout_seconds = timeout_seconds

shotgun/codebase/core/extractors/__init__.py

@@ -0,0 +1,23 @@
+"""Language-specific AST extraction framework.
+
+This module provides a Protocol-based architecture for extracting
+definitions, relationships, and metadata from source code ASTs.
+
+Usage:
+    from shotgun.codebase.core.extractors import get_extractor, SupportedLanguage
+
+    extractor = get_extractor(SupportedLanguage.PYTHON)
+    decorators = extractor.extract_decorators(node)
+"""
+
+from __future__ import annotations
+
+from .factory import get_extractor
+from .protocol import LanguageExtractor
+from .types import SupportedLanguage
+
+__all__ = [
+    "SupportedLanguage",
+    "LanguageExtractor",
+    "get_extractor",
+]