srcodex 0.2.0__py3-none-any.whl

srcodex/backend/services/logger_setup.py

@@ -0,0 +1,91 @@
+"""
+Logging configuration for srcodex
+Sets up file logging to .srcodex/.debug/backend.log
+"""
+import logging
+from pathlib import Path
+from .config_loader import get_config
+
+
+class ColoredFormatter(logging.Formatter):
+    """Custom formatter with ANSI colors"""
+
+    COLORS = {
+        'DEBUG': '\033[36m',      # Cyan
+        'INFO': '\033[37m',       # White
+        'WARNING': '\033[33m',    # Yellow (entire line)
+        'ERROR': '\033[31m',      # Red (entire line)
+        'CRITICAL': '\033[91m',   # Bright Red (entire line)
+    }
+    RESET = '\033[0m'
+    BOLD = '\033[1m'
+
+    def format(self, record):
+        # Format the message first
+        formatted = super().format(record)
+
+        # Color entire line based on level
+        if record.levelname in ['WARNING', 'ERROR', 'CRITICAL']:
+            color = self.COLORS.get(record.levelname, self.RESET)
+            return f"{color}{formatted}{self.RESET}"
+
+        # Special handling for iteration markers (bold white)
+        if 'Iteration' in record.msg:
+            return f"{self.BOLD}{formatted}{self.RESET}"
+
+        # Tool calls in magenta
+        if 'Tool #' in record.msg:
+            return f"\033[35m{formatted}{self.RESET}"
+
+        return formatted
+
+
+def setup_backend_logging():
+    """
+    Configure backend logging to write to .srcodex/.debug/backend.log
+    Creates the .debug directory if it doesn't exist
+    """
+    try:
+        config = get_config()
+        debug_dir = config.project_root / ".srcodex" / ".debug"
+        debug_dir.mkdir(parents=True, exist_ok=True)
+
+        log_file = debug_dir / "backend.log"
+
+        # Get root logger
+        root_logger = logging.getLogger()
+        root_logger.setLevel(logging.INFO)
+
+        # Clear any existing handlers to avoid duplicates
+        root_logger.handlers.clear()
+
+        # File handler - NO colors (plain text for file)
+        file_handler = logging.FileHandler(log_file, mode='a')
+        file_handler.setLevel(logging.INFO)
+        file_handler.setFormatter(
+            logging.Formatter('%(asctime)s [%(levelname)s] %(name)s: %(message)s', datefmt='%Y-%m-%d %H:%M:%S')
+        )
+        root_logger.addHandler(file_handler)
+
+        # Console handler - WITH colors (for terminal output)
+        console_handler = logging.StreamHandler()
+        console_handler.setLevel(logging.INFO)
+        colored_formatter = ColoredFormatter('%(asctime)s [%(levelname)s] %(message)s', datefmt='%H:%M:%S')
+        console_handler.setFormatter(colored_formatter)
+        root_logger.addHandler(console_handler)
+
+        logger = logging.getLogger(__name__)
+        logger.info(f"Backend logging initialized: {log_file}")
+
+        return log_file
+
+    except Exception as e:
+        # Fallback to console-only logging if config fails
+        logging.basicConfig(
+            level=logging.INFO,
+            format='%(asctime)s [%(levelname)s] %(name)s: %(message)s',
+            datefmt='%Y-%m-%d %H:%M:%S'
+        )
+        logger = logging.getLogger(__name__)
+        logger.warning(f"Could not set up file logging: {e}")
+        return None

srcodex/backend/services/session_manager.py

@@ -0,0 +1,81 @@
+"""
+Conversation Session Manager
+Handles saving and loading conversation history across TUI restarts.
+"""
+import json
+from pathlib import Path
+from datetime import datetime
+from typing import List, Dict, Optional
+
+
+class SessionManager:
+    """Manages persistent conversation sessions"""
+
+    def __init__(self, project_root: str):
+        self.project_root = Path(project_root)
+        self.sessions_dir = self.project_root / ".srcodex" / "conversations"
+        self.current_session_file = self.sessions_dir / "latest.json"
+
+        # Create directories if they don't exist
+        self.sessions_dir.mkdir(parents=True, exist_ok=True)
+
+    def save_session(self, messages: List[Dict], metadata: Optional[Dict] = None):
+        """
+        Save conversation history to disk
+
+        Args:
+            messages: List of conversation messages [{"role": "user", "content": "..."}]
+            metadata: Optional metadata (token counts, timestamps, etc.)
+        """
+        session_data = {
+            "created_at": datetime.now().isoformat(),
+            "message_count": len(messages),
+            "metadata": metadata or {},
+            "messages": messages
+        }
+
+        with open(self.current_session_file, 'w') as f:
+            json.dump(session_data, f, indent=2)
+
+    def load_session(self) -> List[Dict]:
+        """
+        Load last conversation history from disk
+
+        Returns:
+            List of messages, or empty list if no session exists
+        """
+        if not self.current_session_file.exists():
+            return []
+
+        try:
+            with open(self.current_session_file, 'r') as f:
+                session_data = json.load(f)
+                return session_data.get("messages", [])
+        except (json.JSONDecodeError, FileNotFoundError):
+            return []
+
+    def load_metadata(self) -> Dict:
+        """
+        Load session metadata from disk
+
+        Returns:
+            Dict of metadata, or empty dict if no session exists
+        """
+        if not self.current_session_file.exists():
+            return {}
+
+        try:
+            with open(self.current_session_file, 'r') as f:
+                session_data = json.load(f)
+                return session_data.get("metadata", {})
+        except (json.JSONDecodeError, FileNotFoundError):
+            return {}
+
+    def clear_session(self):
+        """Delete current session file"""
+        if self.current_session_file.exists():
+            self.current_session_file.unlink()
+
+    def session_exists(self) -> bool:
+        """Check if a saved session exists"""
+        return self.current_session_file.exists()

srcodex/backend/services/status_tracker.py

@@ -0,0 +1,91 @@
+"""
+Status tracker for Claude query execution
+
+Tracks iteration progress, tool calls, and elapsed time for live UI updates.
+"""
+import time
+from typing import Optional, Dict, Any
+
+
+class StatusTracker:
+    """Tracks query execution status for live UI updates"""
+
+    def __init__(self):
+        self.query_start_time: Optional[float] = None
+        self.current_iteration: int = 0
+        self.max_iterations: int = 5
+        self.current_status: str = ""
+        self.is_active: bool = False
+
+    def start_query(self):
+        """Start tracking a new query"""
+        self.query_start_time = time.time()
+        self.current_iteration = 0
+        self.current_status = ""
+        self.is_active = True
+
+    def end_query(self):
+        """Stop tracking the query"""
+        self.is_active = False
+        self.current_status = ""
+
+    def start_iteration(self, iteration: int):
+        """Start a new iteration"""
+        self.current_iteration = iteration
+
+    def set_tool_status(self, tool_name: str, total_tools: int = 1):
+        """
+        Update status with current tool being called
+
+        Args:
+            tool_name: Name of the tool (e.g., 'search_symbols')
+            total_tools: Total number of tools in this batch
+        """
+        if total_tools > 1:
+            self.current_status = f"Calling {tool_name}() +{total_tools-1} more"
+        else:
+            self.current_status = f"Calling {tool_name}()"
+
+    def set_processing_status(self, total_tools: int):
+        """Update status to show processing multiple tools"""
+        self.current_status = f"Processing {total_tools} tools..."
+
+    def set_preparing_answer(self):
+        """Update status to show Claude is preparing the answer"""
+        self.current_status = "Preparing answer..."
+
+    def set_complete(self):
+        """Mark query as complete"""
+        self.current_status = "Complete"
+
+    def get_elapsed_seconds(self) -> int:
+        """Get elapsed time in whole seconds"""
+        if not self.query_start_time:
+            return 0
+        return int(time.time() - self.query_start_time)
+
+    def get_status_message(self) -> Dict[str, Any]:
+        """
+        Get current status as a dict for streaming to frontend
+
+        Returns:
+            {"type": "status", "content": "Calling search_symbols() +34 more", "elapsed": 5}
+        """
+        return {
+            "type": "status",
+            "content": self.current_status,
+            "elapsed": self.get_elapsed_seconds()
+        }
+
+    def format_status_line(self) -> str:
+        """
+        Format status line for display
+
+        Returns:
+            "Calling search_symbols() +34 more | 5s"
+        """
+        if not self.is_active or not self.current_status:
+            return ""
+
+        elapsed = self.get_elapsed_seconds()
+        return f"{self.current_status} | {elapsed}s"

srcodex/cli.py

@@ -0,0 +1,255 @@
+#!/usr/bin/env python3
+"""
+srcodex CLI - Semantic code explorer with AI-powered analysis
+"""
+import click
+import sys
+import os
+import json
+import time
+import subprocess
+from pathlib import Path
+from datetime import datetime
+
+from srcodex.tui.app import SrcodexApp
+from srcodex.core.config import get_config
+from srcodex.indexer.indexer import Indexer
+from srcodex.indexer.field_access_analyzer import FieldAccessAnalyzer
+from srcodex.indexer.reference_ingestor import ReferenceIngestor
+from srcodex.indexer.reference_resolver import ReferenceResolver
+
+@click.command()
+@click.argument('path', default='.', type=click.Path(exists=True))
+@click.option('--reindex', is_flag=True, help='Force re-indexing')
+@click.option('--debug', is_flag=True, help='Enable debug logging')
+def main(path, reindex, debug):
+    """
+    Launch srcodex TUI
+
+    EXAMPLES:
+        srcodex                  # Index current directory and launch
+        srcodex /path/to/code    # Index specific directory
+        srcodex --reindex        # Force re-index and launch
+    """
+
+    click.echo("srcodex v0.1.0 - Semantic code explorer")
+    click.echo()
+
+    project_path = Path(path).resolve()
+    srcodex_dir = project_path / ".srcodex"
+
+    # Check if .srcodex exists
+    if not srcodex_dir.exists() or reindex:
+        if not reindex:
+            # Prompt user
+            if not click.confirm(f"No .srcodex/ found in {project_path}\nIndex this directory?"):
+                click.echo("Cancelled.")
+                return
+
+        click.echo(f"\nIndexing {project_path}...")
+        click.echo("This may take a few minutes for large codebases...\n")
+
+        # Run indexer
+        try:
+            run_indexer(project_path, debug)
+            click.echo("\nIndexing complete!")
+        except Exception as e:
+            click.echo(f"\nError during indexing: {e}", err=True)
+            sys.exit(1)
+
+    # Check for API key
+    if not check_api_key():
+        click.echo("\nError: No API key found!", err=True)
+        click.echo("Please set either:")
+        click.echo("  - ANTHROPIC_API_KEY (for public API)")
+        click.echo("  - AMD_LLM_API_KEY (for enterprise gateway)")
+        click.echo("\nSee .env.example for configuration details")
+        sys.exit(1)
+
+    # Start backend server
+    click.echo("\nStarting backend server...")
+    backend_process = start_backend(project_path, debug)
+
+    try:
+        # Launch TUI (blocking)
+        click.echo("Launching TUI...\n")
+        launch_tui(project_path)
+    except KeyboardInterrupt:
+        click.echo("\n\nShutting down...")
+    finally:
+        # Cleanup
+        if backend_process:
+            backend_process.terminate()
+            backend_process.wait()
+
+
+def run_indexer(project_path: Path, debug: bool = False):
+    """Run indexer and generate .srcodex/"""
+    # Create .srcodex/ directory structure
+    srcodex_dir = project_path / ".srcodex"
+    data_dir = srcodex_dir / "data"
+    data_dir.mkdir(parents=True, exist_ok=True)
+
+    # Database path
+    project_name = project_path.name
+    db_path = data_dir / f"{project_name}.db"
+
+    indexer = Indexer(str(db_path), verbose=debug)
+
+    try:
+        # Connect to database
+        indexer.connect_db()
+
+        # Track timing
+        start_time = time.time()
+
+        # Stage 1: Index symbols (always force-clear for new indexing)
+        click.echo("Stage 1: Extracting symbols with CTags...")
+        indexer.index_directory(str(project_path), extensions=['.c', '.h', '.cpp', '.cc', '.py'], force_clear=True)
+
+        # Stage 1.5: Field access analysis
+        click.echo("Stage 1.5: Analyzing field access patterns...")
+        field_analyzer = FieldAccessAnalyzer(indexer.conn, str(project_path))
+        field_analyzer.analyze_all_functions_parallel(clear_existing=True)
+
+        # Stage 2-4: Build cscope and resolve references
+        click.echo("Stage 2: Building cscope database...")
+        cscope_dir = data_dir / "cscope"
+        cscope_dir.mkdir(exist_ok=True)
+        indexer.build_cscope_database(str(cscope_dir))
+
+        click.echo("Stage 3: Ingesting references...")
+        ingestor = ReferenceIngestor(indexer.conn, str(project_path), cscope_dir)
+        ingestor.ingest_callees(clear_existing=True)
+        ingestor.ingest_callers(clear_existing=True)
+        ingestor.ingest_includes(clear_existing=True)
+
+        click.echo("Stage 4: Resolving semantic edges...")
+
+        resolver = ReferenceResolver(indexer.conn)
+        resolver.resolve_callees(clear_existing=True)
+        resolver.resolve_includes(clear_existing=True)
+
+        # Collect statistics
+        cursor = indexer.conn.cursor()
+
+        files_count = cursor.execute("SELECT COUNT(*) FROM files").fetchone()[0]
+        symbols_count = cursor.execute("SELECT COUNT(*) FROM symbols").fetchone()[0]
+        calls_count = cursor.execute("SELECT COUNT(*) FROM symbol_edges WHERE edge_type = 'CALLS'").fetchone()[0]
+        includes_count = cursor.execute("SELECT COUNT(*) FROM symbol_edges WHERE edge_type = 'INCLUDES'").fetchone()[0]
+        accesses_count = cursor.execute("SELECT COUNT(*) FROM symbol_edges WHERE edge_type = 'ACCESSES'").fetchone()[0]
+        avg_lines = 500  # Estimate: average C/Python file size
+
+        duration = time.time() - start_time
+
+        # Generate metadata.json
+        metadata = {
+            "project": {
+                "name": project_name,
+                "source_root": str(project_path),
+                "indexed_at": datetime.now().isoformat(),
+                "indexer_version": "0.1.0"
+            },
+            "stats": {
+                "files_indexed": files_count,
+                "total_symbols": symbols_count,
+                "avg_lines_per_file": int(avg_lines),
+                "edges": {
+                    "calls": calls_count,
+                    "includes": includes_count,
+                    "accesses": accesses_count
+                },
+                "indexing_duration_seconds": round(duration, 2)
+            },
+            "paths": {
+                "source_root": ".",
+                "database": f".srcodex/data/{project_name}.db"
+            }
+        }
+
+        metadata_path = srcodex_dir / "metadata.json"
+        with open(metadata_path, 'w') as f:
+            json.dump(metadata, f, indent=2)
+
+        # Summary
+        click.echo(f"\n✓ Indexed {files_count:,} files, {symbols_count:,} symbols")
+        click.echo(f"✓ Edges: {calls_count:,} CALLS, {includes_count:,} INCLUDES, {accesses_count:,} ACCESSES")
+        click.echo(f"✓ Generated .srcodex/metadata.json")
+        click.echo(f"✓ Duration: {duration:.1f} seconds")
+
+    finally:
+        indexer.close_db()
+
+
+def start_backend(project_path: Path, debug: bool = False):
+    """Start FastAPI backend in background subprocess"""
+
+    # Set environment variable for project root
+    env = os.environ.copy()
+    env['SRCODEX_PROJECT_ROOT'] = str(project_path)
+
+    # Start uvicorn in background
+    stdout = None if debug else subprocess.DEVNULL
+    stderr = None if debug else subprocess.DEVNULL
+
+    proc = subprocess.Popen(
+        [sys.executable, '-m', 'uvicorn', 'srcodex.backend.chat:app', '--port', '8000'],
+        env=env,
+        stdout=stdout,
+        stderr=stderr,
+        cwd=str(project_path)
+    )
+
+    time.sleep(2)
+    return proc
+
+
+def launch_tui(project_path: Path):
+    """Launch Textual TUI"""
+    os.chdir(project_path)
+    app = SrcodexApp()
+    app.run()
+
+
+def check_api_key() -> bool:
+    """Check if any API key is configured"""
+    amd_key = os.getenv("AMD_LLM_API_KEY")
+    anthropic_key = os.getenv("ANTHROPIC_API_KEY")
+
+    return (amd_key and amd_key != "dummy") or bool(anthropic_key)
+
+
+@click.command()
+@click.argument('path', type=click.Path(exists=True))
+def index(path):
+    """Index a codebase without launching TUI"""
+    project_path = Path(path).resolve()
+    click.echo(f"Indexing {project_path}...")
+    run_indexer(project_path)
+    click.echo("Complete!")
+
+
+@click.command()
+def info():
+    """Show project stats from .srcodex/metadata.json"""
+    try:
+        config = get_config()
+        stats = config.stats
+
+        click.echo(f"\nProject: {config.project_name}")
+        click.echo(f"Source root: {config.source_root}")
+        click.echo(f"Indexed at: {config.indexed_at}")
+        click.echo(f"\nStats:")
+        click.echo(f"  Files: {stats['files_indexed']:,}")
+        click.echo(f"  Symbols: {stats['total_symbols']:,}")
+        click.echo(f"  Call edges: {stats['edges']['calls']:,}")
+        click.echo(f"  Include edges: {stats['edges']['includes']:,}")
+        click.echo(f"  Field access edges: {stats['edges']['accesses']:,}")
+    except FileNotFoundError as e:
+        click.echo(f"Error: {e}", err=True)
+        click.echo("No .srcodex/ found in current directory")
+        sys.exit(1)
+
+
+if __name__ == '__main__':
+    main()

srcodex/core/config.py

@@ -0,0 +1,113 @@
+"""
+Configuration Loader - Reads .srcodex/metadata.json
+Provides project paths and stats to all services
+"""
+import json
+from pathlib import Path
+from typing import Dict, Any
+
+
+class ProjectConfig:
+    """Loads and provides access to .srcodex/metadata.json"""
+
+    def __init__(self, project_root: Path = None):
+        """
+        Initialize config loader
+
+        Args:
+            project_root: Path to project root (defaults to searching upwards from cwd)
+        """
+        if project_root is None:
+            # Search upwards from current directory for .srcodex/
+            project_root = self._find_project_root()
+
+        self.project_root = Path(project_root)
+        self.srcodex_dir = self.project_root / ".srcodex"
+        self.metadata_file = self.srcodex_dir / "metadata.json"
+
+        # Load metadata
+        if not self.metadata_file.exists():
+            raise FileNotFoundError(
+                f"No .srcodex/metadata.json found in {self.project_root}\n"
+                f"Run indexer first to generate .srcodex/ directory"
+            )
+
+        with open(self.metadata_file, 'r') as f:
+            self.metadata = json.load(f)
+
+    def _find_project_root(self) -> Path:
+        """
+        Search upwards from current directory for .srcodex/ directory
+
+        Returns:
+            Path to project root containing .srcodex/
+
+        Raises:
+            FileNotFoundError if .srcodex/ not found in any parent directory
+        """
+        current = Path.cwd()
+
+        # Search up to 10 levels (prevent infinite loop)
+        for _ in range(10):
+            srcodex_dir = current / ".srcodex"
+            if srcodex_dir.exists() and srcodex_dir.is_dir():
+                return current
+
+            # Move to parent
+            parent = current.parent
+            if parent == current:
+                # Reached filesystem root
+                break
+            current = parent
+
+        raise FileNotFoundError(
+            f"No .srcodex/ directory found in {Path.cwd()} or any parent directory.\n"
+            f"Run indexer first to generate .srcodex/ directory"
+        )
+
+        with open(self.metadata_file, 'r') as f:
+            self.metadata = json.load(f)
+
+    @property
+    def project_name(self) -> str:
+        """Get project name"""
+        return self.metadata["project"]["name"]
+
+    @property
+    def source_root(self) -> Path:
+        """Get absolute path to source root"""
+        return self.project_root / self.metadata["paths"]["source_root"]
+
+    @property
+    def database_path(self) -> Path:
+        """Get absolute path to database"""
+        return self.project_root / self.metadata["paths"]["database"]
+
+    @property
+    def stats(self) -> Dict[str, Any]:
+        """Get project statistics"""
+        return self.metadata["stats"]
+
+    @property
+    def indexed_at(self) -> str:
+        """Get indexing timestamp"""
+        return self.metadata["project"]["indexed_at"]
+
+
+# Global config instance (singleton pattern)
+_config = None
+
+def get_config(project_root: Path = None) -> ProjectConfig:
+    """
+    Get global project configuration (singleton)
+
+    Args:
+        project_root: Path to project root (only used on first call)
+
+    Returns:
+        ProjectConfig instance
+    """
+    global _config
+    if _config is None:
+        _config = ProjectConfig(project_root)
+    return _config

srcodex/core/logger.py

@@ -0,0 +1,23 @@
+import logging
+from pathlib import Path
+
+# Create logs directory
+LOG_DIR = Path(__file__).parent / "logs"
+LOG_DIR.mkdir(exist_ok=True)
+
+# Single log file (overwrites on each run)
+log_file = LOG_DIR / "srcodex.log"
+
+# Setup logger
+logger = logging.getLogger("srcodex")
+logger.setLevel(logging.DEBUG)
+
+# File handler (mode='w' overwrites the file)
+handler = logging.FileHandler(log_file, mode='w')
+handler.setLevel(logging.DEBUG)
+
+# Format
+formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
+handler.setFormatter(formatter)
+
+logger.addHandler(handler)