srcodex 0.2.0__py3-none-any.whl

srcodex/indexer/ctags_parser.py

@@ -0,0 +1,456 @@
+#!/usr/bin/env python3
+"""
+SRC Code Explorer - CTags Parser
+Wrapper around Universal CTags to extract symbols from C code
+"""
+
+import subprocess
+import json
+import os
+import tempfile
+from typing import List, Dict, Optional
+from pathlib import Path
+from tqdm import tqdm
+from .ctags_compat import verify_ctags_compatibility
+
+
+class CTagsParser:
+    """Parse C source code using Universal CTags"""
+
+    def __init__(self, ctags_bin: str = "ctags"):
+        """
+        Initialize CTags parser
+
+        Args:
+            ctags_bin: Path to ctags binary (default: "ctags")
+        """
+        self.ctags_bin = ctags_bin
+        self._verify_ctags()
+
+    def _verify_ctags(self):
+        """Verify that ctags is installed and compatible"""
+        verify_ctags_compatibility(self.ctags_bin)
+
+    def parse_root(self, root_dir: str, extensions: List[str] = None, source_root: Optional[str] = None) -> Dict[str, List[Dict]]:
+        """
+        Parse entire directory tree with SINGLE ctags invocation (efficient for large codebases).
+        This is the RECOMMENDED method for production indexing. Runs ctags once on all files,
+        vastly faster than per-file invocation (n files: 1 invocation vs n invocations).
+
+        Args:
+            root_dir: Root directory to scan
+            extensions: File extensions to include (default: ['.c', '.h'])
+            source_root: Root directory for canonical path computation (default: root_dir)
+                        All returned paths will be relative to this directory in POSIX format.
+
+        Returns:
+            Dictionary mapping CANONICAL file paths (rel_posix from source_root) to symbol lists:
+            {
+                'path/to/file.c': [symbol1, symbol2, ...],
+                'path/to/file.h': [symbol3, ...],
+            }
+        """
+        if extensions is None:
+            extensions = ['.c', '.h']
+
+        root_path = Path(root_dir).resolve()
+        if not root_path.exists():
+            raise FileNotFoundError(f"Directory not found: {root_dir}")
+
+        # Determine source_root for canonical path computation
+        if source_root is None:
+            source_root_path = root_path
+        else:
+            source_root_path = Path(source_root).resolve()
+
+        # Find all matching files
+        file_list = []
+        for ext in extensions:
+            file_list.extend(root_path.rglob(f'*{ext}'))
+
+        if not file_list:
+            return {}
+
+        # Run ctags ONCE on all files using -L (file list from stdin)
+        # This avoids "Argument list too long" errors on large codebases
+        with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.filelist') as f:
+            filelist_path = f.name
+            for file_path in file_list:
+                f.write(f"{file_path}\n")
+
+        try:
+            cmd = [
+                self.ctags_bin,
+                "--output-format=json",
+                "--fields=+nKSz",
+                "--kinds-C=+p",
+                "-f", "-",
+                "-L", filelist_path
+            ]
+
+            # Stream ctags output with progress bar
+            process = subprocess.Popen(
+                cmd,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                text=True
+            )
+
+            # Parse JSON output - TWO PASS approach PER FILE
+            # CRITICAL: Build anon_to_typedef separately for EACH file to avoid cross-file pollution
+            # (Same __anonXXX token can appear in multiple files with different typedef names)
+
+            # Pass 1: Parse all tags and group by file (with progress)
+            raw_tags = []
+            with tqdm(desc="Running ctags", unit=" symbols") as pbar:
+                for line in iter(process.stdout.readline, ''):
+                    if not line:
+                        break
+                    line = line.strip()
+                    if not line or line.startswith('!'):
+                        continue
+                    try:
+                        tag = json.loads(line)
+                        raw_tags.append(tag)
+                        pbar.update(1)
+                    except json.JSONDecodeError:
+                        continue
+
+            # Wait for process to complete
+            process.wait()
+            if process.returncode != 0:
+                stderr = process.stderr.read()
+                print(f"Warning: ctags failed with code {process.returncode}: {stderr}")
+                return {}
+
+            print(f"✓ CTags complete: {len(raw_tags)} symbols extracted")
+
+        except Exception as e:
+            print(f"Warning: ctags failed: {e}")
+            return {}
+        finally:
+            # Clean up temp file
+            try:
+                os.unlink(filelist_path)
+            except OSError:
+                pass
+
+        # Two-pass symbol processing:
+        # Pass 1: Build typedef mappings for anonymous struct/union/enum resolution
+        # Pass 2: Parse all symbols with resolved typedef names
+        # Note: Pass 1 must complete before Pass 2 since member symbols may reference
+        # typedefs defined later in the file
+
+        # Pass 1: Collect typedef mappings per file
+        anon_to_typedef_by_file = {}
+        for tag in raw_tags:
+            if tag.get('kind') == 'typedef':
+                file_path = tag.get('path')
+                if file_path:
+                    file_path_abs = Path(file_path).resolve()
+                    file_path_canonical = file_path_abs.relative_to(source_root_path).as_posix()
+
+                    typeref = tag.get('typeref', '')
+                    if typeref.startswith('struct:') or typeref.startswith('union:') or typeref.startswith('enum:'):
+                        anon_name = typeref.split(':', 1)[1]
+                        typedef_name = tag.get('name')
+                        if anon_name.startswith('__anon') and typedef_name:
+                            if file_path_canonical not in anon_to_typedef_by_file:
+                                anon_to_typedef_by_file[file_path_canonical] = {}
+                            anon_to_typedef_by_file[file_path_canonical][anon_name] = typedef_name
+
+        # Pass 2: Parse all symbols with typedef resolution
+        results = {}
+        with tqdm(total=len(raw_tags), desc="Parsing symbols", unit=" tags") as pbar:
+            for tag in raw_tags:
+                file_path = tag.get('path')
+                if not file_path:
+                    pbar.update(1)
+                    continue
+
+                # Normalize key: canonical rel_posix
+                file_path_abs = Path(file_path).resolve()
+                file_path_canonical = file_path_abs.relative_to(source_root_path).as_posix()
+
+                # Use ONLY this file's anon mapping
+                file_anon_map = anon_to_typedef_by_file.get(file_path_canonical, {})
+                symbol = self._parse_tag(tag, file_path, file_anon_map)
+                if symbol:
+                    if file_path_canonical not in results:
+                        results[file_path_canonical] = []
+                    results[file_path_canonical].append(symbol)
+
+                pbar.update(1)
+
+        return results
+
+    def parse_file(self, file_path: str) -> List[Dict]:
+        """
+        Parse a single file and extract symbols.
+
+        This method is INEFFICIENT for bulk indexing (runs ctags once per file).
+        Use parse_root() for production indexing of directories.
+        This method is kept for:
+        - Debugging individual files
+        - Incremental updates of single files
+        - Testing
+
+        Args:
+            file_path: Path to C source file
+
+        Returns:
+            List of symbol dictionaries with keys:
+                - name: Symbol name
+                - type: Symbol type (function, variable, struct, etc.)
+                - line: Line number
+                - signature: Full signature (if available)
+                - scope: Scope (global, static, etc.)
+                - scope_kind: Parent scope kind (struct, union, enum)
+                - scope_name: Parent scope name (PowerState, Dummy, etc.)
+        """
+        if not os.path.exists(file_path):
+            raise FileNotFoundError(f"File not found: {file_path}")
+
+        # Run ctags with JSON output
+        cmd = [
+            self.ctags_bin,
+            "--output-format=json",
+            "--fields=+nKSz",  # +n (line numbers), +K (kind), +S (signature), +z (scope)
+            "--kinds-C=+p",    # Include function prototypes (modern syntax)
+            "-f", "-",         # Output to stdout
+            file_path
+        ]
+
+        try:
+            result = subprocess.run(
+                cmd,
+                capture_output=True,
+                text=True,
+                check=True
+            )
+        except subprocess.CalledProcessError as e:
+            print(f"Warning: ctags failed on {file_path}: {e}")
+            return []
+
+        # Parse JSON output - TWO PASS approach:
+        # Pass 1: Build mapping of anonymous structs to typedef names
+        # Pass 2: Parse all tags and resolve anonymous struct references
+
+        raw_tags = []
+        anon_to_typedef = {}  # Maps __anonXXX -> typedef name
+
+        for line in result.stdout.strip().split('\n'):
+            if not line or line.startswith('!'):
+                continue
+
+            try:
+                tag = json.loads(line)
+                raw_tags.append(tag)
+
+                # If this is a typedef for a struct/union/enum, record the mapping
+                if tag.get('kind') == 'typedef':
+                    typeref = tag.get('typeref', '')
+                    if typeref.startswith('struct:') or typeref.startswith('union:') or typeref.startswith('enum:'):
+                        # typeref is like "struct:__anondd0b9e6c0108"
+                        anon_name = typeref.split(':', 1)[1]
+                        typedef_name = tag.get('name')
+                        if anon_name.startswith('__anon') and typedef_name:
+                            anon_to_typedef[anon_name] = typedef_name
+            except json.JSONDecodeError:
+                continue
+
+        # Pass 2: Parse all tags with resolved scope names
+        symbols = []
+        for tag in raw_tags:
+            symbol = self._parse_tag(tag, file_path, anon_to_typedef)
+            if symbol:
+                symbols.append(symbol)
+
+        return symbols
+
+    def _parse_tag(self, tag: Dict, file_path: str, anon_to_typedef: Dict[str, str] = None) -> Optional[Dict]:
+        """
+        Parse a ctags tag into our symbol format
+        Args:
+            tag: Raw ctags tag dictionary
+            file_path: Source file path
+            anon_to_typedef: Mapping from anonymous struct names to typedef names
+        Returns:
+            Symbol dictionary or None if invalid
+        """
+        if anon_to_typedef is None:
+            anon_to_typedef = {}
+        # Extract basic info
+        name = tag.get('name')
+        kind = tag.get('kind')
+        line = tag.get('line', 0)
+
+        if not name or not kind:
+            return None
+
+        if name.startswith('__anon'):
+            return None
+
+        # Store raw ctags kind BEFORE normalization
+        kind_raw = kind
+
+        # Extract raw typeref and signature from ctags (before we process them)
+        # Store NULL if not provided - DO NOT invent values
+        raw_typeref = tag.get('typeref') if 'typeref' in tag else None
+        raw_signature = tag.get('signature') if 'signature' in tag else None
+
+        # Handle typedef structs/unions/enums - treat them as struct/union/enum with the typedef name
+        if kind == 'typedef' and raw_typeref:
+            if raw_typeref.startswith('struct:'):
+                symbol_type = 'struct'
+            elif raw_typeref.startswith('union:'):
+                symbol_type = 'union'
+            elif raw_typeref.startswith('enum:'):
+                symbol_type = 'enum'
+            else:
+                symbol_type = 'typedef'
+        else:
+            # Map ctags kinds to our NORMALIZED types
+            # NOTE: Both 'prototype' and 'function' map to 'function', but we keep kind_raw to distinguish
+            type_map = {
+                'function': 'function',
+                'prototype': 'function',  # Declaration in .h
+                'variable': 'variable',
+                'struct': 'struct',
+                'union': 'union',
+                'enum': 'enum',
+                'enumerator': 'enumerator',
+                'typedef': 'typedef',
+                'macro': 'macro',
+                'member': 'member',
+                'header': 'header',
+            }
+            symbol_type = type_map.get(kind, kind)
+
+        # Extract scope information
+        scope = 'global'
+        scope_kind = None
+        scope_name = None
+
+        # Extract parent scope (struct/union/enum/class)
+        if 'scopeKind' in tag and 'scope' in tag:
+            parent_scope_name = tag['scope']
+
+            # Resolve anonymous struct names to their typedef names
+            if parent_scope_name.startswith('__anon') and parent_scope_name in anon_to_typedef:
+                parent_scope_name = anon_to_typedef[parent_scope_name]
+
+            # Store scope info (skip only if still anonymous after resolution)
+            if not parent_scope_name.startswith('__anon'):
+                scope_kind = tag['scopeKind']
+                scope_name = parent_scope_name
+
+        # Detect file-local scope (static in C)
+        # ctags provides this via the 'file' boolean field or 'fileScope' in extras
+        is_file_scope = None  # NULL = unknown
+
+        # Check the 'file' boolean field (most reliable)
+        if 'file' in tag:
+            is_file_scope = 1 if tag['file'] else 0
+        # Fallback: check 'extras' for 'fileScope'
+        elif 'extras' in tag:
+            # Normalize extras: can be list or string depending on ctags version
+            extras = tag.get('extras')
+            if isinstance(extras, str):
+                # Old format: comma-separated string
+                extras = [e.strip() for e in extras.split(',') if e.strip()]
+            elif not isinstance(extras, list):
+                # Unknown format, treat as empty
+                extras = []
+
+            # Check if fileScope is in the list
+            if 'fileScope' in extras:
+                is_file_scope = 1
+            else:
+                is_file_scope = 0
+
+        # Keep old 'scope' field for backwards compatibility (deprecated)
+        if is_file_scope == 1:
+            scope = 'static'
+
+        return {
+            'name': name,
+            'type': symbol_type,         # Normalized type (prototype -> function)
+            'kind_raw': kind_raw,        # Raw ctags kind (prototype, function, etc.)
+            'line': line,
+            'signature': raw_signature,  # Raw from ctags, NULL if not available
+            'typeref': raw_typeref,      # Raw from ctags, NULL if not available
+            'scope': scope,  # Deprecated: kept for backwards compatibility
+            'scope_kind': scope_kind,
+            'scope_name': scope_name,
+            'is_file_scope': is_file_scope,
+            'file_path': file_path
+        }
+
+    def parse_directory(self, dir_path: str, extensions: List[str] = None) -> Dict[str, List[Dict]]:
+        """
+        Parse all files in a directory recursively.
+
+        DEPRECATED: Use parse_root() instead for better performance.
+        This method now delegates to parse_root().
+
+        Args:
+            dir_path: Directory to scan
+            extensions: File extensions to include (default: ['.c', '.h'])
+        Returns:
+            Dictionary mapping file paths to symbol lists
+        """
+        return self.parse_root(dir_path, extensions)
+
+
+# Simple test
+if __name__ == "__main__":
+    import sys
+
+    if len(sys.argv) < 2:
+        print("Usage: python ctags_parser.py <file_or_directory>")
+        sys.exit(1)
+
+    parser = CTagsParser()
+    path = sys.argv[1]
+
+    if os.path.isfile(path):
+        symbols = parser.parse_file(path)
+        print(f"Found {len(symbols)} symbols in {path}:")
+        for sym in symbols:  # Show all symbols
+            # Build qualified name if it has a parent scope
+            if sym.get('scope_kind') and sym.get('scope_name'):
+                qualified = f"{sym['scope_name']}.{sym['name']}"
+                scope_info = f" ({sym['scope_kind']}:{sym['scope_name']})"
+            else:
+                qualified = sym['name']
+                scope_info = ""
+
+            # Add file-scope indicator
+            file_scope_indicator = ""
+            if sym.get('is_file_scope') == 1:
+                file_scope_indicator = " [file-local]"
+            elif sym.get('is_file_scope') == 0:
+                file_scope_indicator = " [global]"
+
+            # Add signature for functions (including return type from typeref)
+            sig_display = ""
+            if sym['type'] == 'function':
+                # Build full signature: "return_type name(params)"
+                return_type = ""
+                if sym.get('typeref'):
+                    # typeref is like "typename:void" or "typename:int"
+                    return_type = sym['typeref'].replace('typename:', '') + ' '
+
+                params = sym.get('signature', '()')
+                sig_display = f"{return_type}{params}"
+
+            print(f"  {sym['type']:12} {qualified:30}{sig_display:40} @ line {sym['line']}{scope_info}{file_scope_indicator}")
+    else:
+        results = parser.parse_directory(path)
+        total = sum(len(syms) for syms in results.values())
+        print(f"Found {total} symbols in {len(results)} files")
+        for file_path, symbols in list(results.items())[:5]:  # Show first 5 files
+            print(f"\n{file_path}: {len(symbols)} symbols")
+            for sym in symbols[:5]:
+                print(f"  {sym['type']:12} {sym['name']:30} @ line {sym['line']}")

srcodex/indexer/explorer.py

@@ -0,0 +1,135 @@
+#!/usr/bin/env python3
+"""
+Explorer - Unified File Discovery Module
+Used by both indexer and cscope to ensure consistent file sets
+
+This module provides FileDiscovery class for finding source files
+with consistent filtering rules across all tools.
+"""
+
+from pathlib import Path
+from typing import List, Set
+
+
+DEFAULT_IGNORE_DIRS = {
+    '.git',
+    '__pycache__',
+    'out',
+    'build',
+    'dist',
+    '.pytest_cache',
+    'node_modules',
+    '.venv',
+    'venv'
+}
+
+
+class FileDiscovery:
+    """
+    Discovers source files in a directory with consistent filtering
+
+    CRITICAL: Both indexer and cscope MUST use this same discovery logic
+    to ensure they index the exact same set of files.
+    """
+
+    def __init__(
+        self,
+        source_root: str,
+        extensions: List[str] = None,
+        ignore_dirs: Set[str] = None
+    ):
+        """
+        Args:
+            source_root: Root directory to scan
+            extensions: File extensions to include (default: ['.c', '.h'])
+            ignore_dirs: Directory names to skip (default: DEFAULT_IGNORE_DIRS)
+        """
+        self.source_root = Path(source_root).resolve()
+        self.extensions = extensions or [
+            '.c', '.h',           # C
+            '.cpp', '.cc', '.cxx', '.hpp', '.hxx',  # C++
+            '.py',                # Python
+            '.mk',                # Makefiles
+            '.java',              # Java
+            '.rs',                # Rust
+        ]
+        self.ignore_dirs = ignore_dirs or DEFAULT_IGNORE_DIRS
+
+        if not self.source_root.exists():
+            raise FileNotFoundError(f"Directory not found: {source_root}")
+
+        if not self.source_root.is_dir():
+            raise NotADirectoryError(f"Not a directory: {source_root}")
+
+    def discover_files(self) -> List[str]:
+        """
+        Find all files matching extensions, with ignore filters
+
+        Returns:
+            List of POSIX-formatted relative paths from source_root
+            Example: ['power.c', 'drivers/thermal.c', 'include/power.h']
+        """
+        files = []
+
+        for ext in self.extensions:
+            for file_path in self.source_root.rglob(f'*{ext}'):
+                if self._should_ignore(file_path):
+                    continue
+                # Convert to relative POSIX path
+                rel_path = file_path.relative_to(self.source_root)
+                files.append(rel_path.as_posix())
+
+        return sorted(files)
+
+    def _should_ignore(self, file_path: Path) -> bool:
+        """
+            True if file should be ignored, False otherwise
+        """
+        for part in file_path.parts:
+            if part in self.ignore_dirs:
+                return True
+
+        return False
+
+    def discover_files_absolute(self) -> List[Path]:
+        """
+        Find all files matching extensions, with ignore filters
+
+        Returns:
+            List of absolute Path objects
+        """
+        files = []
+
+        for ext in self.extensions:
+            for file_path in self.source_root.rglob(f'*{ext}'):
+                if self._should_ignore(file_path):
+                    continue
+
+                files.append(file_path)
+
+        return sorted(files)
+
+    def get_stats(self) -> dict:
+        """
+        Returns:
+            Dictionary with file counts and extensions
+        """
+        files = self.discover_files()
+
+        # Count by extension
+        ext_counts = {}
+        for file_path in files:
+            ext = Path(file_path).suffix
+            ext_counts[ext] = ext_counts.get(ext, 0) + 1
+
+        return {
+            'total_files': len(files),
+            'extensions': ext_counts,
+            'source_root': str(self.source_root)
+        }
+
+
+# Convenience function for quick usage
+def discover_files(source_root: str, extensions: List[str] = None) -> List[str]:
+    discovery = FileDiscovery(source_root, extensions)
+    return discovery.discover_files()