contextinator 0.0.post81__py3-none-any.whl

contextinator/__init__.py

@@ -0,0 +1,34 @@
+"""
+Contextinator: Intelligent Codebase Understanding for AI Agents.
+
+Transform any codebase into semantically-aware, searchable knowledge 
+for AI-powered workflows using AST parsing and vector embeddings.
+"""
+
+try:
+    #  the _version.py file is auto-generated when you build the package.
+    from ._version import version as __version__
+except ImportError:
+    # when in dev
+    __version__ = "0.0.0+unknown"
+
+
+
+# Core functionality exports
+from .chunking import chunk_repository
+from .embedding import embed_chunks
+from .vectorstore import store_repository_embeddings, ChromaVectorStore
+from .tools import semantic_search, full_text_search, symbol_search, regex_search
+
+__all__ = [
+    "__version__",
+    "chunk_repository",
+    "embed_chunks",
+    "store_repository_embeddings",
+    "semantic_search",
+    "full_text_search",
+    "symbol_search",       
+    "regex_search",
+    "read_file",  
+    "ChromaVectorStore",
+]

contextinator/__main__.py

@@ -0,0 +1,13 @@
+"""
+Entry point for running Contextinator as a module.
+
+This allows the package to be executed with:
+    python -m contextinator <command> [options]
+
+This is the recommended way to run Contextinator when installed as a package.
+"""
+
+from .cli import main
+
+if __name__ == '__main__':
+    main()

contextinator/_version.py

@@ -0,0 +1,34 @@
+# file generated by setuptools-scm
+# don't change, don't track in version control
+
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
+
+TYPE_CHECKING = False
+if TYPE_CHECKING:
+    from typing import Tuple
+    from typing import Union
+
+    VERSION_TUPLE = Tuple[Union[int, str], ...]
+    COMMIT_ID = Union[str, None]
+else:
+    VERSION_TUPLE = object
+    COMMIT_ID = object
+
+version: str
+__version__: str
+__version_tuple__: VERSION_TUPLE
+version_tuple: VERSION_TUPLE
+commit_id: COMMIT_ID
+__commit_id__: COMMIT_ID
+
+__version__ = version = '0.0.post81'
+__version_tuple__ = version_tuple = (0, 0, 'post81')
+
+__commit_id__ = commit_id = None

contextinator/chunking/__init__.py

@@ -0,0 +1,34 @@
+"""
+Chunking module for Contextinator.
+
+This module provides comprehensive functionality for parsing source code files,
+extracting semantic chunks using AST analysis, and managing the chunking pipeline.
+
+The main components include:
+- File discovery with intelligent ignore patterns
+- AST parsing using Tree-sitter for multiple languages  
+- Semantic node extraction (functions, classes, methods)
+- Chunk splitting and deduplication
+- AST visualization for debugging
+"""
+
+# Core chunking functionality
+from .ast_parser import parse_file
+from .ast_visualizer import save_ast_overview
+from .chunk_service import chunk_repository, load_chunks, save_chunks
+from .context_builder import build_context
+from .file_discovery import discover_files
+from .node_collector import collect_nodes
+from .splitter import split_chunk
+
+__all__ = [
+    'build_context',
+    'chunk_repository',
+    'collect_nodes',
+    'discover_files',
+    'load_chunks',
+    'parse_file',
+    'save_ast_overview',
+    'save_chunks',
+    'split_chunk',
+]

contextinator/chunking/ast_parser.py

@@ -0,0 +1,551 @@
+"""
+Abstract Syntax Tree (AST) parsing module for Contextinator.
+
+This module provides functionality to parse source code files using Tree-sitter
+parsers and extract semantic code chunks like functions, classes, and methods.
+"""
+
+from pathlib import Path
+import uuid
+from typing import Any, Dict, List, Optional, TYPE_CHECKING
+
+from ..config import SUPPORTED_EXTENSIONS
+from ..utils.logger import logger
+
+# Tree-sitter imports with graceful fallback
+try:
+    from tree_sitter import Language, Parser
+    
+    # Language-specific imports
+    import tree_sitter_bash
+    import tree_sitter_c
+    import tree_sitter_c_sharp
+    import tree_sitter_cpp
+    import tree_sitter_go
+    import tree_sitter_java
+    import tree_sitter_javascript
+    import tree_sitter_json
+    import tree_sitter_kotlin
+    import tree_sitter_lua
+    import tree_sitter_markdown
+    import tree_sitter_php
+    import tree_sitter_python
+    import tree_sitter_rust
+    import tree_sitter_solidity
+    import tree_sitter_sql
+    import tree_sitter_swift
+    import tree_sitter_toml
+    import tree_sitter_typescript
+    import tree_sitter_yaml
+    
+    # Optional: dockerfile (not available on Windows)
+    try:
+        import tree_sitter_dockerfile
+        HAS_DOCKERFILE = True
+    except ImportError:
+        tree_sitter_dockerfile = None
+        HAS_DOCKERFILE = False
+        logger.debug("tree-sitter-dockerfile not available (Windows platform)")
+    
+    from .ast_visualizer import save_ast_visualization
+    
+    # Language module mapping for parser creation
+    LANGUAGE_MODULES: Dict[str, Any] = {
+        'python': tree_sitter_python,
+        'javascript': tree_sitter_javascript, 
+        'typescript': tree_sitter_typescript,
+        'tsx': tree_sitter_typescript,  # TSX uses the same TypeScript module
+        'java': tree_sitter_java,
+        'go': tree_sitter_go,
+        'rust': tree_sitter_rust,
+        'cpp': tree_sitter_cpp,
+        'c': tree_sitter_c,
+        'csharp': tree_sitter_c_sharp,
+        'cs': tree_sitter_c_sharp,  # Alternative C# extension
+        'php': tree_sitter_php,
+        'bash': tree_sitter_bash,
+        'sh': tree_sitter_bash,  # Shell scripts
+        'sql': tree_sitter_sql,
+        'kotlin': tree_sitter_kotlin,
+        'kt': tree_sitter_kotlin,  # Kotlin extension
+        'yaml': tree_sitter_yaml,
+        'yml': tree_sitter_yaml,  # Alternative YAML extension
+        'markdown': tree_sitter_markdown,
+        'md': tree_sitter_markdown,  # Markdown extension
+        'json': tree_sitter_json,
+        'toml': tree_sitter_toml,
+        'swift': tree_sitter_swift,
+        'solidity': tree_sitter_solidity,
+        'sol': tree_sitter_solidity,  # Solidity extension
+        'lua': tree_sitter_lua,
+    }
+    
+    # Add dockerfile support if available (platform-dependent)
+    if HAS_DOCKERFILE:
+        LANGUAGE_MODULES['dockerfile'] = tree_sitter_dockerfile
+    
+    TREE_SITTER_AVAILABLE = True
+    logger.info("Tree-sitter imports successful")
+    
+except ImportError as e:
+    TREE_SITTER_AVAILABLE = False
+    LANGUAGE_MODULES = {}
+    logger.warning(f"Tree-sitter import failed: {e}")
+    logger.info("💡 Install missing modules with: pip install tree-sitter tree-sitter-python tree-sitter-javascript ...")
+    
+    if TYPE_CHECKING:
+        from tree_sitter import Parser
+
+# Node types to extract per language for semantic chunking
+NODE_TYPES: Dict[str, List[str]] = {
+    'python': ['function_definition', 'class_definition', 'decorated_definition', 'import_statement', 'import_from_statement'],
+    'javascript': ['function_declaration', 'function_expression', 'arrow_function', 'class_declaration', 'method_definition', 'import_statement'],
+    'typescript': ['function_declaration', 'function_expression', 'arrow_function', 'class_declaration', 'method_definition', 'interface_declaration', 'import_statement'],
+    'tsx': ['function_declaration', 'function_expression', 'arrow_function', 'class_declaration', 'method_definition', 'interface_declaration', 'import_statement'],
+    'java': ['class_declaration', 'method_declaration', 'constructor_declaration', 'interface_declaration', 'import_declaration'],
+    'go': ['function_declaration', 'method_declaration', 'type_declaration', 'import_declaration'],
+    'rust': ['function_item', 'impl_item', 'struct_item', 'enum_item', 'trait_item', 'use_declaration'],
+    'cpp': ['function_definition', 'class_specifier', 'struct_specifier', 'preproc_include'],
+    'c': ['function_definition', 'struct_specifier', 'preproc_include'],
+    'csharp': ['class_declaration', 'method_declaration', 'constructor_declaration', 'interface_declaration', 'property_declaration', 'using_directive'],
+    'cs': ['class_declaration', 'method_declaration', 'constructor_declaration', 'interface_declaration', 'property_declaration', 'using_directive'],
+    'php': ['function_definition', 'class_declaration', 'method_declaration', 'namespace_use_declaration'],
+    'bash': ['function_definition', 'command'],
+    'sh': ['function_definition', 'command'],
+    'sql': ['create_table_statement', 'create_view_statement', 'create_function_statement', 'create_procedure_statement'],
+    'kotlin': ['class_declaration', 'function_declaration', 'property_declaration', 'object_declaration', 'import_header'],
+    'kt': ['class_declaration', 'function_declaration', 'property_declaration', 'object_declaration', 'import_header'],
+    'yaml': ['block_mapping', 'block_sequence'],
+    'yml': ['block_mapping', 'block_sequence'],
+    'markdown': ['section', 'heading', 'code_block'],
+    'md': ['section', 'heading', 'code_block'],
+    'dockerfile': ['instruction'],
+    'json': ['object', 'array'],
+    'toml': ['table', 'key_value'],
+    'swift': ['class_declaration', 'function_declaration', 'protocol_declaration', 'struct_declaration', 'import_declaration'],
+    'solidity': ['contract_declaration', 'function_definition', 'struct_definition', 'event_definition'],
+    'sol': ['contract_declaration', 'function_definition', 'struct_definition', 'event_definition'],
+    'lua': ['function_definition', 'local_function', 'table_constructor'],
+}
+
+
+PARENT_NODE_TYPES: Dict[str, List[str]] = {
+    'python': ['class_definition'],
+    'javascript': ['class_declaration'],
+    'typescript': ['class_declaration', 'interface_declaration'],
+    'tsx': ['class_declaration', 'interface_declaration'],
+    'java': ['class_declaration', 'interface_declaration'],
+    'go': ['type_declaration'],
+    'rust': ['impl_item', 'struct_item', 'enum_item', 'trait_item'],
+    'cpp': ['class_specifier', 'struct_specifier'],
+    'c': ['struct_specifier'],
+    'csharp': ['class_declaration', 'interface_declaration'],
+    'cs': ['class_declaration', 'interface_declaration'],
+    'php': ['class_declaration'],
+    'bash': [],
+    'sh': [],
+    'sql': [],
+    'kotlin': ['class_declaration', 'object_declaration'],
+    'kt': ['class_declaration', 'object_declaration'],
+    'yaml': [],
+    'yml': [],
+    'markdown': [],
+    'md': [],
+    'dockerfile': [],
+    'json': [],
+    'toml': [],
+    'swift': ['class_declaration', 'struct_declaration', 'protocol_declaration'],
+    'solidity': ['contract_declaration', 'struct_definition'],
+    'sol': ['contract_declaration', 'struct_definition'],
+    'lua': ['table_constructor'],
+}
+
+# Cache for parsers to avoid recreation
+_parser_cache: Dict[str, "Parser"] = {}
+
+
+def parse_file(
+    file_path: Path, 
+    save_ast: bool = False, 
+    chunks_dir: Optional[Path] = None,
+    repo_path: Optional[Path] = None
+) -> Optional[Dict[str, Any]]:
+    """
+    Parse a file and return its AST representation with extracted nodes.
+    
+    Args:
+        file_path: Path to the file to parse (absolute path)
+        save_ast: Whether to save AST visualization data
+        chunks_dir: Repository-specific chunks directory for AST data (required if save_ast=True)
+        repo_path: Repository root path for computing relative paths (optional)
+    
+    Returns:
+        Dictionary containing AST nodes and metadata, or None if parsing fails
+        
+    Raises:
+        ValidationError: If save_ast is True but chunks_dir is None
+        FileSystemError: If file cannot be read
+    """
+    from ..utils.exceptions import ValidationError, FileSystemError, ParsingError
+    
+    if save_ast and chunks_dir is None:
+        raise ValidationError("chunks_dir is required when save_ast=True", "chunks_dir", "Path object")
+    
+    # Compute repo-relative path with forward slashes for cross-platform compatibility
+    if repo_path:
+        try:
+            relative_path = file_path.relative_to(repo_path)
+            # Convert to forward slashes for consistency
+            file_path_str = relative_path.as_posix()
+        except ValueError:
+            # If file is not relative to repo_path, use absolute path
+            logger.warning(f"File {file_path} is not within repo {repo_path}, using absolute path")
+            file_path_str = str(file_path)
+    else:
+        file_path_str = str(file_path)
+        
+    try:
+        language = SUPPORTED_EXTENSIONS.get(file_path.suffix)
+        if not language:
+            logger.debug(f"Unsupported file extension: {file_path.suffix}")
+            return None
+        
+        # Handle file reading errors gracefully
+        try:
+            with open(file_path, 'r', encoding='utf-8', errors='ignore') as f:
+                content = f.read()
+        except (OSError, IOError, PermissionError) as e:
+            raise FileSystemError(f"Cannot read file: {e}", str(file_path), "read")
+        
+        if not TREE_SITTER_AVAILABLE:
+            # Fallback when Tree-sitter unavailable
+            logger.warning(f"Tree-sitter not available, using fallback for {file_path}")
+            result = _fallback_parse(file_path, file_path_str, language, content)
+            
+            if save_ast and chunks_dir:
+                _save_ast_safely(file_path, language, None, content, result['nodes'], chunks_dir, result.get('tree_info'))
+            
+            return result
+
+        # Try AST parsing with fallback
+        try:
+            parser = get_parser(language)
+            if not parser:
+                logger.warning(f"No parser available for {language}, using fallback for {file_path}")
+                return _fallback_parse(file_path, file_path_str, language, content)
+            
+            tree = parser.parse(bytes(content, 'utf-8'))
+            nodes = extract_nodes(tree.root_node, content, language)
+            
+            logger.debug(f"Parsed {file_path} - Found {len(nodes)} semantic nodes")
+            
+        except Exception as e:
+            # Fallback to file-level chunking on any parsing error
+            logger.warning(f"AST parsing failed for {file_path}, using fallback: {e}")
+            return _fallback_parse(file_path, file_path_str, language, content)
+        
+        # If no nodes extracted, fallback to file-level
+        if not nodes:
+            logger.warning(f"No semantic nodes found in {file_path}, using file-level chunking")
+            result = _fallback_parse(file_path, file_path_str, language, content)
+        else:
+            result = {
+                'file_path': file_path_str,
+                'language': language,
+                'content': content,
+                'nodes': nodes,
+                'tree_info': {
+                    'has_ast': True,
+                    'root_node_type': tree.root_node.type,
+                    'total_nodes': _count_nodes(tree.root_node),
+                    'tree_depth': _get_tree_depth(tree.root_node)
+                }
+            }
+        
+        # Save AST visualization if requested
+        if save_ast and chunks_dir:
+            root_node = tree.root_node if 'tree_info' in result and result['tree_info'].get('has_ast', False) else None
+            _save_ast_safely(file_path, language, root_node, content, result['nodes'], chunks_dir, result.get('tree_info'))
+        
+        return result
+    
+    except (ValidationError, FileSystemError):
+        # Re-raise our custom exceptions
+        raise
+    except Exception as e:
+        # Pattern 1: Log unexpected errors and continue
+        logger.error(f"Unexpected error parsing {file_path}: {e}")
+        return None
+
+
+def _fallback_parse(file_path: Path, file_path_str: str, language: str, content: str) -> Dict[str, Any]:
+    """
+    Fallback parsing when tree-sitter is unavailable or fails.
+    
+    Args:
+        file_path: Absolute path to the file being parsed (for logging/display)
+        file_path_str: Repo-relative file path string to store in metadata
+        language: Programming language identifier
+        content: File content
+        
+    Returns:
+        Dictionary with file-level chunk information
+    """
+    return {
+        'file_path': file_path_str,
+        'language': language,
+        'content': content,
+        'nodes': [{
+            'type': 'file',
+            'name': file_path.name,
+            'content': content,
+            'start_line': 1,
+            'end_line': len(content.splitlines()),
+            'start_byte': 0,
+            'end_byte': len(content.encode('utf-8'))
+        }],
+        'tree_info': {
+            'has_ast': False,
+            'fallback_reason': 'tree-sitter not available or language modules missing',
+            'parser_available': TREE_SITTER_AVAILABLE
+        }
+    }
+
+
+def get_parser(language: str) -> Optional["Parser"]:
+    """
+    Get tree-sitter parser for language with caching.
+    
+    Args:
+        language: Programming language identifier
+        
+    Returns:
+        Parser instance or None if unavailable
+    """
+    global _parser_cache
+    
+    if not TREE_SITTER_AVAILABLE:
+        return None
+    
+    # Return cached parser
+    if language in _parser_cache:
+        return _parser_cache[language]
+    
+    try:
+        # Get language module
+        lang_module = LANGUAGE_MODULES.get(language)
+        if not lang_module:
+            logger.warning(f"No language module available for {language}")
+            return None
+        
+        # Handle special case for TypeScript/TSX which have different API
+        if language == 'typescript':
+            lang_obj = Language(lang_module.language_typescript())
+        elif language == 'tsx':
+            lang_obj = Language(lang_module.language_tsx())
+        else:
+            # Create Language object from module for other languages
+            lang_obj = Language(lang_module.language())
+        
+        # Create parser with language
+        parser = Parser(lang_obj)
+        _parser_cache[language] = parser
+        return parser
+        
+    except Exception as e:
+        logger.warning(f"Error creating parser for {language}: {e}")
+        return None
+
+def extract_nodes(root_node: Any, content: str, language: str) -> List[Dict[str, Any]]:
+    """
+    Extract relevant nodes from AST based on language-specific node types.
+    
+    Args:
+        root_node: Root node of the AST
+        content: Source code content
+        language: Programming language
+    
+    Returns:
+        List of extracted nodes with metadata including hierarchy
+    """
+    target_types = NODE_TYPES.get(language, [])
+    if not target_types:
+        return []
+    
+    parent_types = set(PARENT_NODE_TYPES.get(language, []))
+    nodes = []
+    content_bytes = content.encode('utf-8')
+    
+    def traverse(node: Any, parent_id: Optional[str] = None, parent_info: Optional[Dict] = None) -> None:
+        """Recursively traverse AST and extract target nodes with hierarchy tracking."""
+        if node.type in target_types:
+            node_id = str(uuid.uuid4())
+            node_content = content_bytes[node.start_byte:node.end_byte].decode('utf-8', errors='ignore')
+            node_name = get_node_name(node, content_bytes)
+            is_parent = node.type in parent_types
+            
+            node_dict = {
+                'id': node_id,
+                'type': node.type,
+                'name': node_name,
+                'content': node_content,
+                'start_line': node.start_point[0] + 1,
+                'end_line': node.end_point[0] + 1,
+                'start_byte': node.start_byte,
+                'end_byte': node.end_byte,
+                'is_parent': is_parent,
+                'parent_id': parent_id,
+                'parent_type': parent_info.get('type') if parent_info else None,
+                'parent_name': parent_info.get('name') if parent_info else None,
+                'children_ids': []
+            }
+            
+            nodes.append(node_dict)
+            
+            if is_parent:
+                for child in node.children:
+                    traverse(child, node_id, {'type': node.type, 'name': node_name})
+            else:
+                for child in node.children:
+                    traverse(child, parent_id, parent_info)
+        else:
+            for child in node.children:
+                traverse(child, parent_id, parent_info)
+    
+    traverse(root_node)
+    
+    # Populate children_ids
+    for node in nodes:
+        if node['is_parent']:
+            node['children_ids'] = [n['id'] for n in nodes if n['parent_id'] == node['id']]
+    
+    return nodes
+
+
+def get_node_name(node: Any, content_bytes: bytes) -> Optional[str]:
+    """
+    Extract name from a node with language-aware and node-type-aware logic.
+    
+    Args:
+        node: AST node
+        content_bytes: Source code as bytes
+        
+    Returns:
+        Node name or generated identifier
+    """
+    try:
+        node_type = node.type
+        
+        # Special handling for different node types
+        if node_type in ('section', 'heading'):
+            for child in node.children:
+                if child.type in ('atx_heading', 'setext_heading'):
+                    text = content_bytes[child.start_byte:child.end_byte].decode('utf-8', errors='ignore')
+                    return text.strip().lstrip('#').strip()[:50]  # First 50 chars
+            first_line = content_bytes[node.start_byte:node.end_byte].decode('utf-8', errors='ignore').split('\n')[0]
+            cleaned = first_line.strip().lstrip('#').strip()[:50]
+            return cleaned if cleaned else f"section_line_{node.start_point[0] + 1}"
+        
+        if node_type == 'arrow_function':
+            parent = node.parent
+            if parent and parent.type in ('variable_declarator', 'lexical_declaration'):
+                for child in parent.children:
+                    if child.type == 'identifier':
+                        return content_bytes[child.start_byte:child.end_byte].decode('utf-8', errors='ignore')
+            return f"arrow_fn_line_{node.start_point[0] + 1}"
+        
+        if node_type in ('object', 'block_mapping'):
+            parent = node.parent
+            if parent and parent.type == 'pair':
+                for child in parent.children:
+                    if child.type in ('string', 'flow_node', 'identifier'):
+                        key = content_bytes[child.start_byte:child.end_byte].decode('utf-8', errors='ignore')
+                        cleaned_key = key.strip('"\'')[:30]
+                        return cleaned_key
+            return f"{node_type}_line_{node.start_point[0] + 1}"
+        
+        if node_type in ('array', 'block_sequence'):
+            parent = node.parent
+            if parent and parent.type == 'pair':
+                for child in parent.children:
+                    if child.type in ('string', 'flow_node', 'identifier'):
+                        key = content_bytes[child.start_byte:child.end_byte].decode('utf-8', errors='ignore')
+                        cleaned_key = key.strip('"\'')[:20]
+                        return f"{cleaned_key}_array"
+            return f"{node_type}_line_{node.start_point[0] + 1}"
+
+        # Generic identifier extraction
+        identifier_types = {'identifier', 'name', 'property_identifier', 'type_identifier', 'field_identifier'}
+        for child in node.children:
+            if child.type in identifier_types:
+                return content_bytes[child.start_byte:child.end_byte].decode('utf-8', errors='ignore')
+        for child in node.children:
+            for grandchild in child.children:
+                if grandchild.type in identifier_types:
+                    return content_bytes[grandchild.start_byte:grandchild.end_byte].decode('utf-8', errors='ignore')
+        
+        return f"anonymous_{node_type}_line_{node.start_point[0] + 1}"
+        
+    except Exception:
+        return f"unknown_line_{node.start_point[0] + 1}" if hasattr(node, 'start_point') else None
+
+
+def _save_ast_safely(file_path: Path, language: str, root_node: Any, content: str, 
+                    nodes: List[Dict[str, Any]], chunks_dir: Path, tree_info: Optional[Dict[str, Any]]) -> None:
+    """
+    Safely save AST visualization with error handling.
+    
+    Args:
+        file_path: Path to source file
+        language: Programming language
+        root_node: AST root node (None for fallback)
+        content: Source code content
+        nodes: Extracted nodes
+        chunks_dir: Directory for AST files
+        tree_info: Tree metadata
+    """
+    try:
+        logger.debug(f"Saving AST for {file_path}")
+        save_ast_visualization(str(file_path), language, root_node, content, nodes, chunks_dir, tree_info)
+    except Exception as e:
+        logger.warning(f"Could not save AST for {file_path}: {e}")
+
+
+def _count_nodes(node: Any) -> int:
+    """
+    Count total number of nodes in the AST.
+    
+    Args:
+        node: AST node
+        
+    Returns:
+        Total node count
+    """
+    return 1 + sum(_count_nodes(child) for child in node.children)
+
+
+def _get_tree_depth(node: Any, current_depth: int = 0) -> int:
+    """
+    Get the maximum depth of the AST.
+    
+    Args:
+        node: AST node
+        current_depth: Current depth level
+        
+    Returns:
+        Maximum tree depth
+    """
+    if not node.children:
+        return current_depth
+    return max(_get_tree_depth(child, current_depth + 1) for child in node.children)
+
+
+__all__ = [
+    'parse_file',
+    'get_parser',
+    'extract_nodes',
+    'get_node_name',
+    'NODE_TYPES',
+    'LANGUAGE_MODULES',
+    'TREE_SITTER_AVAILABLE',
+]