raggiecode 0.2.1__py3-none-any.whl

indexing/export_to_json.py

@@ -0,0 +1,346 @@
+#!/usr/bin/env python3
+"""
+Export SQLite code index to JSON format.
+"""
+
+import json
+import sqlite3
+from pathlib import Path
+
+
+def export_to_json(db_path, output_file):
+    """Export SQLite database to JSON format."""
+    print(f"Reading database from {db_path}...")
+    
+    conn = sqlite3.connect(db_path)
+    conn.row_factory = sqlite3.Row
+    
+    # Initialize JSON structure
+    json_index = {
+        "files": {},
+        "total_functions": 0,
+        "total_classes": 0,
+        "total_variables": 0,
+        "total_methods": 0,
+        "total_type_defs": 0,
+        "total_structs": 0,
+        "total_interfaces": 0
+    }
+    
+    cursor = conn.cursor()
+    
+    # Get all files
+    cursor.execute("SELECT * FROM files ORDER BY id")
+    files = cursor.fetchall()
+    
+    print(f"Exporting {len(files)} files...")
+    
+    for file_row in files:
+        file_id = file_row['id']
+        relative_path = file_row['path']
+        
+        file_info = {
+            "path": relative_path,
+            "absolute_path": file_row['absolute_path'],
+            "language": file_row['language'],
+            "functions": [],
+            "classes": [],
+            "variables": [],
+            "type_aliases": [],
+            "structs": [],
+            "interfaces": []
+        }
+        
+        # Get top-level functions (no parent)
+        cursor.execute(
+            """SELECT * FROM functions 
+               WHERE file_id = ? AND parent_id IS NULL 
+               ORDER BY id""",
+            (file_id,)
+        )
+        for func_row in cursor.fetchall():
+            func_info = {
+                "type": func_row['type'],
+                "name": func_row['name'],
+                "location": json.loads(func_row['location']),
+                "parameters": json.loads(func_row['parameters']),
+                "return_type": func_row['return_type'],
+                "docstring": func_row['docstring']
+            }
+            if func_row['receiver']:
+                func_info['receiver'] = func_row['receiver']
+            file_info["functions"].append(func_info)
+            json_index["total_functions"] += 1
+        
+        # Get classes (top-level only)
+        cursor.execute(
+            """SELECT * FROM classes 
+               WHERE file_id = ? AND parent_id IS NULL 
+               ORDER BY id""",
+            (file_id,)
+        )
+        classes_map = {}  # Map class_id to class info for method lookup
+        
+        for class_row in cursor.fetchall():
+            class_id = class_row['id']
+            class_info = {
+                "type": "class",
+                "name": class_row['name'],
+                "location": json.loads(class_row['location']),
+                "base_classes": json.loads(class_row['base_classes']),
+                "docstring": class_row['docstring'],
+                "methods": [],
+                "nested_classes": [],
+                "variables": []
+            }
+            classes_map[class_id] = class_info
+            file_info["classes"].append(class_info)
+            json_index["total_classes"] += 1
+        
+        # Get nested classes
+        cursor.execute(
+            """SELECT * FROM classes 
+               WHERE file_id = ? AND parent_id IS NOT NULL 
+               ORDER BY id""",
+            (file_id,)
+        )
+        for class_row in cursor.fetchall():
+            class_id = class_row['id']
+            parent_id = class_row['parent_id']
+            
+            class_info = {
+                "type": "class",
+                "name": class_row['name'],
+                "location": json.loads(class_row['location']),
+                "base_classes": json.loads(class_row['base_classes']),
+                "docstring": class_row['docstring'],
+                "methods": [],
+                "nested_classes": [],
+                "variables": []
+            }
+            
+            # Add to parent's nested_classes
+            if parent_id in classes_map:
+                classes_map[parent_id]["nested_classes"].append(class_info)
+                classes_map[class_id] = class_info
+                json_index["total_classes"] += 1
+        
+        # Get methods (functions with parent)
+        cursor.execute(
+            """SELECT * FROM functions 
+               WHERE file_id = ? AND parent_id IS NOT NULL 
+               ORDER BY id""",
+            (file_id,)
+        )
+        for func_row in cursor.fetchall():
+            parent_id = func_row['parent_id']
+            parent_type = func_row['parent_type']
+            
+            method_info = {
+                "type": func_row['type'],
+                "name": func_row['name'],
+                "location": json.loads(func_row['location']),
+                "parameters": json.loads(func_row['parameters']),
+                "return_type": func_row['return_type'],
+                "docstring": func_row['docstring']
+            }
+            if func_row['receiver']:
+                method_info['receiver'] = func_row['receiver']
+            
+            # Add to parent class's methods
+            if parent_type == 'class' and parent_id in classes_map:
+                classes_map[parent_id]["methods"].append(method_info)
+                json_index["total_methods"] += 1
+        
+        # Get top-level variables (no parent)
+        cursor.execute(
+            """SELECT * FROM variables 
+               WHERE file_id = ? AND parent_id IS NULL 
+               ORDER BY id""",
+            (file_id,)
+        )
+        for var_row in cursor.fetchall():
+            var_info = {
+                "type": var_row['type'],
+                "name": var_row['name'],
+                "location": json.loads(var_row['location'])
+            }
+            if var_row['field_type']:
+                var_info['field_type'] = var_row['field_type']
+            file_info["variables"].append(var_info)
+            json_index["total_variables"] += 1
+        
+        # Get class attributes (variables with parent)
+        cursor.execute(
+            """SELECT * FROM variables 
+               WHERE file_id = ? AND parent_id IS NOT NULL 
+               ORDER BY id""",
+            (file_id,)
+        )
+        for var_row in cursor.fetchall():
+            parent_id = var_row['parent_id']
+            parent_type = var_row['parent_type']
+            
+            var_info = {
+                "type": var_row['type'],
+                "name": var_row['name'],
+                "location": json.loads(var_row['location'])
+            }
+            if var_row['field_type']:
+                var_info['field_type'] = var_row['field_type']
+            
+            # Add to parent class's variables
+            if parent_type == 'class' and parent_id in classes_map:
+                classes_map[parent_id]["variables"].append(var_info)
+                json_index["total_variables"] += 1
+        
+        # Get type aliases
+        cursor.execute(
+            "SELECT * FROM type_aliases WHERE file_id = ? ORDER BY id",
+            (file_id,)
+        )
+        for type_row in cursor.fetchall():
+            type_info = {
+                "type": "type_alias",
+                "name": type_row['name'],
+                "location": json.loads(type_row['location']),
+                "type_definition": type_row['type_definition']
+            }
+            file_info["type_aliases"].append(type_info)
+            json_index["total_type_defs"] += 1
+        
+        # Get structs
+        cursor.execute(
+            "SELECT * FROM structs WHERE file_id = ? ORDER BY id",
+            (file_id,)
+        )
+        for struct_row in cursor.fetchall():
+            struct_info = {
+                "type": "struct",
+                "name": struct_row['name'],
+                "location": json.loads(struct_row['location']),
+                "methods": [],
+                "fields": []
+            }
+            file_info["structs"].append(struct_info)
+            json_index["total_structs"] += 1
+        
+        # Get interfaces
+        cursor.execute(
+            "SELECT * FROM interfaces WHERE file_id = ? ORDER BY id",
+            (file_id,)
+        )
+        for interface_row in cursor.fetchall():
+            interface_info = {
+                "type": "interface",
+                "name": interface_row['name'],
+                "location": json.loads(interface_row['location']),
+                "methods": []
+            }
+            file_info["interfaces"].append(interface_info)
+            json_index["total_interfaces"] += 1
+        
+        json_index["files"][relative_path] = file_info
+    
+    # Export frontend tables
+    _export_frontend_tables(conn, json_index)
+    
+    conn.close()
+    
+    # Write to JSON file
+    print(f"Writing JSON to {output_file}...")
+    with open(output_file, 'w', encoding='utf-8') as f:
+        json.dump(json_index, f, indent=2)
+    
+    print(f"\nExport complete!")
+    print(f"JSON saved to {output_file}")
+    
+    # Print summary
+    print("\n--- Export Summary ---")
+    print(f"Total files: {len(json_index['files'])}")
+    print(f"Total functions: {json_index['total_functions']}")
+    print(f"Total methods: {json_index['total_methods']}")
+    print(f"Total classes: {json_index['total_classes']}")
+    print(f"Total structs: {json_index['total_structs']}")
+    print(f"Total interfaces: {json_index['total_interfaces']}")
+    print(f"Total variables: {json_index['total_variables']}")
+    print(f"Total type aliases: {json_index['total_type_defs']}")
+    
+    # Frontend summary
+    frontend_keys = ["frontend_components", "markup_elements", "style_selectors",
+                     "frontend_events", "frontend_bindings", "render_relationships",
+                     "frontend_diagnostics"]
+    has_frontend = any(k in json_index and json_index[k] for k in frontend_keys)
+    if has_frontend:
+        print("\n--- Frontend Summary ---")
+        for k in frontend_keys:
+            if k in json_index:
+                print(f"  {k}: {len(json_index[k])}")
+
+
+def _export_frontend_tables(conn, json_index):
+    """Export frontend semantic tables to the JSON index."""
+    cursor = conn.cursor()
+    
+    frontend_tables = [
+        ("frontend_components", "frontend_components"),
+        ("markup_elements", "markup_elements"),
+        ("style_selectors", "style_selectors"),
+        ("style_custom_properties", "style_custom_properties"),
+        ("style_custom_property_usages", "style_custom_property_usages"),
+        ("style_keyframes", "style_keyframes"),
+        ("style_imports", "style_imports"),
+        ("style_selector_matches", "style_selector_matches"),
+        ("frontend_events", "frontend_events"),
+        ("frontend_bindings", "frontend_bindings"),
+        ("render_relationships", "render_relationships"),
+        ("frontend_diagnostics", "frontend_diagnostics"),
+    ]
+    
+    for table_name, key in frontend_tables:
+        try:
+            cursor.execute(f"SELECT * FROM {table_name} ORDER BY id")
+            rows = cursor.fetchall()
+            json_index[key] = []
+            for row in rows:
+                entry = {}
+                for col in row.keys():
+                    val = row[col]
+                    if val is not None and isinstance(val, str) and val.startswith('{'):
+                        try:
+                            val = json.loads(val)
+                        except (json.JSONDecodeError, TypeError):
+                            pass
+                    entry[col] = val
+                json_index[key].append(entry)
+        except sqlite3.OperationalError:
+            pass
+
+
+def main():
+    """Main entry point for CLI usage."""
+    import sys
+    
+    if len(sys.argv) < 2:
+        print("Usage: python export_to_json.py <db_file> [output_json]")
+        print("Example: python export_to_json.py code_index.db code_index.json")
+        sys.exit(1)
+    
+    db_path = sys.argv[1]
+    output_file = sys.argv[2] if len(sys.argv) > 2 else db_path.rsplit('.', 1)[0] + '.json'
+    
+    if not Path(db_path).exists():
+        print(f"Error: Database file not found: {db_path}")
+        sys.exit(1)
+    
+    try:
+        export_to_json(db_path, output_file)
+    except Exception as e:
+        print(f"Error during export: {e}")
+        import traceback
+        traceback.print_exc()
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

indexing/extractors.py

@@ -0,0 +1,189 @@
+"""
+Node extraction functions for different language constructs.
+"""
+
+from indexing.node_utils import (
+    get_node_location,
+    extract_name,
+    extract_node_text,
+    extract_parameters,
+    extract_return_type,
+    is_method,
+    extract_base_classes,
+    extract_variable_name,
+    extract_docstring,
+    extract_field_text,
+    extract_go_receiver,
+    extract_go_type_name,
+    count_branches
+)
+
+
+def extract_function_info(node, source_code, language, class_node_type):
+    """Extract information about a function definition."""
+    name = extract_name(node, source_code, language) or "unknown"
+    params = extract_parameters(node, source_code, language)
+    return_type = extract_return_type(node, source_code)
+    method_flag = is_method(node, language, class_node_type)
+    docstring = extract_docstring(node, source_code, language)
+    
+    # Count branches in function body
+    branch_count = count_branches(node, language, source_code)
+    
+    # Handle Go method receivers
+    receiver = None
+    if language == "go":
+        receiver = extract_go_receiver(node, source_code)
+        if receiver:
+            method_flag = True
+    
+    result = {
+        "type": "method" if method_flag else "function",
+        "name": name,
+        "location": get_node_location(node),
+        "parameters": params,
+        "return_type": return_type,
+        "docstring": docstring,
+        "branch_count": branch_count
+    }
+    
+    if receiver:
+        result["receiver"] = receiver
+    
+    return result
+
+
+def extract_class_info(node, source_code, language):
+    """Extract information about a class definition."""
+    name = extract_name(node, source_code, language) or "unknown"
+    base_classes = extract_base_classes(node, source_code, language)
+    docstring = extract_docstring(node, source_code, language)
+    
+    return {
+        "type": "class",
+        "name": name,
+        "location": get_node_location(node),
+        "base_classes": base_classes,
+        "docstring": docstring,
+        "methods": [],
+        "nested_classes": [],
+        "variables": []
+    }
+
+
+def extract_variable_info(node, source_code, language):
+    """Extract information about a variable assignment."""
+    name, is_attribute = extract_variable_name(node, source_code, language)
+    
+    if not name:
+        return None
+    
+    if is_attribute:
+        return {
+            "type": "attribute",
+            "name": name,
+            "location": get_node_location(node)
+        }
+    
+    return {
+        "type": "variable",
+        "name": name,
+        "location": get_node_location(node)
+    }
+
+
+def extract_type_alias_info(node, source_code, language=None):
+    """Extract information about type alias definitions."""
+    name = extract_name(node, source_code, language) or "unknown"
+    type_def = extract_field_text(node, "type", source_code)
+    
+    return {
+        "type": "type_alias",
+        "name": name,
+        "location": get_node_location(node),
+        "type_definition": type_def
+    }
+
+
+def extract_macro_info(node, source_code, language):
+    """Extract information about a macro definition (C preproc_def, Rust macro_definition)."""
+    name = extract_name(node, source_code, language) or "unknown"
+    
+    # Extract parameters for function-like macros (C preproc_function_def)
+    params = []
+    params_node = node.child_by_field_name("parameters")
+    if params_node is None:
+        # C preproc_function_def uses preproc_params
+        for child in node.children:
+            if child.type == "preproc_params":
+                params_node = child
+                break
+    if params_node:
+        for child in params_node.children:
+            if child.type == "identifier":
+                params.append({"name": extract_node_text(child, source_code), "type": None})
+    
+    return {
+        "type": "macro",
+        "name": name,
+        "location": get_node_location(node),
+        "parameters": params,
+        "return_type": None,
+        "docstring": None,
+        "branch_count": 0,
+    }
+
+
+def extract_struct_info(node, source_code, language):
+    """Extract information about a struct definition (C, C++, etc.)."""
+    name = extract_name(node, source_code, language) or "unknown"
+    return {
+        "type": "struct",
+        "name": name,
+        "location": get_node_location(node),
+    }
+
+
+def extract_go_struct_info(node, source_code):
+    """Extract information about a Go struct definition."""
+    name = extract_go_type_name(node, source_code) or "unknown"
+    
+    return {
+        "type": "struct",
+        "name": name,
+        "location": get_node_location(node),
+        "methods": [],
+        "fields": []
+    }
+
+
+def extract_interface_info(node, source_code, language):
+    """Extract information about an interface/trait definition (Rust, etc.)."""
+    name = extract_name(node, source_code, language) or "unknown"
+    return {
+        "type": "interface",
+        "name": name,
+        "location": get_node_location(node),
+    }
+
+
+def extract_go_interface_info(node, source_code):
+    """Extract information about a Go interface definition."""
+    name = extract_go_type_name(node, source_code) or "unknown"
+    
+    return {
+        "type": "interface",
+        "name": name,
+        "location": get_node_location(node),
+        "methods": []
+    }
+
+
+def extract_enum_info(node, source_code, language):
+    """Extract information about an enum definition (Rust, etc.)."""
+    name = extract_name(node, source_code, language) or "unknown"
+    return {
+        "type": "enum",
+        "name": name,
+        "location": get_node_location(node),
+    }

indexing/file_utils.py

@@ -0,0 +1,97 @@
+"""
+File and directory utilities for the code indexer.
+"""
+
+import os
+from pathlib import Path
+from indexing.language_config import get_language_for_extension, get_extensions_for_languages
+from indexing.frontend_config import load_frontend_config
+from pathspec import PathSpec
+from pathspec.patterns import GitWildMatchPattern
+
+
+def detect_language(file_path):
+    """Detect the language based on file extension."""
+    return get_language_for_extension(file_path.suffix)
+
+
+def load_ignore_patterns(root_dir):
+    """Load ignore patterns from .aiignore, falling back to .gitignore.
+
+    If .aiignore exists in the root directory, its patterns are used to
+    exclude files from indexing. If .aiignore does not exist, .gitignore
+    is used as a fallback.
+    """
+    root_path = Path(root_dir)
+    aiignore_path = root_path / '.aiignore'
+    gitignore_path = root_path / '.gitignore'
+
+    ignore_path = None
+    if aiignore_path.exists():
+        ignore_path = aiignore_path
+    elif gitignore_path.exists():
+        ignore_path = gitignore_path
+
+    if ignore_path is None:
+        return PathSpec.from_lines(GitWildMatchPattern, [])
+
+    with open(ignore_path, 'r', encoding='utf-8') as f:
+        patterns = f.read().splitlines()
+
+    return PathSpec.from_lines(GitWildMatchPattern, patterns)
+
+
+def collect_files_to_index(root_dir, languages, exclude_dirs=None):
+    """Collect all files to index for given languages, respecting .aiignore (or .gitignore).
+
+    Args:
+        root_dir: Root directory to search.
+        languages: List of language names to include.
+        exclude_dirs: Optional list of additional directory names to exclude.
+                      Generated directory exclusions from frontend config are always applied.
+    """
+    root_path = Path(root_dir)
+    extensions = get_extensions_for_languages(languages)
+
+    # Load ignore patterns (.aiignore takes priority over .gitignore)
+    ignore_spec = load_ignore_patterns(root_dir)
+
+    # Load generated directory exclusions from frontend config
+    frontend_cfg = load_frontend_config(root_dir)
+    excluded_dirs = set(frontend_cfg.generated_dir_exclusions)
+    if exclude_dirs:
+        excluded_dirs.update(exclude_dirs)
+
+    files = []
+    ext_set = set(extensions)
+
+    for dirpath, dirnames, filenames in os.walk(root_path):
+        # Prune ignored directories before entering them
+        # Build relative path for the current directory to check against ignore spec
+        current_rel = os.path.relpath(dirpath, root_path)
+        if current_rel != '.':
+            # Check if this directory itself is ignored
+            if ignore_spec.match_file(current_rel + '/') or ignore_spec.match_file(current_rel):
+                dirnames[:] = []
+                continue
+        # Skip test directories and generated directories
+        dirnames[:] = [d for d in dirnames if d not in ('test', 'tests') and d not in excluded_dirs]
+        for filename in filenames:
+            if any(filename.endswith(ext) for ext in ext_set):
+                file_path = Path(dirpath) / filename
+                relative_path = file_path.relative_to(root_path)
+                if not ignore_spec.match_file(str(relative_path)):
+                    files.append(file_path)
+
+    return files
+
+
+def read_file_content(file_path):
+    """Read file content in binary mode for faster processing."""
+    with open(file_path, 'rb') as f:
+        return f.read()
+
+
+def get_relative_path(file_path, root_dir):
+    """Get relative path from root directory."""
+    return str(Path(file_path).relative_to(root_dir))