codegraphy 0.1.0__py3-none-any.whl

repolens/db/store.py

@@ -0,0 +1,162 @@
+import json
+import sqlite3
+from urllib.parse import urlparse
+from contextlib import contextmanager
+
+from .schema import get_schema
+
+try:
+    import psycopg2
+    from psycopg2.extras import Json
+except ImportError:
+    psycopg2 = None
+
+class Store:
+    def __init__(self, db_url: str):
+        self.db_url = db_url
+        self.is_postgres = db_url.startswith("postgres")
+        
+        if self.is_postgres and psycopg2 is None:
+            raise ImportError("psycopg2-binary is required for postgres support")
+            
+        if self.is_postgres:
+            parsed = urlparse(db_url)
+            self.conn_kwargs = {
+                'dbname': parsed.path[1:],
+                'user': parsed.username,
+                'password': parsed.password,
+                'host': parsed.hostname,
+                'port': parsed.port,
+            }
+            # Remove None values
+            self.conn_kwargs = {k: v for k, v in self.conn_kwargs.items() if v is not None}
+        else:
+            # Handle sqlite:///path
+            self.db_path = db_url.replace("sqlite:///", "")
+            if not self.db_path:
+                self.db_path = "codegraphy.db"
+
+    @contextmanager
+    def get_connection(self):
+        if self.is_postgres:
+            conn = psycopg2.connect(**self.conn_kwargs)
+        else:
+            conn = sqlite3.connect(self.db_path)
+            conn.execute("PRAGMA foreign_keys = ON")
+            
+        try:
+            yield conn
+            conn.commit()
+        except Exception:
+            conn.rollback()
+            raise
+        finally:
+            conn.close()
+
+    def init_schema(self):
+        schema_sql = get_schema(self.db_url)
+        with self.get_connection() as conn:
+            cursor = conn.cursor()
+            if self.is_postgres:
+                cursor.execute(schema_sql)
+            else:
+                # sqlite3 executescript for multiple statements
+                cursor.executescript(schema_sql)
+
+    def get_file_hash(self, file_path: str) -> str:
+        with self.get_connection() as conn:
+            cursor = conn.cursor()
+            if self.is_postgres:
+                cursor.execute("SELECT git_hash FROM cg_files WHERE file_path = %s", (file_path,))
+            else:
+                cursor.execute("SELECT git_hash FROM cg_files WHERE file_path = ?", (file_path,))
+            row = cursor.fetchone()
+            return row[0] if row else None
+
+    def upsert_file(self, file_path: str, git_hash: str, symbols: list, edges: list):
+        """
+        Replace symbols and edges for a file.
+        """
+        with self.get_connection() as conn:
+            cursor = conn.cursor()
+            
+            # Param style
+            p = "%s" if self.is_postgres else "?"
+            
+            # Upsert file
+            if self.is_postgres:
+                cursor.execute(f"""
+                    INSERT INTO cg_files (file_path, git_hash, symbol_count, last_indexed)
+                    VALUES ({p}, {p}, {p}, NOW())
+                    ON CONFLICT (file_path) DO UPDATE 
+                    SET git_hash = EXCLUDED.git_hash, symbol_count = EXCLUDED.symbol_count, last_indexed = NOW()
+                """, (file_path, git_hash, len(symbols)))
+            else:
+                cursor.execute(f"""
+                    INSERT INTO cg_files (file_path, git_hash, symbol_count, last_indexed)
+                    VALUES ({p}, {p}, {p}, CURRENT_TIMESTAMP)
+                    ON CONFLICT(file_path) DO UPDATE 
+                    SET git_hash=excluded.git_hash, symbol_count=excluded.symbol_count, last_indexed=CURRENT_TIMESTAMP
+                """, (file_path, git_hash, len(symbols)))
+
+            # Delete old symbols (cascade deletes edges)
+            cursor.execute(f"DELETE FROM cg_symbols WHERE file_path = {p}", (file_path,))
+
+            # Insert new symbols
+            if symbols:
+                symbol_records = []
+                for s in symbols:
+                    extra_val = Json(s.extra) if self.is_postgres else json.dumps(s.extra)
+                    symbol_records.append((
+                        s.name, s.qualified_name, s.kind, s.file_path,
+                        s.line_start, s.line_end, s.summary, s.raw_signature, extra_val
+                    ))
+                
+                cursor.executemany(f"""
+                    INSERT INTO cg_symbols (name, qualified_name, kind, file_path, line_start, line_end, summary, raw_signature, extra)
+                    VALUES ({p}, {p}, {p}, {p}, {p}, {p}, {p}, {p}, {p})
+                """, symbol_records)
+
+            # To insert edges, we need their IDs. The simplest way is to map qualified_name -> id
+            # Note: For edges where the target doesn't exist yet, we might have missing IDs.
+            # To handle this robustly without failing, we only insert edges where both from and to exist.
+            # However, the spec says from_id, to_id.
+            # We must get IDs for all symbols first.
+            
+            # For simplicity in this step, we will bulk insert edges later or inside a second pass?
+            # Actually we can just look up ids.
+            # If to_qualified doesn't exist in DB, the edge is dropped.
+            if edges:
+                quals = set()
+                for e in edges:
+                    quals.add(e.from_qualified)
+                    quals.add(e.to_qualified)
+                
+                if quals:
+                    # SQLite limit for variables is 999, but a single file rarely exceeds that.
+                    # For safety, we can query in batches, or use placeholders.
+                    quals_list = list(quals)
+                    qual_to_id = {}
+                    
+                    # Batch fetch to avoid hitting sqlite limits
+                    batch_size = 500
+                    for i in range(0, len(quals_list), batch_size):
+                        batch = quals_list[i:i+batch_size]
+                        placeholders = ",".join([p] * len(batch))
+                        cursor.execute(f"SELECT id, qualified_name FROM cg_symbols WHERE qualified_name IN ({placeholders})", tuple(batch))
+                        for row in cursor.fetchall():
+                            qual_to_id[row[1]] = row[0]
+                    
+                    edge_records = []
+                    for e in edges:
+                        from_id = qual_to_id.get(e.from_qualified)
+                        to_id = qual_to_id.get(e.to_qualified)
+                        if from_id and to_id:
+                            edge_records.append((from_id, to_id, e.relation))
+                    
+                    if edge_records:
+                        cursor.executemany(f"""
+                            INSERT INTO cg_edges (from_id, to_id, relation)
+                            VALUES ({p}, {p}, {p})
+                            ON CONFLICT DO NOTHING
+                        """, edge_records)

repolens/indexer/__init__.py

@@ -0,0 +1,5 @@
+from .base import BaseIndexer, Symbol, Edge
+from .python import PythonIndexer
+from .walker import index_path
+
+__all__ = ["BaseIndexer", "Symbol", "Edge", "PythonIndexer", "index_path"]

repolens/indexer/base.py

@@ -0,0 +1,27 @@
+from dataclasses import dataclass, field
+
+@dataclass
+class Symbol:
+    name: str
+    qualified_name: str
+    kind: str
+    file_path: str
+    line_start: int
+    line_end: int
+    summary: str = ""
+    raw_signature: str = ""
+    extra: dict = field(default_factory=dict)
+
+@dataclass
+class Edge:
+    from_qualified: str
+    to_qualified: str
+    relation: str
+
+class BaseIndexer:
+    def can_handle(self, file_path: str) -> bool:
+        raise NotImplementedError
+
+    def index_file(self, file_path: str, source: str) -> tuple[list[Symbol], list[Edge]]:
+        """Returns symbols and edges found in the file."""
+        raise NotImplementedError

repolens/indexer/python.py

@@ -0,0 +1,177 @@
+import ast
+import os
+from .base import BaseIndexer, Symbol, Edge
+
+def _get_module_path(file_path: str) -> str:
+    # Basic conversion from file path to dotted module path
+    # e.g., src/foo/bar.py -> src.foo.bar
+    # This is a simple approximation.
+    base = os.path.splitext(file_path)[0]
+    return base.replace(os.sep, '.')
+
+class PythonIndexer(BaseIndexer):
+    def can_handle(self, file_path: str) -> bool:
+        return file_path.endswith('.py')
+
+    def index_file(self, file_path: str, source: str) -> tuple[list[Symbol], list[Edge]]:
+        try:
+            tree = ast.parse(source, filename=file_path)
+        except SyntaxError:
+            return [], []
+
+        module_path = _get_module_path(file_path)
+        
+        symbols = []
+        edges = []
+        
+        # We need a visitor to traverse the AST.
+        class Visitor(ast.NodeVisitor):
+            def __init__(self):
+                self.current_scope = []
+                self.current_class = None
+
+            def _get_dotted_name(self, node):
+                if isinstance(node, ast.Name):
+                    return node.id
+                elif isinstance(node, ast.Attribute):
+                    val = self._get_dotted_name(node.value)
+                    if val:
+                        return f"{val}.{node.attr}"
+                return None
+
+            def get_qualname(self, name):
+                if not self.current_scope:
+                    return f"{module_path}.{name}" if module_path else name
+                return f"{module_path}.{'.'.join(self.current_scope)}.{name}"
+
+            def visit_ClassDef(self, node):
+                qualname = self.get_qualname(node.name)
+                summary = ast.get_docstring(node) or ""
+                if summary:
+                    summary = summary.splitlines()[0]
+                
+                symbols.append(Symbol(
+                    name=node.name,
+                    qualified_name=qualname,
+                    kind="class",
+                    file_path=file_path,
+                    line_start=node.lineno,
+                    line_end=node.end_lineno,
+                    summary=summary
+                ))
+                
+                # Inherits edges
+                for base in node.bases:
+                    target = self._get_dotted_name(base)
+                    if target:
+                        edges.append(Edge(
+                            from_qualified=qualname,
+                            to_qualified=target, # Approximated base name
+                            relation="inherits"
+                        ))
+
+                self.current_scope.append(node.name)
+                prev_class = self.current_class
+                self.current_class = node.name
+                self.generic_visit(node)
+                self.current_class = prev_class
+                self.current_scope.pop()
+
+            def visit_FunctionDef(self, node):
+                self._visit_func(node)
+            def visit_AsyncFunctionDef(self, node):
+                self._visit_func(node)
+
+            def _visit_func(self, node):
+                qualname = self.get_qualname(node.name)
+                summary = ast.get_docstring(node) or ""
+                if summary:
+                    summary = summary.splitlines()[0]
+                
+                kind = "method" if self.current_class else "function"
+                
+                symbols.append(Symbol(
+                    name=node.name,
+                    qualified_name=qualname,
+                    kind=kind,
+                    file_path=file_path,
+                    line_start=node.lineno,
+                    line_end=node.end_lineno,
+                    summary=summary
+                ))
+                
+                self.current_scope.append(node.name)
+                self.generic_visit(node)
+                self.current_scope.pop()
+
+            def visit_Import(self, node):
+                for alias in node.names:
+                    # module level import
+                    # e.g., import os
+                    qualname = self.get_qualname(alias.asname or alias.name)
+                    symbols.append(Symbol(
+                        name=alias.asname or alias.name,
+                        qualified_name=qualname,
+                        kind="import",
+                        file_path=file_path,
+                        line_start=node.lineno,
+                        line_end=node.end_lineno,
+                        extra={"module": alias.name}
+                    ))
+                    
+                    edges.append(Edge(
+                        from_qualified=module_path,
+                        to_qualified=alias.name,
+                        relation="imports"
+                    ))
+
+            def visit_ImportFrom(self, node):
+                if node.module:
+                    for alias in node.names:
+                        name = alias.asname or alias.name
+                        qualname = self.get_qualname(name)
+                        symbols.append(Symbol(
+                            name=name,
+                            qualified_name=qualname,
+                            kind="import",
+                            file_path=file_path,
+                            line_start=node.lineno,
+                            line_end=node.end_lineno,
+                            extra={"module": node.module, "original_name": alias.name}
+                        ))
+                        
+                        edges.append(Edge(
+                            from_qualified=module_path,
+                            to_qualified=f"{node.module}.{alias.name}",
+                            relation="imports"
+                        ))
+
+            def visit_Call(self, node):
+                # Extract simple calls
+                target = self._get_dotted_name(node.func)
+                    
+                if target and self.current_scope:
+                    caller = self.get_qualname("")[:-1] # strip trailing dot
+                    edges.append(Edge(
+                        from_qualified=caller,
+                        to_qualified=target, # We don't have full res, will be resolved or approximated during usages query
+                        relation="calls"
+                    ))
+                
+                self.generic_visit(node)
+
+        visitor = Visitor()
+        visitor.visit(tree)
+        
+        # Add a symbol for the file itself
+        symbols.append(Symbol(
+            name=os.path.basename(file_path),
+            qualified_name=module_path,
+            kind="file",
+            file_path=file_path,
+            line_start=1,
+            line_end=len(source.splitlines()),
+            summary=""
+        ))
+        
+        return symbols, edges

repolens/indexer/walker.py

@@ -0,0 +1,77 @@
+import os
+import hashlib
+import subprocess
+from .python import PythonIndexer
+from ..db.store import Store
+
+INDEXERS = [PythonIndexer()]
+
+def sha256(content: bytes) -> str:
+    return hashlib.sha256(content).hexdigest()
+
+def get_files_to_index(root: str, exclude: list[str]) -> list[str]:
+    # Use git ls-files if possible
+    try:
+        result = subprocess.run(
+            ['git', 'ls-files'],
+            cwd=root,
+            capture_output=True,
+            text=True,
+            check=True
+        )
+        files = result.stdout.splitlines()
+        # Make paths absolute
+        files = [os.path.join(root, f) for f in files]
+    except (subprocess.CalledProcessError, FileNotFoundError):
+        # Fallback to os.walk
+        files = []
+        for dirpath, dirnames, filenames in os.walk(root):
+            # rudimentary exclude
+            dirnames[:] = [d for d in dirnames if d not in exclude and not d.startswith('.')]
+            for f in filenames:
+                files.append(os.path.join(dirpath, f))
+    
+    # Filter excludes
+    if exclude:
+        filtered = []
+        for f in files:
+            if not any(ex in f for ex in exclude):
+                filtered.append(f)
+        files = filtered
+        
+    return files
+
+def index_path(root: str, store: Store, plugins: list, exclude: list[str] = None):
+    exclude = exclude or ['.git', 'node_modules', '__pycache__', '.venv']
+    files = get_files_to_index(root, exclude)
+    
+    indexed_count = 0
+    for path in files:
+        indexer = next((i for i in INDEXERS if i.can_handle(path)), None)
+        if not indexer:
+            continue
+            
+        try:
+            with open(path, 'rb') as f:
+                content_bytes = f.read()
+        except OSError:
+            continue
+            
+        file_hash = sha256(content_bytes)
+        
+        # Check if unchanged
+        if store.get_file_hash(path) == file_hash:
+            continue
+            
+        content_str = content_bytes.decode('utf-8', errors='replace')
+        symbols, edges = indexer.index_file(path, content_str)
+        
+        # Apply plugins
+        for plugin in plugins:
+            symbols = [plugin.on_symbol(s) for s in symbols]
+            edges.extend(plugin.extra_edges(symbols))
+            
+        store.upsert_file(path, file_hash, symbols, edges)
+        indexed_count += 1
+        
+    return indexed_count

repolens/mcp/__init__.py

@@ -0,0 +1,3 @@
+from .server import start_server
+
+__all__ = ["start_server"]