ctxgraph 0.1.0__py3-none-any.whl

ctxgraph/exclude/patterns.py

@@ -0,0 +1,75 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Optional
+
+
+DEFAULT_EXCLUDE = [
+    "__pycache__",
+    "*.pyc",
+    ".git",
+    ".svn",
+    ".hg",
+    "node_modules",
+    "venv",
+    ".venv",
+    "env",
+    ".env",
+    "dist",
+    "build",
+    "*.egg-info",
+    ".pytest_cache",
+    ".mypy_cache",
+    ".ruff_cache",
+    ".tox",
+    ".nox",
+    "migrations",
+    "*.min.js",
+    "*.min.css",
+]
+
+
+def should_exclude(
+    file_path: Path,
+    root_path: Path,
+    user_patterns: Optional[list[str]] = None,
+) -> bool:
+    patterns = list(DEFAULT_EXCLUDE)
+    if user_patterns:
+        patterns.extend(user_patterns)
+
+    rel_path = _relative_path(file_path, root_path)
+
+    for pattern in patterns:
+        if _matches_pattern(rel_path, pattern):
+            return True
+
+    return False
+
+
+def _matches_pattern(path: str, pattern: str) -> bool:
+    if pattern.startswith("*."):
+        return path.endswith(pattern[1:])
+
+    if pattern.endswith("/"):
+        return pattern.rstrip("/") in path.split("/")
+
+    if "*" not in pattern:
+        return pattern in path.split("/")
+
+    if pattern.startswith("*") and pattern.endswith("*"):
+        mid = pattern[1:-1]
+        return mid in path
+    elif pattern.startswith("*"):
+        return path.endswith(pattern[1:])
+    elif pattern.endswith("*"):
+        return path.startswith(pattern[:-1])
+
+    return pattern in path
+
+
+def _relative_path(file_path: Path, root_path: Path) -> str:
+    try:
+        return str(file_path.relative_to(root_path)).replace("\\", "/")
+    except ValueError:
+        return file_path.name

ctxgraph/graph/builder.py

@@ -0,0 +1,76 @@
+from __future__ import annotations
+
+import time
+from pathlib import Path
+from typing import Optional
+
+from ctxgraph.analyzers.python.importer import analyze_imports
+from ctxgraph.analyzers.python.semantic import enrich_node_summary
+from ctxgraph.analyzers.python.symbols import analyze_symbols
+from ctxgraph.exclude.patterns import should_exclude
+from ctxgraph.graph.models import Graph
+from ctxgraph.graph.storage import Storage
+
+
+def build_graph(
+    repo_path: str | Path,
+    db_path: Optional[str | Path] = None,
+    exclude_patterns: Optional[list[str]] = None,
+) -> dict:
+    repo_path = Path(repo_path).resolve()
+    if db_path is None:
+        db_path = repo_path / ".ctxgraph" / "graph.db"
+
+    db_path = Path(db_path)
+    start = time.time()
+
+    storage = Storage(db_path)
+    storage.connect()
+    combined = Graph()
+    stats = {"files_analyzed": 0, "files_skipped": 0, "errors": 0}
+
+    python_files = list(repo_path.rglob("*.py"))
+    for file_path in python_files:
+        if should_exclude(file_path, repo_path, exclude_patterns):
+            stats["files_skipped"] += 1
+            continue
+
+        try:
+            import_graph = analyze_imports(file_path, repo_path)
+            combined.merge(import_graph)
+
+            symbol_graph = analyze_symbols(file_path, repo_path)
+            combined.merge(symbol_graph)
+
+            for node in combined.nodes.values():
+                if node.path and node.path in str(file_path):
+                    if not node.summary:
+                        summary = enrich_node_summary(node, file_path)
+                        if summary:
+                            node.summary = summary
+
+            stats["files_analyzed"] += 1
+        except Exception:
+            stats["errors"] += 1
+
+    storage.save_graph(combined)
+    storage.save_metadata("build_time", str(time.time()))
+    storage.save_metadata("repo_path", str(repo_path))
+    storage.save_metadata("file_count", str(stats["files_analyzed"]))
+    storage.close()
+
+    elapsed = time.time() - start
+    stats["elapsed_seconds"] = round(elapsed, 2)
+    stats["total_nodes"] = len(combined.nodes)
+    stats["total_edges"] = len(combined.edges)
+
+    return stats
+
+
+def get_storage(repo_path: str | Path) -> Optional[Storage]:
+    db_path = Path(repo_path) / ".ctxgraph" / "graph.db"
+    if not db_path.exists():
+        return None
+    storage = Storage(db_path)
+    storage.connect()
+    return storage

ctxgraph/graph/models.py

@@ -0,0 +1,83 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Optional
+
+
+@dataclass
+class Node:
+    id: str
+    type: str
+    name: str
+    path: Optional[str] = None
+    parent_id: Optional[str] = None
+    summary: Optional[str] = None
+    importance: float = 0.5
+    size_bytes: int = 0
+    lineno: int = 0
+
+    def __hash__(self):
+        return hash(self.id)
+
+    def __eq__(self, other):
+        return isinstance(other, Node) and self.id == other.id
+
+
+@dataclass
+class Edge:
+    source_id: str
+    target_id: str
+    relation: str
+    weight: float = 1.0
+
+    def __hash__(self):
+        return hash((self.source_id, self.target_id, self.relation))
+
+    def __eq__(self, other):
+        return (
+            isinstance(other, Edge)
+            and self.source_id == other.source_id
+            and self.target_id == other.target_id
+            and self.relation == other.relation
+        )
+
+
+@dataclass
+class Graph:
+    nodes: dict[str, Node] = field(default_factory=dict)
+    edges: list[Edge] = field(default_factory=list)
+
+    def add_node(self, node: Node):
+        self.nodes[node.id] = node
+
+    def add_edge(self, edge: Edge):
+        self.edges.append(edge)
+
+    def get_node(self, node_id: str) -> Optional[Node]:
+        return self.nodes.get(node_id)
+
+    def get_edges_from(self, source_id: str) -> list[Edge]:
+        return [e for e in self.edges if e.source_id == source_id]
+
+    def get_edges_to(self, target_id: str) -> list[Edge]:
+        return [e for e in self.edges if e.target_id == target_id]
+
+    def get_neighbors(self, node_id: str) -> list[str]:
+        result = set()
+        for e in self.edges:
+            if e.source_id == node_id:
+                result.add(e.target_id)
+            if e.target_id == node_id:
+                result.add(e.source_id)
+        return list(result)
+
+    def merge(self, other: Graph):
+        for node_id, node in other.nodes.items():
+            if node_id not in self.nodes:
+                self.nodes[node_id] = node
+        existing = {(e.source_id, e.target_id, e.relation) for e in self.edges}
+        for e in other.edges:
+            key = (e.source_id, e.target_id, e.relation)
+            if key not in existing:
+                self.edges.append(e)
+                existing.add(key)

ctxgraph/graph/query.py

@@ -0,0 +1,135 @@
+from __future__ import annotations
+
+import re
+from collections import Counter
+from typing import Optional
+
+from ctxgraph.graph.models import Node
+from ctxgraph.graph.storage import Storage
+
+
+def search_relevant_nodes(
+    storage: Storage,
+    query: str,
+    max_nodes: int = 15,
+    max_depth: int = 2,
+) -> list[tuple[Node, float]]:
+    tokens = _tokenize(query)
+    if not tokens:
+        return []
+
+    scored: dict[str, float] = {}
+    seen_ids: set[str] = set()
+
+    matched_nodes = storage.search_nodes(query)
+    for node in matched_nodes:
+        score = _compute_relevance(node, tokens)
+        if score > 0:
+            scored[node.id] = score
+            seen_ids.add(node.id)
+
+    if not scored:
+        for token in tokens:
+            token_nodes = storage.search_nodes(token)
+            for node in token_nodes:
+                if node.id not in seen_ids:
+                    seen_ids.add(node.id)
+                    score = _compute_relevance(node, tokens)
+                    if score > 0:
+                        scored[node.id] = score
+
+    if not scored:
+        return []
+
+    seed_ids = set(scored.keys())
+    edge_ids = set()
+
+    for _ in range(max_depth):
+        edges = storage.get_edges_for_nodes(seed_ids | edge_ids)
+        new_ids = set()
+        for e in edges:
+            if e.source_id in (seed_ids | edge_ids):
+                new_ids.add(e.target_id)
+            if e.target_id in (seed_ids | edge_ids):
+                new_ids.add(e.source_id)
+        edge_ids |= new_ids
+
+    all_ids = seed_ids | edge_ids
+    for nid in edge_ids:
+        if nid not in scored:
+            node = storage.get_node(nid)
+            if node:
+                neighbors = _count_matched_neighbors(nid, storage, seed_ids)
+                scored[nid] = 0.1 * neighbors
+
+    ranked = sorted(scored.items(), key=lambda x: x[1], reverse=True)
+    ranked = ranked[:max_nodes]
+
+    result = []
+    for nid, score in ranked:
+        node = storage.get_node(nid)
+        if node:
+            result.append((node, round(score, 3)))
+
+    return result
+
+
+def generate_context_subgraph(
+    storage: Storage,
+    query: str,
+    max_nodes: int = 15,
+) -> tuple[list[Node], list[tuple[str, str, str]]]:
+    ranked = search_relevant_nodes(storage, query, max_nodes)
+    nodes = [n for n, _ in ranked]
+    node_ids = {n.id for n in nodes}
+
+    edges = storage.get_edges_for_nodes(node_ids)
+    edge_list = [
+        (e.source_id, e.target_id, e.relation)
+        for e in edges
+        if e.source_id in node_ids and e.target_id in node_ids
+    ]
+
+    return nodes, edge_list
+
+
+def _tokenize(text: str) -> list[str]:
+    text = text.lower()
+    tokens = re.findall(r"[a-zA-Z_][a-zA-Z0-9_]*", text)
+    stopwords = {
+        "the", "a", "an", "in", "on", "at", "to", "for", "of", "is",
+        "fix", "bug", "implement", "add", "change", "update", "remove",
+        "need", "want", "please", "can", "how", "what", "where", "why",
+        "this", "that", "with", "from", "by", "be", "has", "have", "do",
+        "does", "did", "will", "would", "could", "should", "may", "might",
+        "file", "function", "class", "code", "issue", "problem", "error",
+        "work", "make", "get", "set",
+    }
+    return [t for t in tokens if t not in stopwords and len(t) > 1]
+
+
+def _compute_relevance(node: Node, tokens: list[str]) -> float:
+    score = 0.0
+    text = f"{node.name} {node.summary or ''} {node.path or ''}".lower()
+
+    for token in tokens:
+        if token in node.name.lower():
+            score += 2.0
+        count = text.count(token)
+        score += count * 0.5
+
+    if node.importance:
+        score *= (0.5 + node.importance)
+
+    return score
+
+
+def _count_matched_neighbors(
+    node_id: str, storage: Storage, matched_ids: set[str]
+) -> int:
+    edges = storage.get_edges_for_nodes({node_id})
+    count = 0
+    for e in edges:
+        if e.source_id in matched_ids or e.target_id in matched_ids:
+            count += 1
+    return count

ctxgraph/graph/storage.py

@@ -0,0 +1,224 @@
+from __future__ import annotations
+
+import sqlite3
+from pathlib import Path
+from typing import Optional
+
+from ctxgraph.graph.models import Edge, Graph, Node
+
+
+def _table_schema() -> str:
+    return """
+    CREATE TABLE IF NOT EXISTS nodes (
+        id TEXT PRIMARY KEY,
+        type TEXT NOT NULL,
+        name TEXT NOT NULL,
+        path TEXT,
+        parent_id TEXT,
+        summary TEXT,
+        importance REAL DEFAULT 0.5,
+        size_bytes INTEGER DEFAULT 0,
+        lineno INTEGER DEFAULT 0
+    );
+
+    CREATE TABLE IF NOT EXISTS edges (
+        source_id TEXT NOT NULL,
+        target_id TEXT NOT NULL,
+        relation TEXT NOT NULL,
+        weight REAL DEFAULT 1.0,
+        PRIMARY KEY (source_id, target_id, relation)
+    );
+
+    CREATE INDEX IF NOT EXISTS idx_edges_source ON edges(source_id);
+    CREATE INDEX IF NOT EXISTS idx_edges_target ON edges(target_id);
+    CREATE INDEX IF NOT EXISTS idx_nodes_path ON nodes(path);
+    CREATE INDEX IF NOT EXISTS idx_nodes_type ON nodes(type);
+
+    CREATE TABLE IF NOT EXISTS metadata (
+        key TEXT PRIMARY KEY,
+        value TEXT
+    );
+    """
+
+
+class Storage:
+    def __init__(self, db_path: str | Path):
+        self.db_path = Path(db_path)
+        self.db_path.parent.mkdir(parents=True, exist_ok=True)
+        self._conn: Optional[sqlite3.Connection] = None
+
+    def connect(self):
+        self._conn = sqlite3.connect(str(self.db_path))
+        self._conn.execute("PRAGMA journal_mode=WAL")
+        self._conn.execute("PRAGMA synchronous=NORMAL")
+        self._conn.row_factory = sqlite3.Row
+        self._init_schema()
+
+    def _init_schema(self):
+        self._conn.executescript(_table_schema())
+        self._conn.commit()
+
+    def close(self):
+        if self._conn:
+            self._conn.close()
+            self._conn = None
+
+    @property
+    def conn(self) -> sqlite3.Connection:
+        if self._conn is None:
+            raise RuntimeError("Storage not connected. Call connect() first.")
+        return self._conn
+
+    def update_node_summary(self, node_id: str, summary: str):
+        self.conn.execute(
+            "UPDATE nodes SET summary = ? WHERE id = ?", (summary, node_id)
+        )
+        self.conn.commit()
+
+    def save_node(self, node: Node):
+        self.conn.execute(
+            """INSERT OR REPLACE INTO nodes
+               (id, type, name, path, parent_id, summary, importance, size_bytes, lineno)
+               VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)""",
+            (
+                node.id,
+                node.type,
+                node.name,
+                node.path,
+                node.parent_id,
+                node.summary,
+                node.importance,
+                node.size_bytes,
+                node.lineno,
+            ),
+        )
+
+    def save_edge(self, edge: Edge):
+        self.conn.execute(
+            """INSERT OR REPLACE INTO edges
+               (source_id, target_id, relation, weight)
+               VALUES (?, ?, ?, ?)""",
+            (edge.source_id, edge.target_id, edge.relation, edge.weight),
+        )
+
+    def save_graph(self, graph: Graph):
+        for node in graph.nodes.values():
+            self.save_node(node)
+        for edge in graph.edges:
+            self.save_edge(edge)
+        self.conn.commit()
+
+    def get_node(self, node_id: str) -> Optional[Node]:
+        row = self.conn.execute(
+            "SELECT * FROM nodes WHERE id = ?", (node_id,)
+        ).fetchone()
+        if row is None:
+            return None
+        return Node(
+            id=row["id"],
+            type=row["type"],
+            name=row["name"],
+            path=row["path"],
+            parent_id=row["parent_id"],
+            summary=row["summary"],
+            importance=row["importance"],
+            size_bytes=row["size_bytes"],
+            lineno=row["lineno"],
+        )
+
+    def search_nodes(self, text: str) -> list[Node]:
+        query = f"%{text}%"
+        rows = self.conn.execute(
+            """SELECT * FROM nodes WHERE
+               name LIKE ? OR summary LIKE ? OR path LIKE ?
+               ORDER BY importance DESC
+               LIMIT 50""",
+            (query, query, query),
+        ).fetchall()
+        return [
+            Node(
+                id=r["id"],
+                type=r["type"],
+                name=r["name"],
+                path=r["path"],
+                parent_id=r["parent_id"],
+                summary=r["summary"],
+                importance=r["importance"],
+                size_bytes=r["size_bytes"],
+                lineno=r["lineno"],
+            )
+            for r in rows
+        ]
+
+    def get_edges_for_nodes(self, node_ids: set[str]) -> list[Edge]:
+        if not node_ids:
+            return []
+        placeholders = ",".join("?" for _ in node_ids)
+        rows = self.conn.execute(
+            f"""SELECT * FROM edges WHERE
+               source_id IN ({placeholders}) OR target_id IN ({placeholders})""",
+            list(node_ids) + list(node_ids),
+        ).fetchall()
+        return [
+            Edge(
+                source_id=r["source_id"],
+                target_id=r["target_id"],
+                relation=r["relation"],
+                weight=r["weight"],
+            )
+            for r in rows
+        ]
+
+    def get_all_nodes(self) -> list[Node]:
+        rows = self.conn.execute("SELECT * FROM nodes").fetchall()
+        return [
+            Node(
+                id=r["id"],
+                type=r["type"],
+                name=r["name"],
+                path=r["path"],
+                parent_id=r["parent_id"],
+                summary=r["summary"],
+                importance=r["importance"],
+                size_bytes=r["size_bytes"],
+                lineno=r["lineno"],
+            )
+            for r in rows
+        ]
+
+    def get_all_edges(self) -> list[Edge]:
+        rows = self.conn.execute("SELECT * FROM edges").fetchall()
+        return [
+            Edge(
+                source_id=r["source_id"],
+                target_id=r["target_id"],
+                relation=r["relation"],
+                weight=r["weight"],
+            )
+            for r in rows
+        ]
+
+    def stats(self) -> dict:
+        node_count = self.conn.execute("SELECT COUNT(*) FROM nodes").fetchone()[0]
+        edge_count = self.conn.execute("SELECT COUNT(*) FROM edges").fetchone()[0]
+        type_counts = self.conn.execute(
+            "SELECT type, COUNT(*) as cnt FROM nodes GROUP BY type"
+        ).fetchall()
+        return {
+            "nodes": node_count,
+            "edges": edge_count,
+            "types": {r["type"]: r["cnt"] for r in type_counts},
+        }
+
+    def save_metadata(self, key: str, value: str):
+        self.conn.execute(
+            "INSERT OR REPLACE INTO metadata (key, value) VALUES (?, ?)",
+            (key, value),
+        )
+        self.conn.commit()
+
+    def get_metadata(self, key: str) -> Optional[str]:
+        row = self.conn.execute(
+            "SELECT value FROM metadata WHERE key = ?", (key,)
+        ).fetchone()
+        return row["value"] if row else None