nelgraph 1.0.0__tar.gz

nelgraph-1.0.0/.gitignore

@@ -0,0 +1,21 @@
+.venv/
+chroma_db/
+data/
+.graphrag_data/
+server_data/
+workspace/
+__pycache__/
+*.pyc
+.env
+node_modules/
+dist/
+build/
+*.log
+.DS_Store
+/updates/
+ngrok.exe
+ngrok_tmp.exe
+
+# MCP compiled JS outputs
+mcp/server.js
+mcp/tools.js

nelgraph-1.0.0/PKG-INFO

@@ -0,0 +1,94 @@
+Metadata-Version: 2.4
+Name: nelgraph
+Version: 1.0.0
+Summary: GraphRAG knowledge base for codebases
+License: MIT
+Requires-Python: >=3.10
+Requires-Dist: chromadb
+Requires-Dist: click
+Requires-Dist: fastapi
+Requires-Dist: gitpython
+Requires-Dist: igraph
+Requires-Dist: json-repair
+Requires-Dist: leidenalg
+Requires-Dist: neo4j
+Requires-Dist: networkx
+Requires-Dist: openai
+Requires-Dist: python-dotenv
+Requires-Dist: requests
+Requires-Dist: rich
+Requires-Dist: tree-sitter
+Requires-Dist: tree-sitter-javascript
+Requires-Dist: tree-sitter-php
+Requires-Dist: tree-sitter-python
+Requires-Dist: tree-sitter-typescript
+Requires-Dist: uvicorn
+Requires-Dist: watchdog
+Description-Content-Type: text/markdown
+
+# nelgraph 🚀
+
+An autonomous, zero-configuration Knowledge Graph builder and semantic search engine optimized for local codebases and AI testing agents. It ingests source code, AST Call Graphs, and Git history into a hybrid Graph-Vector database (**Neo4j** + **ChromaDB**) using **DeepSeek V4-Flash**.
+
+## 🛠️ Installation
+
+```bash
+pip install nelgraph
+```
+
+## 🚀 CLI Usage
+
+### 1. Initialize GraphRAG for your project
+Navigate to your project directory and run:
+
+```bash
+nelgraph init
+```
+
+During initialization, it will prompt you for your OpenRouter API key, configure the local `.env` and `.graphrag_data/` directory, start Neo4j inside a local Docker container, build structural nodes and indexes, and automatically install a git post-commit hook so the graph auto-syncs.
+
+### 2. Manual Synchronization
+If you want to manually trigger incremental synchronization:
+
+```bash
+nelgraph sync
+```
+
+### 3. Check Status
+View current database metrics, indexed function counts, and enrichment coverage:
+
+```bash
+nelgraph status
+```
+
+### 4. Run Watcher
+Run a file watcher that auto-syncs the graph in the background when files change:
+
+```bash
+nelgraph watch
+```
+
+## 🔌 Programmatic Python API
+
+You can import `nelgraph` directly into your scripts or AI testing agents:
+
+```python
+import nelgraph
+
+# Configure (if .env is not present or needs custom config)
+nelgraph.configure(
+    codebase_path="/absolute/path/to/project",
+    openrouter_api_key="your-openrouter-key"
+)
+
+# 1. Get snapshot of prioritized functions
+snapshot = nelgraph.get_snapshot()
+print(f"Total functions: {snapshot['total']}")
+
+# 2. Retrieve detailed context for a function
+ctx = nelgraph.get_function_context("process_order")
+print(ctx.get("function", {}).get("raw_code"))
+
+# 3. Mark function as tested/verified
+nelgraph.mark_tested("process_order")
+```

nelgraph-1.0.0/README.md

@@ -0,0 +1,66 @@
+# nelgraph 🚀
+
+An autonomous, zero-configuration Knowledge Graph builder and semantic search engine optimized for local codebases and AI testing agents. It ingests source code, AST Call Graphs, and Git history into a hybrid Graph-Vector database (**Neo4j** + **ChromaDB**) using **DeepSeek V4-Flash**.
+
+## 🛠️ Installation
+
+```bash
+pip install nelgraph
+```
+
+## 🚀 CLI Usage
+
+### 1. Initialize GraphRAG for your project
+Navigate to your project directory and run:
+
+```bash
+nelgraph init
+```
+
+During initialization, it will prompt you for your OpenRouter API key, configure the local `.env` and `.graphrag_data/` directory, start Neo4j inside a local Docker container, build structural nodes and indexes, and automatically install a git post-commit hook so the graph auto-syncs.
+
+### 2. Manual Synchronization
+If you want to manually trigger incremental synchronization:
+
+```bash
+nelgraph sync
+```
+
+### 3. Check Status
+View current database metrics, indexed function counts, and enrichment coverage:
+
+```bash
+nelgraph status
+```
+
+### 4. Run Watcher
+Run a file watcher that auto-syncs the graph in the background when files change:
+
+```bash
+nelgraph watch
+```
+
+## 🔌 Programmatic Python API
+
+You can import `nelgraph` directly into your scripts or AI testing agents:
+
+```python
+import nelgraph
+
+# Configure (if .env is not present or needs custom config)
+nelgraph.configure(
+    codebase_path="/absolute/path/to/project",
+    openrouter_api_key="your-openrouter-key"
+)
+
+# 1. Get snapshot of prioritized functions
+snapshot = nelgraph.get_snapshot()
+print(f"Total functions: {snapshot['total']}")
+
+# 2. Retrieve detailed context for a function
+ctx = nelgraph.get_function_context("process_order")
+print(ctx.get("function", {}).get("raw_code"))
+
+# 3. Mark function as tested/verified
+nelgraph.mark_tested("process_order")
+```

nelgraph-1.0.0/nelgraph/__init__.py

@@ -0,0 +1,185 @@
+"""
+GraphRAG Knowledge Base — Internal Python Module
+
+Cách dùng nhanh nhất:
+
+    import graphrag
+    graphrag.configure(codebase_path="/path/to/project", openrouter_api_key="sk-...")
+    graphrag.run_init()
+
+    ctx = graphrag.get_function_context("processOrder")
+    snap = graphrag.get_snapshot()
+    changes = graphrag.get_changes("abc123f")
+    graphrag.mark_tested("processOrder")
+"""
+
+# --- Public API ---
+from nelgraph.knowledge_base import (
+    get_function_context,
+    get_snapshot,
+    get_changes,
+    mark_tested,
+    search,
+    run_init,
+    run_sync,
+)
+
+__version__ = "1.0.0"
+
+__all__ = [
+    "configure",
+    "get_function_context",
+    "get_snapshot",
+    "get_changes",
+    "mark_tested",
+    "search",
+    "run_init",
+    "run_sync",
+]
+
+
+def configure(
+    codebase_path: str = None,
+    openrouter_api_key: str = None,
+    neo4j_uri: str = None,
+    neo4j_password: str = None,
+    neo4j_user: str = None,
+    llm_model: str = None,
+    embedding_model: str = None,
+    embedding_dimensions: int = None,
+):
+    """
+    Cấu hình graphrag bằng code thay vì .env file.
+    Gọi hàm này TRƯỚC khi dùng bất kỳ function nào khác.
+
+    Args:
+        codebase_path:        Đường dẫn tuyệt đối đến codebase cần analyze.
+        openrouter_api_key:   API key của OpenRouter.
+        neo4j_uri:            URI kết nối Neo4j (default: bolt://127.0.0.1:7687).
+        neo4j_password:       Password Neo4j.
+        neo4j_user:           Username Neo4j (default: neo4j).
+        llm_model:            Model ID trên OpenRouter cho LLM enrichment.
+        embedding_model:      Model ID trên OpenRouter cho embeddings.
+        embedding_dimensions: Số chiều vector (default: 512).
+
+    Ví dụ:
+        graphrag.configure(
+            codebase_path="/home/user/opensourcepos",
+            openrouter_api_key="sk-or-...",
+        )
+    """
+    import os
+    import nelgraph.config as _cfg
+
+    if codebase_path:
+        import os as _os
+        codebase_path = _os.path.abspath(codebase_path).replace("\\", "/")
+        _cfg.CODEBASE_PATH = codebase_path
+        os.environ["CODEBASE_PATH"] = codebase_path
+
+        # Recalculate dependent paths
+        _cfg.GRAPHRAG_DATA_DIR = _os.path.join(codebase_path, ".graphrag_data").replace("\\", "/")
+        _cfg.NEO4J_DATA_DIR = _os.path.join(_cfg.GRAPHRAG_DATA_DIR, "neo4j", "data").replace("\\", "/")
+        _cfg.NEO4J_LOGS_DIR = _os.path.join(_cfg.GRAPHRAG_DATA_DIR, "neo4j", "logs").replace("\\", "/")
+        _cfg.CHROMA_PATH = _os.path.join(_cfg.GRAPHRAG_DATA_DIR, "chromadb").replace("\\", "/")
+        _cfg.SYNC_STATE_PATH = _os.path.join(_cfg.GRAPHRAG_DATA_DIR, "sync_state.json").replace("\\", "/")
+
+    if openrouter_api_key:
+        _cfg.OPENROUTER_API_KEY = openrouter_api_key
+        os.environ["OPENROUTER_API_KEY"] = openrouter_api_key
+        # Reset lazy clients so they pick up the new key
+        _reset_ai_clients()
+
+    if neo4j_uri:
+        _cfg.NEO4J_URI = neo4j_uri
+        os.environ["NEO4J_URI"] = neo4j_uri
+
+    if neo4j_password:
+        _cfg.NEO4J_PASSWORD = neo4j_password
+        os.environ["NEO4J_PASSWORD"] = neo4j_password
+        # Reset Neo4j singleton
+        import nelgraph.graph.neo4j_client as _nc
+        _nc._client = None
+
+    if neo4j_user:
+        _cfg.NEO4J_USER = neo4j_user
+
+    if llm_model:
+        _cfg.LLM_MODEL = llm_model
+
+    if embedding_model:
+        _cfg.EMBEDDING_MODEL = embedding_model
+
+    if embedding_dimensions:
+        _cfg.EMBEDDING_DIMENSIONS = embedding_dimensions
+
+
+def _reset_ai_clients():
+    """Reset tất cả lazy OpenAI client singletons để pick up config mới."""
+    try:
+        import nelgraph.embeddings.embedder as _emb
+        _emb._openai_client = None
+    except Exception:
+        pass
+    try:
+        import nelgraph.extractors.testing_enricher as _te
+        _te._client_ai = None
+    except Exception:
+        pass
+    try:
+        import nelgraph.extractors.llm_extractor as _le
+        _le._client = None
+    except Exception:
+        pass
+    try:
+        import nelgraph.community.summarizer as _sm
+        _sm._client_ai = None
+    except Exception:
+        pass
+
+
+def status() -> dict:
+    """
+    Trả về trạng thái hiện tại của graph.
+    Không cần Neo4j đang chạy — nếu không kết nối được thì báo offline.
+
+    Returns:
+        {
+            "neo4j": "connected" | "offline",
+            "codebase_path": "...",
+            "last_sync": "...",
+            "total_functions": 0,
+            "enriched_functions": 0,
+        }
+    """
+    import nelgraph.config as _cfg
+    from nelgraph.initialize_graph import _load_sync_state
+
+    result = {
+        "neo4j": "offline",
+        "codebase_path": _cfg.CODEBASE_PATH,
+        "last_sync": None,
+        "total_functions": 0,
+        "enriched_functions": 0,
+    }
+
+    sync_state = _load_sync_state()
+    if sync_state:
+        result["last_sync"] = sync_state.get("last_sync_time")
+
+    try:
+        from nelgraph.graph.neo4j_client import get_client
+        client = get_client()
+        stats = client.run("""
+            OPTIONAL MATCH (f:Function) WITH count(f) as total
+            OPTIONAL MATCH (f2:Function) WHERE f2.how_it_works IS NOT NULL
+            RETURN total, count(f2) as enriched
+        """)
+        if stats:
+            result["neo4j"] = "connected"
+            result["total_functions"] = stats[0]["total"]
+            result["enriched_functions"] = stats[0]["enriched"]
+    except Exception:
+        pass
+
+    return result

nelgraph-1.0.0/nelgraph/cli.py

@@ -0,0 +1,144 @@
+import click
+from rich.console import Console
+import os
+import sys
+import requests
+import nelgraph
+
+if sys.platform.startswith("win"):
+    try:
+        sys.stdout.reconfigure(encoding="utf-8")
+        sys.stderr.reconfigure(encoding="utf-8")
+    except Exception:
+        pass
+
+console = Console()
+
+def _check_for_updates():
+    try:
+        # Check PyPI version dynamically (timeout after 2s to prevent CLI blocking)
+        res = requests.get("https://pypi.org/pypi/nelgraph/json", timeout=2)
+        latest = res.json()["info"]["version"]
+        from nelgraph import __version__
+        if latest != __version__:
+            console.print(
+                f"[yellow]Update available: {__version__} → {latest}[/yellow]\n"
+                f"Run: [bold]pip install --upgrade nelgraph[/bold]"
+            )
+    except Exception:
+        pass
+
+@click.group()
+def main():
+    """nelgraph — Codebase Knowledge Graph & Semantic Search CLI"""
+    _check_for_updates()
+
+def _install_git_hook(codebase_path: str):
+    git_dir = os.path.join(codebase_path, ".git")
+    if not os.path.exists(git_dir):
+        return
+    hooks_dir = os.path.join(git_dir, "hooks")
+    os.makedirs(hooks_dir, exist_ok=True)
+    hook_path = os.path.join(hooks_dir, "post-commit")
+    
+    # Simple shell hook to run nelgraph sync in background silently
+    hook_content = """#!/bin/sh
+# Auto-sync graph in background after commit
+nelgraph sync --silent &
+"""
+    try:
+        with open(hook_path, "w", newline="\n", encoding="utf-8") as f:
+            f.write(hook_content)
+        # Make hook executable
+        import stat
+        st = os.stat(hook_path)
+        os.chmod(hook_path, st.st_mode | stat.S_IEXEC)
+        console.print("[green]✓ Git post-commit hook installed successfully.[/green]")
+    except Exception as e:
+        console.print(f"[yellow]⚠ Warning: Could not install Git post-commit hook: {e}[/yellow]")
+
+@main.command()
+@click.option("--key", help="OpenRouter API key")
+@click.option("--path", default=".", help="Path to codebase (default: current dir)")
+def init(key, path):
+    """
+    Khởi tạo GraphRAG cho project hiện tại.
+    Tạo .env, start Neo4j Docker, parse + enrich toàn bộ codebase.
+    """
+    abs_path = os.path.abspath(path).replace("\\", "/")
+    
+    # Load env from target path if exists
+    from dotenv import load_dotenv
+    env_path = os.path.join(abs_path, ".env")
+    if os.path.exists(env_path):
+        load_dotenv(env_path)
+        
+    api_key = key or os.getenv("OPENROUTER_API_KEY")
+    if not api_key:
+        api_key = click.prompt("OpenRouter API key", hide_input=True)
+        
+    # Write or update .env in target directory
+    lines = []
+    if os.path.exists(env_path):
+        with open(env_path, "r", encoding="utf-8") as f:
+            lines = f.readlines()
+            
+    has_key = False
+    has_path = False
+    new_lines = []
+    for line in lines:
+        if line.strip().startswith("OPENROUTER_API_KEY="):
+            new_lines.append(f"OPENROUTER_API_KEY={api_key}\n")
+            has_key = True
+        elif line.strip().startswith("CODEBASE_PATH="):
+            new_lines.append(f"CODEBASE_PATH={abs_path}\n")
+            has_path = True
+        else:
+            new_lines.append(line)
+            
+    if not has_key:
+        new_lines.append(f"OPENROUTER_API_KEY={api_key}\n")
+    if not has_path:
+        new_lines.append(f"CODEBASE_PATH={abs_path}\n")
+        
+    with open(env_path, "w", encoding="utf-8") as f:
+        f.writelines(new_lines)
+        
+    console.print(f"[green]✓ Configured {env_path}[/green]")
+    
+    # Programmatically configure nelgraph
+    nelgraph.configure(codebase_path=abs_path, openrouter_api_key=api_key)
+    
+    # Run full initialization pipeline
+    nelgraph.run_init()
+    
+    # Install git post-commit hook
+    _install_git_hook(abs_path)
+
+@main.command()
+@click.option("--silent", is_flag=True, help="Run silently without printing to stdout")
+def sync(silent):
+    """Sync thủ công — parse files đã thay đổi kể từ lần sync cuối."""
+    if silent:
+        # Redirect stdout/stderr to devnull
+        sys.stdout = open(os.devnull, 'w')
+        sys.stderr = open(os.devnull, 'w')
+        
+    # Run sync pipeline
+    nelgraph.run_sync()
+
+@main.command()
+def status():
+    """Xem trạng thái graph hiện tại."""
+    # Run status helper
+    from nelgraph.initialize_graph import run_status
+    run_status()
+
+@main.command()
+def watch():
+    """Chạy file watcher — tự sync khi có file thay đổi."""
+    from nelgraph.updater.watcher import start_watcher
+    start_watcher()
+
+if __name__ == "__main__":
+    main()

nelgraph-1.0.0/nelgraph/community/__init__.py

@@ -0,0 +1 @@
+# GraphRAG Community Detection and Summarization Package

nelgraph-1.0.0/nelgraph/community/detector.py

@@ -0,0 +1,68 @@
+import networkx as nx
+import igraph as ig
+import leidenalg
+from nelgraph.graph.neo4j_client import get_client
+
+
+def build_networkx_graph() -> nx.Graph:
+    """Convert Neo4j graph to NetworkX graph for algorithms."""
+    client = get_client()
+
+    G = nx.Graph()
+
+    # Get all nodes
+    nodes = client.run("MATCH (n) WHERE n.name IS NOT NULL RETURN elementId(n) as id, labels(n) as labels, n.name as name")
+    for record in nodes:
+        G.add_node(record["id"], name=record["name"], label=record["labels"][0] if record["labels"] else "Unknown")
+
+    # Get all edges
+    edges = client.run("MATCH (a)-[r]->(b) WHERE a.name IS NOT NULL AND b.name IS NOT NULL RETURN elementId(a) as from_id, elementId(b) as to_id, type(r) as rel_type")
+    for record in edges:
+        G.add_edge(record["from_id"], record["to_id"], rel_type=record["rel_type"])
+
+    print(f"[Community] NetworkX graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges")
+    return G
+
+
+def detect_communities() -> dict:
+    """
+    Run Leiden algorithm via igraph/leidenalg.
+    Returns mapping node_id -> community_id.
+    Saves community_id to Neo4j nodes.
+    """
+    G = build_networkx_graph()
+    if G.number_of_nodes() == 0:
+        print("[Community] Empty graph, skipping community detection.")
+        return {}
+
+    # Convert NetworkX to igraph
+    nx_nodes = list(G.nodes())
+    node_id_map = {n: i for i, n in enumerate(nx_nodes)}
+
+    ig_graph = ig.Graph()
+    ig_graph.add_vertices(len(nx_nodes))
+
+    for u, v in G.edges():
+        ig_graph.add_edge(node_id_map[u], node_id_map[v])
+
+    # Run Leiden algorithm
+    partition = leidenalg.find_partition(ig_graph, leidenalg.ModularityVertexPartition)
+
+    # Build result mapping: neo4j_node_id -> community_id
+    result = {}
+    for community_id, members in enumerate(partition):
+        for member_idx in members:
+            neo4j_id = nx_nodes[member_idx]
+            result[neo4j_id] = community_id
+
+    print(f"[Community] Detected {len(partition)} communities.")
+
+    # Save community IDs to Neo4j
+    client = get_client()
+    for node_id, community_id in result.items():
+        client.run("""
+            MATCH (n) WHERE elementId(n) = $node_id
+            SET n.community_id = $community_id
+        """, {"node_id": node_id, "community_id": community_id})
+
+    return result