nervapack 0.1.0__py3-none-any.whl

nervapack/__init__.py

@@ -0,0 +1 @@
+# NervaPack module

nervapack/cli.py

@@ -0,0 +1,278 @@
+import typer
+from rich.console import Console
+
+app = typer.Typer(help="NervaPack: Privacy-first, offline knowledge graph for developers.")
+console = Console()
+
+@app.command()
+def init():
+    """
+    Initialize a NervaPack graph in the current directory.
+    """
+    console.print("[bold green]Initializing NervaPack graph...[/bold green]")
+    # TODO: Implement initialization logic
+    console.print("Initialization complete.")
+
+@app.command()
+def ingest(path: str = typer.Argument(".", help="Path to the repository to ingest")):
+    """
+    Ingest a repository, building the AST and Vector graph.
+    """
+    from nervapack.parser.ast_parser import scan_directory
+    from nervapack.graph.builder import GraphBuilder
+
+    console.print(f"[bold blue]Ingesting repository at {path}...[/bold blue]")
+    
+    console.print("Scanning directory for code entities...")
+    entities = scan_directory(path)
+    console.print(f"Found {len(entities)} AST entities.")
+
+    console.print("Building deterministic Structural Graph...")
+    builder = GraphBuilder()
+    graph = builder.build_from_entities(entities)
+    builder.save_graph()
+    console.print(f"Graph saved with {graph.number_of_nodes()} nodes and {graph.number_of_edges()} edges.")
+    
+    console.print("Ingesting AST nodes into Vector Store...")
+    try:
+        from nervapack.graph.vector_store import VectorStore
+        vstore = VectorStore()
+        
+        # We'll just ingest them with basic code text for now (in production, LLMSummarizer would summarize them first)
+        ast_docs = []
+        for e in entities:
+            # We construct a mock 'summary' that just describes the node to avoid calling local Ollama during tests
+            node_id = f"{e.type}:{e.file_path}:{e.name}:{e.start_line}"
+            ast_docs.append({
+                "node_id": node_id,
+                "summary": f"This is a {e.type} named {e.name} in {e.file_path}. Code:\n{e.content}",
+                "file_path": e.file_path
+            })
+        
+        vstore.ingest_ast_entities(ast_docs)
+        console.print("AST Vector ingestion complete.")
+
+        from nervapack.parser.md_chunker import scan_markdown_directory
+        console.print("Scanning directory for Markdown docs...")
+        md_chunks = scan_markdown_directory(path)
+        console.print(f"Found {len(md_chunks)} Markdown chunks.")
+        
+        if md_chunks:
+            # Ingest to vector store
+            vstore.ingest_chunks(md_chunks)
+            
+            # Add to graph and Bind with LLM
+            from nervapack.llm.summarizer import LLMSummarizer
+            llm = LLMSummarizer()
+            
+            console.print("Binding documentation to AST (this may take a while)...")
+            for i, chunk in enumerate(md_chunks):
+                md_node_id = f"md_{chunk['file_path']}_{i}"
+                if not graph.has_node(md_node_id):
+                    graph.add_node(md_node_id, type="markdown", header=chunk['header'], content=chunk['content'], file_path=chunk['file_path'])
+                
+                # Try to bind
+                matched_ids = llm.bind_docs_to_ast(chunk['content'], ast_docs)
+                for matched_id in matched_ids:
+                    if graph.has_node(matched_id):
+                        graph.add_edge(md_node_id, matched_id, relation="EXPLAINS")
+            
+            builder.save_graph()
+            console.print("Semantic binding complete.")
+
+    except Exception as e:
+        console.print(f"[bold red]Error during ingestion:[/bold red] {e}")
+
+    console.print("[bold green]Ingestion complete.[/bold green]")
+
+@app.command()
+def sync(path: str = typer.Argument(".", help="Path to the repository to sync")):
+    """
+    Sync graph with modified git files.
+    """
+    from nervapack.git.tracker import GitTracker
+    from nervapack.graph.builder import GraphBuilder
+    from nervapack.graph.vector_store import VectorStore
+    from nervapack.parser.ast_parser import ASTParser
+    from nervapack.parser.md_chunker import MarkdownChunker
+    import os
+
+    console.print("[bold blue]Syncing changed files with NervaPack graph...[/bold blue]")
+    
+    tracker = GitTracker(path)
+    if not tracker.repo:
+        console.print("[bold red]Not a git repository. Cannot sync.[/bold red]")
+        raise typer.Exit(1)
+        
+    changed_files = tracker.get_changed_files()
+    if not changed_files:
+        console.print("[bold green]No files changed. Graph is up to date.[/bold green]")
+        raise typer.Exit(0)
+        
+    console.print(f"Found {len(changed_files)} changed files.")
+    
+    try:
+        builder = GraphBuilder()
+        graph = builder.load_graph()
+        vstore = VectorStore()
+    except Exception as e:
+        console.print(f"[bold red]Failed to load graph or vector store. Run 'nervapack ingest' first.[/bold red]")
+        raise typer.Exit(1)
+
+    ast_parser = ASTParser()
+    md_chunker = MarkdownChunker()
+    
+    ast_docs = []
+    
+    # Pre-gather all existing ast_docs for LLM binding
+    # We reconstruct a simple mock summary for existing nodes just for binding if needed.
+    # In a real app, we'd pull these from the graph directly.
+    for node, data in graph.nodes(data=True):
+        if data.get("type") in ["class", "function", "import"]:
+            ast_docs.append({
+                "node_id": node,
+                "summary": f"This is a {data.get('type')} named {data.get('name')} in {data.get('file_path')}. Code:\n{data.get('content')}"
+            })
+
+    for f in changed_files:
+        file_path = os.path.join(path, f)
+        
+        # 1. Prune old nodes and vectors
+        builder.remove_nodes_for_file(file_path)
+        vstore.delete_by_file(file_path)
+        
+        if not os.path.exists(file_path):
+            console.print(f"Removed [red]{f}[/red]")
+            continue
+            
+        # 2. Re-parse and insert
+        if file_path.endswith(('.py', '.js', '.jsx', '.ts', '.tsx')):
+            entities = ast_parser.parse_file(file_path)
+            
+            # Add to graph
+            file_node_id = f"file:{file_path}"
+            if not graph.has_node(file_node_id):
+                graph.add_node(file_node_id, type="file", path=file_path)
+                
+            for entity in entities:
+                entity_node_id = f"{entity.type}:{entity.file_path}:{entity.name}:{entity.start_line}"
+                graph.add_node(
+                    entity_node_id, 
+                    type=entity.type, 
+                    name=entity.name, 
+                    file_path=entity.file_path,
+                    start_line=entity.start_line,
+                    end_line=entity.end_line,
+                    content=entity.content
+                )
+                graph.add_edge(file_node_id, entity_node_id, relation="DEFINES")
+                
+                # Add to vector store
+                node_summary = {"node_id": entity_node_id, "summary": f"This is a {entity.type} named {entity.name} in {entity.file_path}. Code:\n{entity.content}", "file_path": entity.file_path}
+                vstore.ingest_ast_entities([node_summary])
+                ast_docs.append(node_summary)
+                
+            console.print(f"Updated AST for [green]{f}[/green]")
+            
+        elif file_path.endswith('.md'):
+            chunks = md_chunker.chunk_file(file_path)
+            if chunks:
+                vstore.ingest_chunks(chunks)
+                
+                from nervapack.llm.summarizer import LLMSummarizer
+                llm = LLMSummarizer()
+                for i, chunk in enumerate(chunks):
+                    md_node_id = f"md_{chunk['file_path']}_{i}"
+                    graph.add_node(md_node_id, type="markdown", header=chunk['header'], content=chunk['content'], file_path=chunk['file_path'])
+                    matched_ids = llm.bind_docs_to_ast(chunk['content'], ast_docs)
+                    for matched_id in matched_ids:
+                        if graph.has_node(matched_id):
+                            graph.add_edge(md_node_id, matched_id, relation="EXPLAINS")
+                            
+            console.print(f"Updated Markdown for [cyan]{f}[/cyan]")
+            
+    builder.save_graph()
+    console.print("[bold green]Sync complete.[/bold green]")
+
+@app.command()
+def query(prompt: str = typer.Argument(..., help="Query to run against the knowledge graph")):
+    """
+    Query the knowledge graph for context.
+    """
+    from nervapack.graph.builder import GraphBuilder
+    from nervapack.graph.vector_store import VectorStore
+    from nervapack.graph.retrieval import GraphRetriever
+
+    console.print(f"[bold magenta]Running query:[/bold magenta] {prompt}")
+    
+    try:
+        builder = GraphBuilder()
+        graph = builder.load_graph()
+    except Exception as e:
+        console.print(f"[bold red]Failed to load graph:[/bold red] {e}. Run 'nervapack ingest' first.")
+        raise typer.Exit(1)
+
+    try:
+        vstore = VectorStore()
+        results = vstore.search(prompt, n_results=3)
+    except Exception as e:
+        console.print(f"[bold red]Failed to query vector store:[/bold red] {e}")
+        raise typer.Exit(1)
+
+    start_nodes = []
+    if results and results['ids'] and len(results['ids']) > 0:
+        start_nodes = results['ids'][0]
+
+    if not start_nodes:
+        console.print("No relevant nodes found in vector search.")
+        raise typer.Exit(0)
+
+    console.print(f"Found {len(start_nodes)} seed nodes. Traversing graph...")
+    retriever = GraphRetriever(graph)
+    subgraph = retriever.retrieve_context(start_nodes, max_hops=1)
+    
+    markdown_context = retriever.format_as_markdown(subgraph)
+    
+    console.print("\n[bold cyan]--- Retrieved Context ---[/bold cyan]\n")
+    console.print(markdown_context)
+    console.print("\n[bold cyan]--- End Context ---[/bold cyan]\n")
+    console.print("Query complete.")
+
+@app.command()
+def status():
+    """
+    Show the status of the local NervaPack graph.
+    """
+    from nervapack.graph.builder import GraphBuilder
+    from nervapack.git.tracker import GitTracker
+    
+    console.print("[bold cyan]NervaPack Status:[/bold cyan]")
+    
+    try:
+        builder = GraphBuilder()
+        graph = builder.load_graph()
+        console.print(f"- Graph loaded: [green]Yes[/green]")
+        console.print(f"- Nodes: [cyan]{graph.number_of_nodes()}[/cyan]")
+        console.print(f"- Edges: [cyan]{graph.number_of_edges()}[/cyan]")
+    except Exception:
+        console.print("- Graph loaded: [red]No (Run 'nervapack ingest')[/red]")
+
+    gitsync = GitTracker()
+    if gitsync.repo:
+        changed = gitsync.get_changed_files()
+        console.print(f"- Git repo detected: [green]Yes[/green]")
+        if changed:
+            console.print(f"- Unsynced changes: [yellow]{len(changed)} file(s)[/yellow]")
+            for f in changed[:5]:
+                console.print(f"  - {f}")
+            if len(changed) > 5:
+                console.print("  - ...")
+        else:
+            console.print("- Unsynced changes: [green]None[/green]")
+    else:
+        console.print("- Git repo detected: [yellow]No[/yellow]")
+        
+    console.print("Status: OK")
+
+if __name__ == "__main__":
+    app()

nervapack/git/tracker.py

@@ -0,0 +1,46 @@
+import git
+from typing import List
+
+class GitTracker:
+    def __init__(self, repo_path: str = "."):
+        self.repo_path = repo_path
+        try:
+            self.repo = git.Repo(self.repo_path)
+        except git.InvalidGitRepositoryError:
+            self.repo = None
+
+    def get_changed_files(self, commit_sha: str = None) -> List[str]:
+        """
+        Gets a list of modified files.
+        If commit_sha is provided, diffs against that.
+        Otherwise diffs the working tree against HEAD.
+        """
+        if not self.repo:
+            return []
+
+        changed_files = []
+        
+        if commit_sha:
+            try:
+                commit = self.repo.commit(commit_sha)
+                diffs = commit.diff(commit.parents[0] if commit.parents else None)
+                for diff in diffs:
+                    changed_files.append(diff.b_path or diff.a_path)
+            except Exception:
+                pass
+        else:
+            # Diff working tree against HEAD
+            diffs = self.repo.index.diff(None)
+            for diff in diffs:
+                changed_files.append(diff.b_path or diff.a_path)
+                
+            # Staged diffs
+            staged_diffs = self.repo.index.diff("HEAD")
+            for diff in staged_diffs:
+                changed_files.append(diff.b_path or diff.a_path)
+
+            # Untracked files
+            changed_files.extend(self.repo.untracked_files)
+
+        # Remove duplicates
+        return list(set(changed_files))

nervapack/graph/builder.py

@@ -0,0 +1,58 @@
+import networkx as nx
+from typing import List
+from nervapack.parser.ast_parser import ParsedEntity
+
+class GraphBuilder:
+    def __init__(self):
+        self.graph = nx.DiGraph()
+
+    def build_from_entities(self, entities: List[ParsedEntity]) -> nx.DiGraph:
+        """
+        Takes a list of ParsedEntity and constructs a directed graph.
+        Files are nodes, entities are nodes. Contains/Defines edges connect them.
+        """
+        for entity in entities:
+            # File Node
+            file_node_id = f"file:{entity.file_path}"
+            if not self.graph.has_node(file_node_id):
+                self.graph.add_node(file_node_id, type="file", path=entity.file_path)
+
+            # Entity Node
+            # Unique ID for the entity
+            entity_node_id = f"{entity.type}:{entity.file_path}:{entity.name}:{entity.start_line}"
+            self.graph.add_node(
+                entity_node_id, 
+                type=entity.type, 
+                name=entity.name, 
+                file_path=entity.file_path,
+                start_line=entity.start_line,
+                end_line=entity.end_line,
+                content=entity.content
+            )
+
+            # Edge from File -> Entity
+            self.graph.add_edge(file_node_id, entity_node_id, relation="DEFINES")
+
+        return self.graph
+
+    def save_graph(self, path: str = ".nervapack/graph.graphml"):
+        import os
+        os.makedirs(os.path.dirname(path), exist_ok=True)
+        nx.write_graphml(self.graph, path)
+
+    def load_graph(self, path: str = ".nervapack/graph.graphml"):
+        self.graph = nx.read_graphml(path)
+        return self.graph
+
+    def remove_nodes_for_file(self, file_path: str):
+        """Removes the file node and all entities associated with it."""
+        nodes_to_remove = []
+        file_node_id = f"file:{file_path}"
+        if self.graph.has_node(file_node_id):
+            nodes_to_remove.append(file_node_id)
+        
+        for node, data in self.graph.nodes(data=True):
+            if data.get("file_path") == file_path:
+                nodes_to_remove.append(node)
+                
+        self.graph.remove_nodes_from(nodes_to_remove)

nervapack/graph/retrieval.py

@@ -0,0 +1,75 @@
+import networkx as nx
+from typing import List, Set, Dict
+
+class GraphRetriever:
+    def __init__(self, graph: nx.DiGraph):
+        self.graph = graph
+
+    def retrieve_context(self, start_node_ids: List[str], max_hops: int = 2) -> nx.DiGraph:
+        """
+        Retrieves a sub-graph using K-Hop BFS from the given start nodes.
+        Uses Betweenness Centrality to prune high-degree "hub" nodes if necessary.
+        """
+        visited = set()
+        queue = [(node_id, 0) for node_id in start_node_ids if self.graph.has_node(node_id)]
+        
+        subgraph_nodes = set()
+
+        while queue:
+            current_node, hops = queue.pop(0)
+            
+            if current_node in visited:
+                continue
+                
+            visited.add(current_node)
+            subgraph_nodes.add(current_node)
+
+            if hops < max_hops:
+                for neighbor in self.graph.neighbors(current_node):
+                    if neighbor not in visited:
+                        # Pruning logic: If degree is extremely high, it might be a utility file/hub.
+                        # For now, we skip pruning if it's within hops, but this is where betweenness centrality could be applied.
+                        queue.append((neighbor, hops + 1))
+                
+                # Also traverse incoming edges
+                for predecessor in self.graph.predecessors(current_node):
+                    if predecessor not in visited:
+                        queue.append((predecessor, hops + 1))
+
+        return self.graph.subgraph(subgraph_nodes).copy()
+
+    def format_as_markdown(self, subgraph: nx.DiGraph) -> str:
+        """
+        Formats the retrieved sub-graph into a minimized, clean Markdown block.
+        """
+        markdown_lines = []
+        markdown_lines.append("# NervaPack Context Retrieval\n")
+        
+        # Group by file
+        files = {}
+        for node, data in subgraph.nodes(data=True):
+            if data.get('type') == 'file':
+                continue
+            
+            file_path = data.get('file_path', 'Unknown')
+            if file_path not in files:
+                files[file_path] = []
+            files[file_path].append(data)
+
+        for file_path, nodes in files.items():
+            markdown_lines.append(f"## File: `{file_path}`\n")
+            # Sort by start line
+            nodes = sorted(nodes, key=lambda x: x.get('start_line', 0))
+            for node_data in nodes:
+                node_type = node_data.get('type', 'entity').upper()
+                name = node_data.get('name', 'Unknown')
+                lines = f"(L{node_data.get('start_line', '?')}-L{node_data.get('end_line', '?')})"
+                markdown_lines.append(f"### {node_type}: {name} {lines}")
+                
+                content = node_data.get('content', '')
+                if content:
+                    markdown_lines.append("```")
+                    markdown_lines.append(content)
+                    markdown_lines.append("```\n")
+
+        return "\n".join(markdown_lines)

nervapack/graph/vector_store.py

@@ -0,0 +1,63 @@
+import chromadb
+from typing import List, Dict
+
+class VectorStore:
+    def __init__(self, db_path: str = ".nervapack/chroma_db"):
+        self.client = chromadb.PersistentClient(path=db_path)
+        # We use a single collection for both AST node summaries and Markdown chunks
+        self.collection = self.client.get_or_create_collection(name="nervapack_nodes")
+
+    def ingest_chunks(self, chunks: List[Dict[str, str]]):
+        """
+        Ingest a list of Markdown chunks into the vector store.
+        """
+        if not chunks:
+            return
+
+        documents = []
+        metadatas = []
+        ids = []
+
+        for i, chunk in enumerate(chunks):
+            documents.append(chunk["content"])
+            metadatas.append({"header": chunk["header"], "file_path": chunk["file_path"], "type": "markdown"})
+            ids.append(f"md_{chunk['file_path']}_{i}")
+
+        self.collection.add(
+            documents=documents,
+            metadatas=metadatas,
+            ids=ids
+        )
+
+    def ingest_ast_entities(self, entities: List[Dict[str, str]]):
+        """
+        Ingest AST summaries into the vector store.
+        """
+        if not entities:
+            return
+
+        documents = []
+        metadatas = []
+        ids = []
+
+        for entity in entities:
+            documents.append(entity["summary"])
+            # Assuming 'file_path' is added to the entity dict in cli.py
+            metadatas.append({"node_id": entity["node_id"], "type": "ast", "file_path": entity.get("file_path", "")})
+            ids.append(entity["node_id"])
+
+        self.collection.add(
+            documents=documents,
+            metadatas=metadatas,
+            ids=ids
+        )
+
+    def search(self, query: str, n_results: int = 5):
+        return self.collection.query(
+            query_texts=[query],
+            n_results=n_results
+        )
+
+    def delete_by_file(self, file_path: str):
+        """Delete all vectors associated with a specific file."""
+        self.collection.delete(where={"file_path": file_path})

nervapack/llm/summarizer.py

@@ -0,0 +1,45 @@
+import ollama
+from typing import List, Dict
+from nervapack.parser.ast_parser import ParsedEntity
+
+class LLMSummarizer:
+    def __init__(self, model_name: str = "llama3"):
+        self.model_name = model_name
+
+    def summarize_entity(self, entity: ParsedEntity) -> str:
+        """
+        Generate a quick summary for an AST node using Ollama.
+        """
+        prompt = f"Summarize the following code block (Type: {entity.type}, Name: {entity.name}):\n\n```\n{entity.content}\n```\n\nSummary:"
+        try:
+            response = ollama.chat(model=self.model_name, messages=[
+                {"role": "system", "content": "You are a concise code summarizer. Output only a 1-3 sentence summary of what the code does."},
+                {"role": "user", "content": prompt}
+            ])
+            return response['message']['content']
+        except Exception as e:
+            return f"Summary unavailable: {str(e)}"
+
+    def bind_docs_to_ast(self, doc_chunk: str, ast_nodes: List[Dict[str, str]]) -> List[str]:
+        """
+        Takes a documentation chunk and a list of candidate AST nodes.
+        Returns a list of node_ids that the documentation EXPLAINS or IMPLEMENTS.
+        """
+        candidates_str = "\n".join([f"ID: {n['node_id']} | Summary: {n['summary']}" for n in ast_nodes])
+        prompt = (
+            f"Given the following documentation chunk:\n\n{doc_chunk}\n\n"
+            f"Which of the following code entities does it explain or implement? "
+            f"Return a comma-separated list of IDs only, or 'None' if none match.\n\n"
+            f"Candidates:\n{candidates_str}\n\nMatched IDs:"
+        )
+        try:
+            response = ollama.chat(model=self.model_name, messages=[
+                {"role": "system", "content": "You are an AI binding engine. Output ONLY a comma-separated list of IDs."},
+                {"role": "user", "content": prompt}
+            ])
+            content = response['message']['content'].strip()
+            if content.lower() == "none" or not content:
+                return []
+            return [i.strip() for i in content.split(",") if i.strip()]
+        except Exception:
+            return []

nervapack/parser/ast_parser.py

@@ -0,0 +1,91 @@
+import os
+from pathlib import Path
+from dataclasses import dataclass
+from typing import List, Dict, Any, Optional
+
+import tree_sitter_python
+import tree_sitter_javascript
+import tree_sitter_typescript
+from tree_sitter import Language, Parser, Node
+
+@dataclass
+class ParsedEntity:
+    name: str
+    type: str  # 'class', 'function', 'import', 'export'
+    file_path: str
+    start_line: int
+    end_line: int
+    content: str
+    metadata: Dict[str, Any]
+
+class ASTParser:
+    def __init__(self):
+        self.languages = {
+            ".py": Language(tree_sitter_python.language()),
+            ".js": Language(tree_sitter_javascript.language()),
+            ".jsx": Language(tree_sitter_javascript.language()),
+            ".ts": Language(tree_sitter_typescript.language_typescript()),
+            ".tsx": Language(tree_sitter_typescript.language_tsx())
+        }
+        self.parsers = {}
+        for ext, lang in self.languages.items():
+            parser = Parser(lang)
+            self.parsers[ext] = parser
+
+    def parse_file(self, file_path: str) -> List[ParsedEntity]:
+        ext = Path(file_path).suffix
+        if ext not in self.parsers:
+            return []
+
+        with open(file_path, "rb") as f:
+            content_bytes = f.read()
+
+        tree = self.parsers[ext].parse(content_bytes)
+        entities = []
+        self._traverse_tree(tree.root_node, content_bytes, file_path, ext, entities)
+        return entities
+
+    def _traverse_tree(self, node: Node, content_bytes: bytes, file_path: str, ext: str, entities: List[ParsedEntity]):
+        # Simple extraction logic for Python and JS/TS
+        if node.type in ["class_definition", "class_declaration"]:
+            name_node = node.child_by_field_name("name")
+            if name_node:
+                name = name_node.text.decode("utf8")
+                entities.append(self._create_entity(name, "class", node, content_bytes, file_path))
+        
+        elif node.type in ["function_definition", "function_declaration", "method_definition"]:
+            name_node = node.child_by_field_name("name")
+            if name_node:
+                name = name_node.text.decode("utf8")
+                entities.append(self._create_entity(name, "function", node, content_bytes, file_path))
+        
+        elif node.type in ["import_statement", "import_from_statement", "import_declaration"]:
+            # For simplicity, extract the entire import block as an entity for the graph
+            name = f"import_{node.start_point[0]}"
+            entities.append(self._create_entity(name, "import", node, content_bytes, file_path))
+
+        for child in node.children:
+            self._traverse_tree(child, content_bytes, file_path, ext, entities)
+
+    def _create_entity(self, name: str, entity_type: str, node: Node, content_bytes: bytes, file_path: str) -> ParsedEntity:
+        return ParsedEntity(
+            name=name,
+            type=entity_type,
+            file_path=file_path,
+            start_line=node.start_point[0] + 1,
+            end_line=node.end_point[0] + 1,
+            content=node.text.decode("utf8"),
+            metadata={}
+        )
+
+def scan_directory(directory: str) -> List[ParsedEntity]:
+    parser = ASTParser()
+    all_entities = []
+    for root, _, files in os.walk(directory):
+        # Ignore common bad directories
+        if any(ignored in root for ignored in [".git", "node_modules", "venv", "__pycache__"]):
+            continue
+        for file in files:
+            file_path = os.path.join(root, file)
+            all_entities.extend(parser.parse_file(file_path))
+    return all_entities

nervapack/parser/md_chunker.py

@@ -0,0 +1,64 @@
+import re
+from typing import List, Dict
+
+class MarkdownChunker:
+    def __init__(self):
+        # A simple regex for matching markdown headers
+        self.header_regex = re.compile(r'^(#{1,6})\s+(.*)')
+
+    def chunk_file(self, file_path: str) -> List[Dict[str, str]]:
+        """
+        Parses a Markdown file and returns chunks separated by headers.
+        """
+        try:
+            with open(file_path, 'r', encoding='utf-8') as f:
+                lines = f.readlines()
+        except Exception:
+            return []
+
+        chunks = []
+        current_chunk = []
+        current_header = "Document Root"
+        
+        for line in lines:
+            match = self.header_regex.match(line)
+            if match:
+                # Save previous chunk
+                if current_chunk:
+                    content = "".join(current_chunk).strip()
+                    if content:
+                        chunks.append({
+                            "header": current_header,
+                            "content": content,
+                            "file_path": file_path
+                        })
+                current_header = match.group(2).strip()
+                current_chunk = [line]
+            else:
+                current_chunk.append(line)
+                
+        # Add the last chunk
+        if current_chunk:
+            content = "".join(current_chunk).strip()
+            if content:
+                chunks.append({
+                    "header": current_header,
+                    "content": content,
+                    "file_path": file_path
+                })
+                
+        return chunks
+
+def scan_markdown_directory(directory: str) -> List[Dict[str, str]]:
+    import os
+    chunker = MarkdownChunker()
+    all_chunks = []
+    for root, _, files in os.walk(directory):
+        # Ignore common bad directories
+        if any(ignored in root for ignored in [".git", "node_modules", "venv", "__pycache__"]):
+            continue
+        for file in files:
+            if file.endswith(".md"):
+                file_path = os.path.join(root, file)
+                all_chunks.extend(chunker.chunk_file(file_path))
+    return all_chunks

nervapack-0.1.0.dist-info/METADATA

@@ -0,0 +1,251 @@
+Metadata-Version: 2.4
+Name: nervapack
+Version: 0.1.0
+Summary: Privacy-first, offline knowledge graph for developers
+Author-email: Preetam Ramdhave <ramdhavepreetam@gmail.com>
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/ramdhavepreetam/NervaPack
+Project-URL: Repository, https://github.com/ramdhavepreetam/NervaPack
+Project-URL: Bug Tracker, https://github.com/ramdhavepreetam/NervaPack/issues
+Project-URL: Documentation, https://github.com/ramdhavepreetam/NervaPack#readme
+Keywords: knowledge-graph,rag,ast,code-search,offline,llm,developer-tools
+Classifier: Development Status :: 4 - Beta
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: Topic :: Software Development :: Build Tools
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: typer>=0.9.0
+Requires-Dist: rich>=13.0.0
+Requires-Dist: tree-sitter>=0.22.3
+Requires-Dist: tree-sitter-python>=0.21.0
+Requires-Dist: tree-sitter-javascript>=0.21.2
+Requires-Dist: tree-sitter-typescript>=0.21.2
+Requires-Dist: networkx>=3.1
+Requires-Dist: ollama>=0.2.0
+Requires-Dist: gitpython>=3.1.30
+Requires-Dist: chromadb>=0.4.24
+Dynamic: license-file
+
+# NervaPack
+
+[![PyPI version](https://img.shields.io/pypi/v/nervapack.svg)](https://pypi.org/project/nervapack/)
+[![Python Versions](https://img.shields.io/pypi/pyversions/nervapack.svg)](https://pypi.org/project/nervapack/)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+
+**NervaPack** is a privacy-first, offline knowledge graph for your codebase. It solves two fundamental problems with standard Vector RAG:
+
+- **Token waste** — chunk-based RAG retrieves blobs of text that may only tangentially relate to your query, bloating your context window.
+- **Privacy risk** — sending code to cloud embedding APIs leaks your proprietary logic.
+
+NervaPack runs 100% on your machine. It uses `tree-sitter` to parse your codebase into a deterministic Abstract Syntax Tree graph, then uses a local Ollama model to draw hard semantic edges between your documentation and your code. Queries traverse this graph with a K-Hop BFS, returning a hyper-targeted, token-efficient context window — no cloud required.
+
+---
+
+## Why NervaPack vs. standard Vector RAG
+
+| | Standard Vector RAG | NervaPack |
+|---|---|---|
+| **Parsing** | Arbitrary text chunks | Deterministic AST nodes (class, function, import) |
+| **Retrieval** | Nearest-neighbor blob | K-Hop BFS on a structural graph |
+| **Doc ↔ Code links** | None | Hard `EXPLAINS` edges drawn by local LLM |
+| **Privacy** | Cloud embeddings | 100% local (ChromaDB + Ollama) |
+| **Incremental sync** | Re-index everything | Surgical per-file update via GitPython diff |
+
+---
+
+## Prerequisites
+
+- **Python 3.10+**
+- **Ollama** — install from [ollama.com](https://ollama.com/), then pull a model:
+  ```bash
+  ollama pull llama3
+  ```
+  NervaPack defaults to `llama3`. Any model that can follow instructions works.
+- **Git** — your project must be a git repository (`git init` if not).
+
+---
+
+## Installation
+
+**Option A — Homebrew (Mac/Linux, recommended)**
+```bash
+brew tap ramdhavepreetam/nervapack
+brew install nervapack
+```
+
+**Option B — pipx (any platform, cleanest Python install)**
+```bash
+pipx install nervapack
+```
+
+**Option C — pip**
+```bash
+pip install nervapack
+```
+
+> On first run, `chromadb` downloads `onnxruntime` embedding models to your cache and `tree-sitter` compiles its language bindings. This is a one-time setup (~1–2 min).
+
+---
+
+## Quick Start
+
+```bash
+cd your-project/
+
+# 1. Build the knowledge graph (run once)
+nervapack ingest .
+
+# 2. Query for context
+nervapack query "How does authentication work?"
+
+# 3. After modifying files, sync the graph incrementally
+nervapack sync .
+
+# 4. Check graph health
+nervapack status
+```
+
+---
+
+## Command Reference
+
+### `nervapack ingest [PATH]`
+
+Scans `PATH` (default: `.`) and builds the full knowledge graph.
+
+What happens:
+1. `tree-sitter` parses all `.py`, `.js`, `.jsx`, `.ts`, `.tsx` files into Classes, Functions, and Imports — exact AST nodes, not text chunks.
+2. All `.md` files are chunked by header hierarchy.
+3. Each Markdown chunk is sent to your local Ollama model. If the model identifies a code entity the prose explains, a hard `EXPLAINS` edge is written into the graph.
+4. All nodes are embedded and stored in a local ChromaDB instance (`.nervapack/chroma_db`).
+
+> The initial LLM binding pass is the slowest step. On a large repo with many docs, budget several minutes.
+
+---
+
+### `nervapack query PROMPT`
+
+Retrieves context from the graph for a natural-language prompt.
+
+What happens:
+1. The prompt is embedded and ChromaDB returns the top-3 most semantically similar nodes.
+2. Those nodes seed a K-Hop Breadth-First Search (default `max_hops=1`) through the NetworkX graph.
+3. Adjacent nodes — including any Markdown docs linked via `EXPLAINS` edges — are collected into a compressed Markdown snippet and printed to your terminal.
+
+The output is designed to be pasted directly into an LLM prompt as context.
+
+---
+
+### `nervapack sync [PATH]`
+
+Incrementally updates the graph for files changed since the last ingest.
+
+What happens:
+1. `GitPython` diffs your working tree to find modified and deleted files.
+2. For each changed file, old graph nodes and ChromaDB vectors are pruned.
+3. Only the changed files are re-parsed and re-ingested.
+
+A full `ingest` on a large codebase can take minutes. `sync` turns that into a 2–5 second surgical update.
+
+---
+
+### `nervapack status`
+
+Prints the current state of the graph: node count, edge count, and any files that are out of sync with the graph.
+
+---
+
+## Configuration
+
+NervaPack reads the Ollama model from the `LLMSummarizer` class (`src/nervapack/llm/summarizer.py`). To use a different model, set `model` to any model you have pulled locally:
+
+```python
+# src/nervapack/llm/summarizer.py
+self.model = "phi3"   # or "mistral", "codellama", etc.
+```
+
+Ollama is expected at `http://localhost:11434` (its default). To use a remote Ollama instance, set `OLLAMA_HOST`:
+
+```bash
+OLLAMA_HOST=http://my-server:11434 nervapack ingest .
+```
+
+---
+
+## Architecture
+
+```
+nervapack ingest .
+       │
+       ├─ ASTParser (tree-sitter)
+       │    └─ ParsedEntity[]: class, function, import
+       │
+       ├─ GraphBuilder (NetworkX DiGraph)
+       │    ├─ Nodes: file, class, function, import, markdown
+       │    └─ Edges: DEFINES, EXPLAINS
+       │
+       ├─ LLMSummarizer (Ollama)
+       │    └─ Draws EXPLAINS edges: markdown → code entity
+       │
+       └─ VectorStore (ChromaDB)
+            └─ Embeds node summaries for semantic search
+
+nervapack query "..."
+       │
+       ├─ VectorStore.search() → seed node IDs
+       └─ GraphRetriever.retrieve_context() → BFS subgraph → Markdown
+```
+
+**Storage layout** (inside your project root):
+```
+.nervapack/
+├── graph.graphml       # NetworkX graph (deterministic structure)
+└── chroma_db/          # ChromaDB (semantic embeddings)
+```
+
+**Source modules:**
+| Module | Responsibility |
+|---|---|
+| `nervapack.parser.ast_parser` | Tree-sitter parsing → `ParsedEntity` objects |
+| `nervapack.parser.md_chunker` | Markdown → header-delimited chunks |
+| `nervapack.graph.builder` | Build and persist the NetworkX DiGraph |
+| `nervapack.graph.vector_store` | ChromaDB ingest and semantic search |
+| `nervapack.graph.retrieval` | K-Hop BFS context extraction |
+| `nervapack.llm.summarizer` | Local Ollama interface for LLM binding |
+| `nervapack.git.tracker` | GitPython diff for incremental sync |
+
+---
+
+## Privacy
+
+NervaPack is 100% offline. No code, documentation, or query ever leaves your machine:
+
+- Embeddings are generated by ChromaDB's built-in local model.
+- LLM calls go exclusively to `localhost:11434` (your Ollama instance).
+- All graph and vector data is stored in `.nervapack/` inside your project.
+
+Add `.nervapack/` to your `.gitignore` to keep it out of version control.
+
+---
+
+## Contributing
+
+1. Fork the repo and create a branch.
+2. Make your changes with tests where applicable.
+3. Open a pull request against `master`.
+
+Bug reports and feature requests go to the [issue tracker](https://github.com/ramdhavepreetam/NervaPack/issues).
+
+---
+
+## License
+
+MIT — see [LICENSE](LICENSE).

nervapack-0.1.0.dist-info/RECORD

@@ -0,0 +1,19 @@
+nervapack/__init__.py,sha256=40vPG8PV_0KBbr_bW_RQIJCfrazA01s1-u5gPWzPFjs,19
+nervapack/cli.py,sha256=joW4W8WcrXW2aQ4SiBwonvF0JD9YMbAgtVPJec705Tw,11193
+nervapack/git/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+nervapack/git/tracker.py,sha256=vpk0UJxBEfxZ1lyVqcRbHxHoW2cA81zpsHhAGErLTkw,1498
+nervapack/graph/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+nervapack/graph/builder.py,sha256=HN0NM3-KjNLem4-QIw_dRYirgVbWQVwTyfvgCPRXiUE,2166
+nervapack/graph/retrieval.py,sha256=SCvbkzty5d6zA_10yMj7GJVwxyRPLy4TLBPK5SN7DVQ,3043
+nervapack/graph/vector_store.py,sha256=5-rBqgwb5RMZvHrEuDtv6kzwV1-XmDrpOE9gLiHBbjQ,2036
+nervapack/llm/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+nervapack/llm/summarizer.py,sha256=NCmY_8K0TaSDFglyq1JyOMOvDOxXxQf0vCtMxPNf5x4,2171
+nervapack/parser/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+nervapack/parser/ast_parser.py,sha256=N-x1Hk1sDRNgOWftB54le7FBix7Tn_FjSo6DBZhJBXA,3603
+nervapack/parser/md_chunker.py,sha256=o3mKfWDaQLdtezo-AUHxO45Ii53d386fJnAwZJnMzL8,2176
+nervapack-0.1.0.dist-info/licenses/LICENSE,sha256=FWEWrQ3gFAwZTFoPP1mSI1RpPc1Qbzpp2X1zuW_jiyk,1073
+nervapack-0.1.0.dist-info/METADATA,sha256=BJMk64mYzV4bzBH9pgszBtm6Mgoag3ZloTNKEUPNjr8,8695
+nervapack-0.1.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+nervapack-0.1.0.dist-info/entry_points.txt,sha256=Ke1CpbU-LPw7I26PCAnpm_n7E0GKtDn8FG0ITWnAgZQ,48
+nervapack-0.1.0.dist-info/top_level.txt,sha256=PKpoqpD5E4FU4pXFYs8qUt7dSQnM_UYGNqF7djWbF04,10
+nervapack-0.1.0.dist-info/RECORD,,

nervapack-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

nervapack-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+nervapack = nervapack.cli:app

nervapack-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Preetam Ramdhave
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

nervapack-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+nervapack