elastro-client 1.3.32__tar.gz → 1.3.34__tar.gz

{elastro_client-1.3.32/elastro_client.egg-info → elastro_client-1.3.34}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: elastro-client
-Version: 1.3.32
+Version: 1.3.34
 Summary: A comprehensive Python library for Elasticsearch management with both programmatic and CLI interfaces
 Author: Austin Jorgensen
 License-Expression: MIT
@@ -29,6 +29,11 @@ Requires-Dist: rich>=10.0.0
 Requires-Dist: rich-click>=1.7.0
 Requires-Dist: fastapi>=0.111.0
 Requires-Dist: uvicorn>=0.30.0
+Requires-Dist: tree-sitter>=0.23.0
+Requires-Dist: tree-sitter-python>=0.23.0
+Requires-Dist: tree-sitter-go>=0.23.0
+Requires-Dist: tree-sitter-javascript>=0.23.0
+Requires-Dist: tree-sitter-typescript>=0.23.0
 Provides-Extra: test
 Requires-Dist: pytest>=7.0.0; extra == "test"
 Requires-Dist: pytest-cov>=3.0.0; extra == "test"

{elastro_client-1.3.32 → elastro_client-1.3.34}/elastro/__init__.py

@@ -4,7 +4,7 @@ Elasticsearch Management Module.
 A module for managing Elasticsearch operations within a pipeline process.
 """
 
-__version__ = "1.3.32"
+__version__ = "1.3.34"
 
 # Core component imports
 from elastro.core.client import ElasticsearchClient

{elastro_client-1.3.32 → elastro_client-1.3.34}/elastro/cli/cli.py

@@ -244,6 +244,7 @@ utils.add_command(utils_templates)
 utils.add_command(aliases)
 
 from elastro.cli.commands.gui import gui
+from elastro.cli.commands.rag import rag_group
 
 # Register Top-Level Groups
 cli.add_command(template_group)
@@ -257,6 +258,7 @@ cli.add_command(ml_group)
 cli.add_command(script_group)
 cli.add_command(painless_group)
 cli.add_command(gui)
+cli.add_command(rag_group)
 
 
 def main() -> None:

elastro_client-1.3.34/elastro/cli/commands/rag.py

@@ -0,0 +1,72 @@
+"""
+Graph RAG Commands for Code Flow Mapping.
+"""
+
+import rich_click as click
+import os
+from typing import Optional
+from elastro.core.client import ElasticsearchClient
+from elastro.core.rag.ingestor import GraphRAGManager
+from elastro.cli.output import format_output
+from rich.console import Console
+
+console = Console()
+
+
+@click.group("rag")
+def rag_group() -> None:
+    """
+    Manage Graph RAG (Retrieval Augmented Generation) capabilities.
+
+    Includes native Code Flow mapping, AST extraction for Python, Go, TypeScript, and Vue,
+    and direct injection into local Elasticsearch for Agentic Codebase Memory.
+    """
+    pass
+
+
+@rag_group.command("ingest")
+@click.argument(
+    "repo_path", type=click.Path(exists=True, file_okay=False, dir_okay=True)
+)
+@click.option(
+    "--index",
+    "-i",
+    type=str,
+    default="fremen_codebase_rag",
+    help="Target Elasticsearch index name (default: fremen_codebase_rag)",
+)
+@click.pass_obj
+def ingest_repo(client: ElasticsearchClient, repo_path: str, index: str) -> None:
+    """
+    Ingest a repository with AST Code Flow Mapping.
+
+    This command scans your codebase, utilizes Tree-sitter polyglot AST parsers
+    to extract 'functions_defined' and 'functions_called', and uses the
+    Elasticsearch Bulk API to stripe the Graph RAG data perfectly into an index.
+
+    Example:
+    ```bash
+    elastro rag ingest /path/to/my/go-repo
+    ```
+    """
+    repo_name = os.path.basename(os.path.abspath(repo_path))
+    console.print(
+        f"[bold cyan]🔍 Initializing Graph RAG AST Parsing for '{repo_name}'[/bold cyan]"
+    )
+
+    manager = GraphRAGManager(client, index)
+
+    try:
+        # We handle index scaffolding internally in GraphRAGManager
+        success_count = manager.ingest_repository(repo_path)
+
+        console.print(
+            f"[bold green]✅ Success![/bold green] Ingested and mapped Code Flows for [bold]{success_count}[/bold] files."
+        )
+        console.print(
+            f"[dim]The AST Graph RAG context is now active in index '{index}'.[/dim]"
+        )
+
+    except Exception as e:
+        console.print(f"[bold red]❌ RAG Ingestion Failed:[/bold red] {str(e)}")
+        exit(1)

elastro_client-1.3.34/elastro/core/rag/ast_parser.py

@@ -0,0 +1,260 @@
+"""
+Abstract Syntax Tree (AST) parser utilizing tree-sitter for Graph RAG Code Flow Mapping.
+Supports polyglot analysis of Python, Go, JavaScript, TypeScript, and Vue.
+Implements Semantic Chunking and Signature Extraction.
+"""
+
+import os
+import tree_sitter  # type: ignore
+from typing import Dict, List, Set, Any
+
+
+class ASTParser:
+    def __init__(self) -> None:
+        self.parsers: Dict[str, tree_sitter.Parser] = {}
+        self._initialize_parsers()
+
+    def _initialize_parsers(self) -> None:
+        """Loads natively compiled tree-sitter language capsules."""
+        try:
+            from tree_sitter import Language, Parser
+            import tree_sitter_python as tsp  # type: ignore
+            import tree_sitter_go as tsg  # type: ignore
+            import tree_sitter_javascript as tsjs  # type: ignore
+            import tree_sitter_typescript as tsts  # type: ignore
+
+            # v0.25+ API natively imports .language() as PyCapsule pointers
+            self.langs = {
+                "python": Language(tsp.language()),
+                "go": Language(tsg.language()),
+                "javascript": Language(tsjs.language()),
+                "typescript": Language(tsts.language_typescript()),
+                # Vue uses specialized HTML embedding, but for script logic, TS/JS parser handles it well
+                "vue": Language(tsts.language_typescript()),
+            }
+
+            for lang_name, lang_obj in self.langs.items():
+                parser = Parser()
+                parser.language = lang_obj
+                self.parsers[lang_name] = parser
+
+        except ImportError as e:
+            # Degrade gracefully if tree-sitter is missing during testing/CI
+            print(f"Graph RAG AST Parser initialization warning: {e}")
+            pass
+
+    def _determine_language(self, ext: str) -> str:
+        ext = ext.lower()
+        if ext == ".py":
+            return "python"
+        elif ext == ".go":
+            return "go"
+        elif ext in [".js", ".jsx"]:
+            return "javascript"
+        elif ext in [".ts", ".tsx"]:
+            return "typescript"
+        elif ext == ".vue":
+            return "vue"
+        return "unsupported"
+
+    def parse_file(self, file_path: str, content: str) -> List[Dict[str, Any]]:
+        """
+        Parses raw code and extracts semantic chunks (functions/classes) with deterministic call graphs.
+        """
+        ext = os.path.splitext(file_path)[1]
+        lang = self._determine_language(ext)
+
+        if lang not in self.parsers:
+            return [
+                {
+                    "chunk_type": "file",
+                    "name": "module",
+                    "content": content,
+                    "functions_defined": [],
+                    "functions_called": [],
+                }
+            ]
+
+        parser = self.parsers[lang]
+        source = content.encode("utf-8")
+        tree = parser.parse(source)
+
+        chunks: List[Dict[str, Any]] = []
+
+        self._extract_chunks(tree.root_node, lang, source, chunks)
+
+        # If no semantic chunks were found, fallback to parsing the whole file as a single module chunk
+        if not chunks:
+            called: Set[str] = set()
+            self._traverse_for_calls(tree.root_node, lang, source, called)
+            chunks.append(
+                {
+                    "chunk_type": "file",
+                    "name": "module",
+                    "content": content,
+                    "functions_defined": [],
+                    "functions_called": sorted(list(called)),
+                }
+            )
+
+        return chunks
+
+    def _extract_chunks(
+        self,
+        node: tree_sitter.Node,
+        lang: str,
+        source: bytes,
+        chunks: List[Dict[str, Any]],
+    ) -> None:
+        """Recursive AST Walk for semantic chunking."""
+        node_type = node.type
+
+        is_chunk = False
+        chunk_name = ""
+        chunk_type = ""
+
+        if lang == "python":
+            if node_type in ["function_definition", "class_definition"]:
+                is_chunk = True
+                chunk_type = "function" if "function" in node_type else "class"
+                name_node = node.child_by_field_name("name")
+                if name_node:
+                    chunk_name = source[
+                        name_node.start_byte : name_node.end_byte
+                    ].decode("utf-8")
+
+        elif lang == "go":
+            if node_type in [
+                "function_declaration",
+                "method_declaration",
+                "type_declaration",
+            ]:
+                is_chunk = True
+                chunk_type = (
+                    "function"
+                    if "function" in node_type or "method" in node_type
+                    else "class"
+                )
+                name_node = node.child_by_field_name("name")
+                if name_node:
+                    chunk_name = source[
+                        name_node.start_byte : name_node.end_byte
+                    ].decode("utf-8")
+
+        elif lang in ["javascript", "typescript", "vue"]:
+            if node_type in [
+                "function_declaration",
+                "method_definition",
+                "class_declaration",
+            ]:
+                is_chunk = True
+                chunk_type = (
+                    "function"
+                    if "function" in node_type or "method" in node_type
+                    else "class"
+                )
+                name_node = node.child_by_field_name("name")
+                if name_node:
+                    chunk_name = source[
+                        name_node.start_byte : name_node.end_byte
+                    ].decode("utf-8")
+            elif node_type == "variable_declarator":
+                name_node = node.child_by_field_name("name")
+                val_node = node.child_by_field_name("value")
+                if name_node and val_node and val_node.type == "arrow_function":
+                    is_chunk = True
+                    chunk_type = "function"
+                    chunk_name = source[
+                        name_node.start_byte : name_node.end_byte
+                    ].decode("utf-8")
+
+        if is_chunk:
+            called: Set[str] = set()
+            # Traverse specifically inside this chunk's boundaries to extract calls
+            self._traverse_for_calls(node, lang, source, called)
+
+            # To ensure proper context window, extract the source of just this node
+            chunk_content = source[node.start_byte : node.end_byte].decode("utf-8")
+
+            chunks.append(
+                {
+                    "chunk_type": chunk_type,
+                    "name": chunk_name or "anonymous",
+                    "content": chunk_content,
+                    "functions_defined": [chunk_name] if chunk_name else [],
+                    "functions_called": sorted(list(called)),
+                }
+            )
+
+        # Continue recursing to find nested functions/classes (e.g., methods inside a class)
+        for child in node.children:
+            self._extract_chunks(child, lang, source, chunks)
+
+    def _traverse_for_calls(
+        self, node: tree_sitter.Node, lang: str, source: bytes, called: Set[str]
+    ) -> None:
+        """Extracts call chains including their parameters/arguments for exact signature mapping."""
+        if node.type == "call_expression":
+            func_node = node.child_by_field_name("function")
+            if func_node:
+                call_text = self._extract_call_chain(func_node, source)
+
+                ignore_list = {
+                    "print",
+                    "len",
+                    "range",
+                    "str",
+                    "int",
+                    "list",
+                    "dict",
+                    "set",
+                    "super",
+                    "append",
+                    "fmt.Println",
+                    "console.log",
+                    "console.error",
+                    "require",
+                    "getattr",
+                    "setattr",
+                    "hasattr",
+                    "isinstance",
+                    "type",
+                }
+
+                if call_text and call_text not in ignore_list:
+                    # Signature Extraction: Pull the arguments as well
+                    args_node = node.child_by_field_name("arguments")
+                    if args_node:
+                        args_text = source[
+                            args_node.start_byte : args_node.end_byte
+                        ].decode("utf-8")
+                        # Sanitize whitespace/newlines
+                        args_text = " ".join(args_text.split())
+                        if len(args_text) < 60:
+                            call_text = f"{call_text}{args_text}"
+
+                    if len(call_text) < 120 and "\n" not in call_text:
+                        called.add(call_text)
+
+        for child in node.children:
+            self._traverse_for_calls(child, lang, source, called)
+
+    def _extract_call_chain(self, node: tree_sitter.Node, source: bytes) -> str:
+        """Recursively builds pure string representation of chained object method calls."""
+        if node.type == "identifier":
+            return source[node.start_byte : node.end_byte].decode("utf-8")
+        elif node.type == "attribute":
+            obj_node = node.child_by_field_name("object")
+            attr_node = node.child_by_field_name("attribute")
+            if obj_node and attr_node:
+                obj_str = self._extract_call_chain(obj_node, source)
+                attr_str = source[attr_node.start_byte : attr_node.end_byte].decode(
+                    "utf-8"
+                )
+                return f"{obj_str}.{attr_str}"
+        elif node.type == "call":
+            func_node = node.child_by_field_name("function")
+            if func_node:
+                return self._extract_call_chain(func_node, source)
+
+        return source[node.start_byte : node.end_byte].decode("utf-8").strip()

elastro_client-1.3.34/elastro/core/rag/ingestor.py

@@ -0,0 +1,189 @@
+"""
+Graph RAG orchestration module.
+Scans physical codebases, leverages the ASTParser to extract functional flow maps,
+and bulk indexes the enriched documents into Elasticsearch.
+Updated with Semantic Chunking and BM25 Support.
+"""
+
+import os
+import json
+import logging
+from typing import List, Dict, Any, Generator
+from elastro.core.client import ElasticsearchClient
+from elastro.core.rag.ast_parser import ASTParser
+
+logger = logging.getLogger("elastro.rag")
+
+
+class GraphRAGManager:
+    def __init__(
+        self, client: ElasticsearchClient, index_name: str = "fremen_codebase_rag"
+    ):
+        self.client = client
+        self.index_name = index_name
+        self.ast_parser = ASTParser()
+
+        # We explicitly target the agent's core polyglot languages
+        self.supported_extensions = {".py", ".go", ".vue", ".js", ".ts", ".jsx", ".tsx"}
+
+        # Prevent indexing binary data, large dependency folders, or generated outputs
+        self.ignore_dirs = {
+            ".git",
+            "node_modules",
+            "venv",
+            ".venv",
+            "env",
+            "__pycache__",
+            "dist",
+            "build",
+            ".next",
+            ".nuxt",
+            "coverage",
+            ".idea",
+            ".vscode",
+        }
+
+    def _should_ignore(self, path: str) -> bool:
+        """Determines if a directory or file should be skipped during ingestion."""
+        parts = os.path.normpath(path).split(os.sep)
+        for part in parts:
+            if part in self.ignore_dirs:
+                return True
+        return False
+
+    def scaffold_index(self) -> None:
+        """Safely creates the Graph RAG index if it does not manually exist."""
+        pipeline_deployed = False
+        # 1. Attempt to build the Dense Vector inference pipeline for Hybrid Search
+        try:
+            self.client.client.ingest.put_pipeline(
+                id="elastro-elser-v2",
+                description="Process text via ELSER for Dense Vector Hybrid Search",
+                processors=[
+                    {
+                        "inference": {
+                            "model_id": ".elser_model_2",
+                            "field_map": {"content": "text_field"},
+                            "target_field": "content_embedding",
+                            "inference_config": {
+                                "text_expansion": {"results_field": "tokens"}
+                            },
+                            "ignore_missing": True,
+                            "ignore_failure": True,
+                        }
+                    }
+                ],
+            )
+            logger.info(
+                "Successfully deployed ELSER inference pipeline framework (Enterprise Mode)."
+            )
+            pipeline_deployed = True
+        except Exception as e:
+            logger.warning(
+                f"Could not scaffold ELSER pipeline (requires ML nodes or Platinum/Enterprise license). Falling back to Open Source BM25 Mode. Error: {e}"
+            )
+
+        # Simple existence check without causing hard errors if it doesn't
+        if not self.client.client.indices.exists(index=self.index_name):
+            logger.info(f"Creating Graph RAG index '{self.index_name}'")
+
+            settings: Dict[str, Any] = {"number_of_shards": 1, "number_of_replicas": 0}
+            if pipeline_deployed:
+                settings["default_pipeline"] = "elastro-elser-v2"
+
+            mappings: Dict[str, Any] = {
+                "properties": {
+                    "repo_name": {"type": "keyword"},
+                    "file_path": {"type": "keyword"},
+                    "extension": {"type": "keyword"},
+                    "chunk_type": {"type": "keyword"},
+                    "chunk_name": {"type": "keyword"},
+                    "content": {"type": "text"},
+                    "functions_defined": {"type": "keyword"},
+                    "functions_called": {"type": "keyword"},
+                }
+            }
+            if pipeline_deployed:
+                mappings["properties"]["content_embedding"] = {"type": "sparse_vector"}
+
+            # We enforce standard RAG architecture with the Semantic Chunking fields.
+            self.client.client.indices.create(
+                index=self.index_name, body={"settings": settings, "mappings": mappings}
+            )
+
+    def scan_and_yield(self, repo_path: str) -> Generator[Dict[str, Any], None, None]:
+        """
+        Walks the repository sequentially. Applies Semantic Chunking and AST parsing.
+        Yields Elasticsearch Bulk API formatted operation dictionaries.
+        """
+        # Ensure we have absolute paths.
+        repo_path = os.path.abspath(repo_path)
+        repo_name = os.path.basename(repo_path)
+
+        for root, dirs, files in os.walk(repo_path):
+            # Prune ignored directories in-place to avoid deep pointless traversals
+            dirs[:] = [
+                d for d in dirs if not self._should_ignore(os.path.join(root, d))
+            ]
+
+            for file in files:
+                file_path = os.path.join(root, file)
+
+                # Check explicit supported AST languages or ignore check
+                ext = os.path.splitext(file)[1].lower()
+                if ext not in self.supported_extensions or self._should_ignore(
+                    file_path
+                ):
+                    continue
+
+                try:
+                    with open(file_path, "r", encoding="utf-8") as f:
+                        content = f.read()
+
+                    # 1. Engage the AST parser to extract the "Graph" semantic chunks
+                    chunks = self.ast_parser.parse_file(file_path, content)
+                    rel_path = os.path.relpath(file_path, repo_path)
+
+                    # 2. Yield the Bulk API _index action for each specific chunk
+                    for i, chunk in enumerate(chunks):
+                        doc_id = f"{repo_name}:{rel_path}::{i}"
+
+                        yield {
+                            "_op_type": "index",
+                            "_index": self.index_name,
+                            "_id": doc_id,
+                            "_source": {
+                                "repo_name": repo_name,
+                                "file_path": rel_path,
+                                "extension": ext,
+                                "chunk_type": chunk.get("chunk_type", "file"),
+                                "chunk_name": chunk.get("name", "module"),
+                                "content": chunk.get("content", content),
+                                "functions_defined": chunk.get("functions_defined", []),
+                                "functions_called": chunk.get("functions_called", []),
+                            },
+                        }
+                except UnicodeDecodeError:
+                    # Skip binary files that masquerade as text extensions
+                    continue
+                except Exception as e:
+                    logger.error(f"Error parsing Graph RAG for {file_path}: {e}")
+
+    def ingest_repository(self, repo_path: str) -> int:
+        """
+        Orchestrates Elasticsearch bulk insertion directly leveraging the client helpers.
+        """
+        from elasticsearch.helpers import bulk
+
+        # Fire up the index blueprint
+        self.scaffold_index()
+
+        # Extract AST Flows & Stream seamlessly to Elastic
+        success_count, failed_inserts = bulk(
+            self.client.client,
+            self.scan_and_yield(repo_path),
+            chunk_size=500,  # Large batch sizes for fast performance on local M4 Mini
+            stats_only=True,
+        )
+
+        return success_count

{elastro_client-1.3.32 → elastro_client-1.3.34/elastro_client.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: elastro-client
-Version: 1.3.32
+Version: 1.3.34
 Summary: A comprehensive Python library for Elasticsearch management with both programmatic and CLI interfaces
 Author: Austin Jorgensen
 License-Expression: MIT
@@ -29,6 +29,11 @@ Requires-Dist: rich>=10.0.0
 Requires-Dist: rich-click>=1.7.0
 Requires-Dist: fastapi>=0.111.0
 Requires-Dist: uvicorn>=0.30.0
+Requires-Dist: tree-sitter>=0.23.0
+Requires-Dist: tree-sitter-python>=0.23.0
+Requires-Dist: tree-sitter-go>=0.23.0
+Requires-Dist: tree-sitter-javascript>=0.23.0
+Requires-Dist: tree-sitter-typescript>=0.23.0
 Provides-Extra: test
 Requires-Dist: pytest>=7.0.0; extra == "test"
 Requires-Dist: pytest-cov>=3.0.0; extra == "test"

{elastro_client-1.3.32 → elastro_client-1.3.34}/elastro_client.egg-info/SOURCES.txt

@@ -37,6 +37,7 @@ elastro/cli/commands/index_recipes.py
 elastro/cli/commands/ingest.py
 elastro/cli/commands/ml.py
 elastro/cli/commands/painless_commands.py
+elastro/cli/commands/rag.py
 elastro/cli/commands/script.py
 elastro/cli/commands/security.py
 elastro/cli/commands/snapshot.py
@@ -58,6 +59,8 @@ elastro/core/logger.py
 elastro/core/query_builder.py
 elastro/core/snapshot.py
 elastro/core/validation.py
+elastro/core/rag/ast_parser.py
+elastro/core/rag/ingestor.py
 elastro/gui/elastro.svg
 elastro/gui/favicon.ico
 elastro/gui/index.html

{elastro_client-1.3.32 → elastro_client-1.3.34}/elastro_client.egg-info/requires.txt

@@ -8,6 +8,11 @@ rich>=10.0.0
 rich-click>=1.7.0
 fastapi>=0.111.0
 uvicorn>=0.30.0
+tree-sitter>=0.23.0
+tree-sitter-python>=0.23.0
+tree-sitter-go>=0.23.0
+tree-sitter-javascript>=0.23.0
+tree-sitter-typescript>=0.23.0
 
 [dev]
 pytest>=7.0.0

{elastro_client-1.3.32 → elastro_client-1.3.34}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "elastro-client"
-version = "1.3.32"
+version = "1.3.34"
 description = "A comprehensive Python library for Elasticsearch management with both programmatic and CLI interfaces"
 readme = "README.md"
 requires-python = ">=3.9,<3.14"
@@ -36,6 +36,11 @@ dependencies = [
     "rich-click>=1.7.0",
     "fastapi>=0.111.0",
     "uvicorn>=0.30.0",
+    "tree-sitter>=0.23.0",
+    "tree-sitter-python>=0.23.0",
+    "tree-sitter-go>=0.23.0",
+    "tree-sitter-javascript>=0.23.0",
+    "tree-sitter-typescript>=0.23.0",
 ]
 
 [project.optional-dependencies]