memra 0.2.2__py3-none-any.whl → 0.2.4__py3-none-any.whl

memra-ops/server_tool_registry.py

@@ -0,0 +1,188 @@
+import importlib
+import logging
+import sys
+import os
+from typing import Dict, Any, List, Optional, Callable
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+class ServerToolRegistry:
+    """Server-side registry for managing and executing tools from logic directory"""
+    
+    def __init__(self):
+        self.tools: Dict[str, Dict[str, Any]] = {}
+        self._add_project_to_path()
+        self._load_builtin_tools()
+    
+    def _add_project_to_path(self):
+        """Add the project root to Python path so we can import logic modules"""
+        # Get the directory containing this file (project root)
+        current_dir = Path(__file__).parent
+        
+        if str(current_dir) not in sys.path:
+            sys.path.insert(0, str(current_dir))
+    
+    def _load_builtin_tools(self):
+        """Load tools from the logic directory"""
+        try:
+            # Load invoice tools
+            from logic.invoice_tools import (
+                DatabaseQueryTool, PDFProcessor, OCRTool, 
+                InvoiceExtractionWorkflow, DataValidator, PostgresInsert
+            )
+            
+            self.register_tool("DatabaseQueryTool", DatabaseQueryTool, "memra", 
+                             "Query database schemas and data")
+            self.register_tool("PDFProcessor", PDFProcessor, "memra", 
+                             "Process PDF files and extract content")
+            self.register_tool("OCRTool", OCRTool, "memra", 
+                             "Perform OCR on images and documents")
+            self.register_tool("InvoiceExtractionWorkflow", InvoiceExtractionWorkflow, "memra", 
+                             "Extract structured data from invoices")
+            self.register_tool("DataValidator", DataValidator, "memra", 
+                             "Validate data against schemas")
+            self.register_tool("PostgresInsert", PostgresInsert, "memra", 
+                             "Insert data into PostgreSQL database")
+            
+            # Load file tools
+            from logic.file_tools import FileReader
+            self.register_tool("FileReader", FileReader, "memra", 
+                             "Read files from the filesystem")
+            
+            logger.info(f"Loaded {len(self.tools)} builtin tools")
+            
+        except ImportError as e:
+            logger.warning(f"Could not load some tools: {e}")
+    
+    def register_tool(self, name: str, tool_class: type, hosted_by: str, description: str):
+        """Register a tool in the registry"""
+        self.tools[name] = {
+            "class": tool_class,
+            "hosted_by": hosted_by,
+            "description": description
+        }
+        logger.debug(f"Registered tool: {name} (hosted by {hosted_by})")
+    
+    def discover_tools(self, hosted_by: Optional[str] = None) -> List[Dict[str, Any]]:
+        """Discover available tools, optionally filtered by host"""
+        tools = []
+        for name, info in self.tools.items():
+            if hosted_by is None or info["hosted_by"] == hosted_by:
+                tools.append({
+                    "name": name,
+                    "hosted_by": info["hosted_by"],
+                    "description": info["description"]
+                })
+        return tools
+    
+    def execute_tool(self, tool_name: str, hosted_by: str, input_data: Dict[str, Any], 
+                    config: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
+        """Execute a tool with the given input data"""
+        if tool_name not in self.tools:
+            return {
+                "success": False,
+                "error": f"Tool '{tool_name}' not found in registry"
+            }
+        
+        tool_info = self.tools[tool_name]
+        if tool_info["hosted_by"] != hosted_by:
+            return {
+                "success": False,
+                "error": f"Tool '{tool_name}' is hosted by '{tool_info['hosted_by']}', not '{hosted_by}'"
+            }
+        
+        try:
+            # Instantiate tool
+            tool_class = tool_info["class"]
+            
+            # Some tools need credentials/config for initialization
+            if tool_name in ["DatabaseQueryTool", "PostgresInsert"]:
+                if "connection" in input_data:
+                    # Parse connection string or use credentials
+                    credentials = self._parse_connection(input_data["connection"])
+                    tool_instance = tool_class(credentials)
+                else:
+                    return {
+                        "success": False,
+                        "error": f"Tool '{tool_name}' requires database credentials"
+                    }
+            elif tool_name == "InvoiceExtractionWorkflow":
+                # This tool needs to be instantiated to initialize the LLM client
+                tool_instance = tool_class()
+            else:
+                tool_instance = tool_class()
+            
+            # Execute tool based on its type
+            result = self._execute_tool_method(tool_instance, tool_name, input_data, config)
+            
+            return {
+                "success": True,
+                "data": result
+            }
+            
+        except Exception as e:
+            logger.error(f"Tool execution failed for {tool_name}: {str(e)}")
+            return {
+                "success": False,
+                "error": str(e)
+            }
+    
+    def _execute_tool_method(self, tool_instance: Any, tool_name: str, 
+                           input_data: Dict[str, Any], config: Optional[Dict[str, Any]]) -> Dict[str, Any]:
+        """Execute the appropriate method on the tool instance"""
+        
+        if tool_name == "DatabaseQueryTool":
+            return tool_instance.get_schema("invoices")  # Default to invoices table
+        
+        elif tool_name == "PDFProcessor":
+            file_path = input_data.get("file", "")
+            return tool_instance.process_pdf(file_path)
+        
+        elif tool_name == "OCRTool":
+            # Assume PDF processor output is passed as input
+            return {"extracted_text": tool_instance.extract_text(input_data)}
+        
+        elif tool_name == "InvoiceExtractionWorkflow":
+            text = input_data.get("extracted_text", "")
+            schema = input_data.get("invoice_schema", {})
+            return tool_instance.extract_data(text, schema)
+        
+        elif tool_name == "DataValidator":
+            data = input_data.get("invoice_data", {})
+            schema = input_data.get("invoice_schema", {})
+            return tool_instance.validate(data, schema)
+        
+        elif tool_name == "PostgresInsert":
+            data = input_data.get("invoice_data", {})
+            return tool_instance.insert_record("invoices", data)
+        
+        elif tool_name == "FileReader":
+            file_path = config.get("path") if config else input_data.get("file_path")
+            if not file_path:
+                raise ValueError("FileReader requires a file path")
+            return tool_instance.read_file(file_path)
+        
+        else:
+            raise ValueError(f"Unknown tool execution method for {tool_name}")
+    
+    def _parse_connection(self, connection_string: str) -> Dict[str, Any]:
+        """Parse a connection string into credentials"""
+        # Simple parser for postgres://user:pass@host:port/database
+        if connection_string.startswith("postgres://"):
+            # This is a simplified parser - in production you'd use a proper URL parser
+            parts = connection_string.replace("postgres://", "").split("/")
+            db_part = parts[1] if len(parts) > 1 else "finance"
+            auth_host = parts[0].split("@")
+            host_port = auth_host[1].split(":") if len(auth_host) > 1 else ["localhost", "5432"]
+            user_pass = auth_host[0].split(":") if len(auth_host) > 1 else ["user", "pass"]
+            
+            return {
+                "host": host_port[0],
+                "port": int(host_port[1]) if len(host_port) > 1 else 5432,
+                "database": db_part,
+                "user": user_pass[0],
+                "password": user_pass[1] if len(user_pass) > 1 else ""
+            }
+        
+        return {"connection_string": connection_string} 

memra-ops/tests/test_llm_text_to_sql.py

@@ -0,0 +1,115 @@
+#!/usr/bin/env python3
+"""
+Test script to verify LLM-based text-to-SQL generation
+"""
+
+import requests
+import json
+import time
+
+def test_llm_text_to_sql():
+    """Test the new LLM-based text-to-SQL generation"""
+    bridge_url = "http://localhost:8081"
+    bridge_secret = "test-secret-for-development"
+    
+    # Test questions to verify LLM vs pattern matching
+    test_questions = [
+        "Show me all invoices from Air Liquide",
+        "What is the total amount of all invoices?",
+        "How many invoices do we have?",
+        "Show me the 3 most recent invoices",
+        "What is the average invoice amount?",
+        "Find invoices with amounts greater than 1000",  # This should test LLM capabilities
+        "Show me invoices from last month",  # This should test LLM capabilities
+        "Which vendor has the highest total invoice amount?"  # Complex query for LLM
+    ]
+    
+    # Schema info for context
+    schema_info = {
+        "schema": {
+            "invoices": {
+                "columns": [
+                    {"name": "id", "type": "integer"},
+                    {"name": "vendor_name", "type": "text"},
+                    {"name": "invoice_number", "type": "text"},
+                    {"name": "invoice_date", "type": "date"},
+                    {"name": "total_amount", "type": "numeric"},
+                    {"name": "tax_amount", "type": "numeric"},
+                    {"name": "line_items", "type": "jsonb"},
+                    {"name": "status", "type": "text"}
+                ]
+            }
+        }
+    }
+    
+    headers = {
+        "Content-Type": "application/json",
+        "X-Bridge-Secret": bridge_secret
+    }
+    
+    print("🧪 Testing LLM-based Text-to-SQL Generation")
+    print("=" * 60)
+    
+    for i, question in enumerate(test_questions, 1):
+        print(f"\n🎯 Test {i}: {question}")
+        print("-" * 50)
+        
+        # Prepare request
+        request_data = {
+            "tool_name": "TextToSQLGenerator",
+            "input_data": {
+                "question": question,
+                "schema_info": schema_info
+            }
+        }
+        
+        try:
+            start_time = time.time()
+            
+            # Make request to MCP bridge
+            response = requests.post(
+                f"{bridge_url}/execute_tool",
+                json=request_data,
+                headers=headers,
+                timeout=30  # Longer timeout for LLM calls
+            )
+            
+            end_time = time.time()
+            duration = end_time - start_time
+            
+            if response.status_code == 200:
+                result = response.json()
+                
+                if result.get("success"):
+                    data = result.get("data", {})
+                    sql_query = data.get("generated_sql", "")
+                    explanation = data.get("explanation", "")
+                    confidence = data.get("confidence", "unknown")
+                    method = data.get("method", "unknown")
+                    
+                    print(f"✅ Success ({duration:.1f}s)")
+                    print(f"📝 SQL: {sql_query}")
+                    print(f"💡 Method: {method}")
+                    print(f"🎯 Confidence: {confidence}")
+                    print(f"📖 Explanation: {explanation}")
+                    
+                    # Highlight if this is using LLM
+                    if method == "llm":
+                        print("🚀 Using LLM generation!")
+                    elif method == "pattern_matching":
+                        print("⚠️  Fallback to pattern matching")
+                    
+                else:
+                    print(f"❌ Failed: {result.get('error', 'Unknown error')}")
+                    
+            else:
+                print(f"❌ HTTP Error: {response.status_code}")
+                print(f"Response: {response.text}")
+                
+        except Exception as e:
+            print(f"❌ Exception: {str(e)}")
+    
+    print(f"\n✨ LLM Text-to-SQL testing completed!")
+
+if __name__ == "__main__":
+    test_llm_text_to_sql() 

memra-ops/tests/test_llm_vs_pattern.py

@@ -0,0 +1,130 @@
+#!/usr/bin/env python3
+"""
+Test script to demonstrate LLM vs Pattern Matching for text-to-SQL
+"""
+
+import requests
+import json
+import time
+
+def test_query(question, description=""):
+    """Test a single query and return results"""
+    bridge_url = "http://localhost:8081"
+    bridge_secret = "test-secret-for-development"
+    
+    schema_info = {
+        "schema": {
+            "invoices": {
+                "columns": [
+                    {"name": "id", "type": "integer"},
+                    {"name": "vendor_name", "type": "text"},
+                    {"name": "invoice_number", "type": "text"},
+                    {"name": "invoice_date", "type": "date"},
+                    {"name": "total_amount", "type": "numeric"},
+                    {"name": "tax_amount", "type": "numeric"},
+                    {"name": "line_items", "type": "jsonb"},
+                    {"name": "status", "type": "text"}
+                ]
+            }
+        }
+    }
+    
+    headers = {
+        "Content-Type": "application/json",
+        "X-Bridge-Secret": bridge_secret
+    }
+    
+    request_data = {
+        "tool_name": "TextToSQLGenerator",
+        "input_data": {
+            "question": question,
+            "schema_info": schema_info
+        }
+    }
+    
+    print(f"\n🎯 {description}")
+    print(f"❓ Question: {question}")
+    print("-" * 60)
+    
+    try:
+        start_time = time.time()
+        
+        response = requests.post(
+            f"{bridge_url}/execute_tool",
+            json=request_data,
+            headers=headers,
+            timeout=30
+        )
+        
+        end_time = time.time()
+        duration = end_time - start_time
+        
+        if response.status_code == 200:
+            result = response.json()
+            
+            if result.get("success"):
+                data = result.get("data", {})
+                sql_query = data.get("generated_sql", "")
+                method = data.get("method", "unknown")
+                confidence = data.get("confidence", "unknown")
+                
+                print(f"✅ Success ({duration:.1f}s)")
+                print(f"📝 SQL: {sql_query}")
+                print(f"💡 Method: {method}")
+                print(f"🎯 Confidence: {confidence}")
+                
+                if method == "llm":
+                    print("🚀 LLM Generation - Advanced natural language understanding!")
+                elif method == "pattern_matching":
+                    print("⚙️  Pattern Matching - Rule-based fallback")
+                else:
+                    print("❓ Unknown method")
+                
+                return True
+            else:
+                print(f"❌ Failed: {result.get('error', 'Unknown error')}")
+                return False
+        else:
+            print(f"❌ HTTP Error: {response.status_code}")
+            return False
+            
+    except Exception as e:
+        print(f"❌ Exception: {str(e)}")
+        return False
+
+def main():
+    """Test various queries to demonstrate LLM capabilities"""
+    print("🧪 LLM vs Pattern Matching Text-to-SQL Comparison")
+    print("=" * 70)
+    
+    # Test cases that should showcase LLM capabilities
+    test_cases = [
+        ("Show me all invoices from Air Liquide", "Simple vendor filtering (both methods can handle)"),
+        ("Find invoices with amounts greater than 1000", "Numeric comparison (LLM advantage)"),
+        ("Show me invoices from last month", "Date range query (LLM advantage)"),
+        ("Which vendor has the highest total invoice amount?", "Complex aggregation with grouping (LLM advantage)"),
+        ("Find invoices where the tax amount is more than 10% of total", "Complex calculation (LLM advantage)"),
+        ("Show me all pending invoices sorted by amount", "Status filtering with sorting (LLM advantage)"),
+        ("What is the average invoice amount?", "Simple aggregation (both methods can handle)"),
+        ("List all unique vendors", "Distinct query (both methods can handle)"),
+        ("Find invoices with line items containing 'software'", "JSON field querying (LLM advantage)"),
+        ("Show me the top 5 vendors by total invoice value", "Complex grouping and ranking (LLM advantage)")
+    ]
+    
+    success_count = 0
+    llm_count = 0
+    pattern_count = 0
+    
+    for question, description in test_cases:
+        if test_query(question, description):
+            success_count += 1
+            # Note: We'd need to parse the response to count methods accurately
+    
+    print(f"\n✨ Testing completed!")
+    print(f"📊 {success_count}/{len(test_cases)} queries successful")
+    print(f"\n🎉 The LLM-based text-to-SQL system is now active!")
+    print(f"🔄 It automatically falls back to pattern matching if LLM fails")
+    print(f"🚀 Complex queries now benefit from advanced natural language understanding")
+
+if __name__ == "__main__":
+    main() 

memra-ops/tests/test_mcp_schema_aware.py

@@ -0,0 +1,124 @@
+#!/usr/bin/env python3
+"""
+Direct test of MCP bridge server schema-aware SQL generation
+"""
+
+import requests
+import json
+
+def test_schema_aware_sql():
+    """Test schema-aware SQL generation directly via MCP bridge"""
+    print("🧪 Testing Schema-Aware SQL Generation (Direct MCP)")
+    print("=" * 55)
+    
+    bridge_url = "http://localhost:8081"
+    bridge_secret = "test-secret-for-development"
+    
+    # First, get the schema information (simulated)
+    schema_info = {
+        "schema": {
+            "invoices": {
+                "columns": [
+                    {"name": "id", "type": "integer"},
+                    {"name": "vendor_name", "type": "text"},
+                    {"name": "invoice_number", "type": "text"},
+                    {"name": "invoice_date", "type": "date"},
+                    {"name": "total_amount", "type": "numeric"},
+                    {"name": "line_items", "type": "jsonb"}
+                ]
+            }
+        }
+    }
+    
+    # Test questions
+    test_questions = [
+        "Show me all invoices from Air Liquide",
+        "What is the total amount of all invoices?",
+        "How many invoices do we have?",
+        "Show me the 3 most recent invoices",
+        "What is the average invoice amount?",
+        "Count invoices from Microsoft"
+    ]
+    
+    print(f"\n🔍 Testing {len(test_questions)} questions with schema context...")
+    
+    for i, question in enumerate(test_questions, 1):
+        print(f"\n--- Test {i}: {question} ---")
+        
+        # Prepare request data
+        request_data = {
+            "tool_name": "TextToSQLGenerator",
+            "input_data": {
+                "question": question,
+                "schema_info": schema_info
+            }
+        }
+        
+        headers = {
+            "Content-Type": "application/json",
+            "X-Bridge-Secret": bridge_secret
+        }
+        
+        try:
+            # Make request to MCP bridge
+            response = requests.post(
+                f"{bridge_url}/execute_tool",
+                json=request_data,
+                headers=headers,
+                timeout=10
+            )
+            
+            if response.status_code == 200:
+                result = response.json()
+                
+                if result.get("success"):
+                    data = result.get("data", {})
+                    generated_sql = data.get("generated_sql", "")
+                    explanation = data.get("explanation", "")
+                    confidence = data.get("confidence", "unknown")
+                    schema_used = data.get("schema_used", {})
+                    
+                    print(f"✅ SQL Generated (confidence: {confidence})")
+                    print(f"   Query: {generated_sql}")
+                    print(f"   Explanation: {explanation}")
+                    
+                    if schema_used:
+                        columns = schema_used.get("columns", [])
+                        print(f"   Schema columns used: {', '.join(columns)}")
+                    
+                    # Check if SQL uses actual column names
+                    schema_columns = ["vendor_name", "total_amount", "invoice_date", "invoice_number"]
+                    uses_schema = any(col in generated_sql for col in schema_columns)
+                    
+                    if uses_schema:
+                        print("🎯 SQL appears to use actual schema column names!")
+                    else:
+                        print("⚠️  SQL might not be using schema information")
+                        
+                else:
+                    print(f"❌ SQL generation failed: {result.get('error')}")
+            else:
+                print(f"❌ HTTP error: {response.status_code}")
+                print(f"   Response: {response.text}")
+                
+        except Exception as e:
+            print(f"❌ Request failed: {str(e)}")
+    
+    print(f"\n✨ Schema-aware SQL generation test completed!")
+
+def main():
+    """Main function"""
+    try:
+        # Check if MCP bridge is running
+        response = requests.get("http://localhost:8081/health", timeout=5)
+        if response.status_code == 200:
+            print("✅ MCP Bridge server is running")
+            test_schema_aware_sql()
+        else:
+            print("❌ MCP Bridge server is not responding")
+    except Exception as e:
+        print(f"❌ Cannot connect to MCP Bridge server: {str(e)}")
+        print("💡 Make sure the MCP bridge server is running on port 8081")
+
+if __name__ == "__main__":
+    main()