memra 0.2.11__py3-none-any.whl → 0.2.13__py3-none-any.whl

memra/demos/etl_invoice_processing/test_postgres_insert.py

@@ -0,0 +1,120 @@
+#!/usr/bin/env python3
+"""
+Test PostgresInsert tool with new invoice_json table
+"""
+
+import os
+import sys
+import json
+import requests
+
+# Add the parent directory to the path to import memra
+sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..'))
+
+from memra.tool_registry import ToolRegistry
+
+def test_postgres_insert():
+    """Test PostgresInsert tool with sample invoice data"""
+    
+    # Sample invoice data that includes due date
+    sample_invoice_data = {
+        "headerSection": {
+            "vendorName": "Test Vendor Inc.",
+            "subtotal": 1500.00
+        },
+        "billingDetails": {
+            "invoiceNumber": "TEST-001",
+            "invoiceDate": "2024-12-01",
+            "dueDate": "2024-12-31"  # This was previously lost!
+        },
+        "chargesSummary": {
+            "document_total": 1695.00,
+            "secondary_tax": 195.00,
+            "lineItemsBreakdown": [
+                {
+                    "description": "Test Service",
+                    "quantity": 1,
+                    "unit_price": 1500.00,
+                    "amount": 1500.00
+                }
+            ]
+        }
+    }
+    
+    print("🔧 Testing PostgresInsert tool with new invoice_json table...")
+    print(f"📄 Sample data includes due date: {sample_invoice_data['billingDetails']['dueDate']}")
+    
+    # Test input data
+    input_data = {
+        "invoice_data": sample_invoice_data,
+        "table_name": "invoice_json"
+    }
+    
+    # Execute the tool
+    try:
+        registry = ToolRegistry()
+        result = registry.execute_tool(
+            tool_name="PostgresInsert",
+            hosted_by="mcp",
+            input_data=input_data,
+            config={
+                "bridge_url": "http://localhost:8081",
+                "bridge_secret": "test-secret-for-development"
+            }
+        )
+        
+        print(f"📊 Result: {json.dumps(result, indent=2)}")
+        
+        if result.get("success"):
+            print("✅ PostgresInsert executed successfully!")
+            print(f"📝 Record ID: {result['data']['record_id']}")
+            print(f"📊 JSON size: {result['data']['inserted_data']['raw_json_size']} bytes")
+            print(f"🔑 JSON keys: {result['data']['inserted_data']['json_keys']}")
+            
+            # Now let's verify the data was stored correctly
+            print("\n🔍 Verifying stored data...")
+            verify_stored_data(result['data']['record_id'])
+            
+        else:
+            print(f"❌ PostgresInsert failed: {result.get('error')}")
+            
+    except Exception as e:
+        print(f"❌ Error executing PostgresInsert: {str(e)}")
+
+def verify_stored_data(record_id):
+    """Verify that the data was stored correctly in the database"""
+    
+    # Query to get the stored JSON
+    query = f"SELECT raw_json FROM invoice_json WHERE id = {record_id}"
+    
+    try:
+        registry = ToolRegistry()
+        result = registry.execute_tool(
+            tool_name="SQLExecutor",
+            hosted_by="mcp",
+            input_data={"sql_query": query},
+            config={
+                "bridge_url": "http://localhost:8081",
+                "bridge_secret": "test-secret-for-development"
+            }
+        )
+        
+        if result.get("success") and result["data"]["results"]:
+            stored_json = result["data"]["results"][0]["raw_json"]
+            print(f"📄 Stored JSON: {json.dumps(stored_json, indent=2)}")
+            
+            # Check if due date is preserved
+            due_date = stored_json.get("billingDetails", {}).get("dueDate")
+            if due_date:
+                print(f"✅ Due date preserved: {due_date}")
+            else:
+                print("❌ Due date not found in stored data")
+                
+        else:
+            print(f"❌ Failed to retrieve stored data: {result.get('error')}")
+            
+    except Exception as e:
+        print(f"❌ Error verifying stored data: {str(e)}")
+
+if __name__ == "__main__":
+    test_postgres_insert() 

memra/demos/etl_invoice_processing/test_remote_upload.py

@@ -0,0 +1,143 @@
+#!/usr/bin/env python3
+"""
+Test script to verify file upload functionality with remote API
+"""
+
+import os
+import base64
+import requests
+import json
+
+def test_remote_upload():
+    """Test the complete upload and processing workflow with remote API"""
+    
+    # Remote API URL
+    API_BASE = "https://api.memra.co"
+    API_KEY = "test-secret-for-development"
+    
+    print("🧪 Testing Remote API File Upload Functionality")
+    print("=" * 50)
+    
+    # Step 1: Check server health
+    print("\n1️⃣ Checking server health...")
+    try:
+        response = requests.get(f"{API_BASE}/health")
+        if response.status_code == 200:
+            print("✅ Server is healthy")
+        else:
+            print(f"❌ Server health check failed: {response.status_code}")
+            return
+    except Exception as e:
+        print(f"❌ Cannot connect to server: {e}")
+        return
+    
+    # Step 2: Discover available tools
+    print("\n2️⃣ Discovering available tools...")
+    try:
+        response = requests.get(f"{API_BASE}/tools/discover", headers={"X-API-Key": API_KEY})
+        if response.status_code == 200:
+            tools = response.json().get("tools", [])
+            print(f"✅ Found {len(tools)} tools:")
+            for tool in tools:
+                print(f"   - {tool['name']}: {tool['description']}")
+        else:
+            print(f"❌ Tool discovery failed: {response.status_code}")
+            print(f"   Response: {response.text}")
+            return
+    except Exception as e:
+        print(f"❌ Tool discovery error: {e}")
+        return
+    
+    # Step 3: Test with a real PDF file
+    print("\n3️⃣ Testing with real PDF file...")
+    pdf_path = "data/invoices/10352260169.PDF"
+    
+    if not os.path.exists(pdf_path):
+        print(f"❌ PDF file not found: {pdf_path}")
+        return
+    
+    print(f"✅ Found PDF file: {pdf_path}")
+    
+    # Step 4: Upload the file
+    print("\n4️⃣ Uploading file to remote API...")
+    try:
+        with open(pdf_path, 'rb') as f:
+            file_content = f.read()
+        
+        file_b64 = base64.b64encode(file_content).decode('utf-8')
+        
+        upload_data = {
+            "filename": os.path.basename(pdf_path),
+            "content": file_b64,
+            "content_type": "application/pdf"
+        }
+        
+        response = requests.post(
+            f"{API_BASE}/upload",
+            json=upload_data,
+            headers={
+                "X-API-Key": API_KEY,
+                "Content-Type": "application/json"
+            }
+        )
+        
+        print(f"Upload response status: {response.status_code}")
+        print(f"Upload response: {response.text}")
+        
+        if response.status_code == 200:
+            result = response.json()
+            if result.get("success"):
+                remote_path = result["data"]["remote_path"]
+                print(f"✅ File uploaded successfully")
+                print(f"   Remote path: {remote_path}")
+                print(f"   File ID: {result['data']['file_id']}")
+                
+                # Step 5: Process the uploaded file
+                print("\n5️⃣ Processing uploaded file...")
+                process_data = {
+                    "tool_name": "PDFProcessor",
+                    "hosted_by": "memra",
+                    "input_data": {
+                        "file": remote_path
+                    }
+                }
+                
+                response = requests.post(
+                    f"{API_BASE}/tools/execute",
+                    json=process_data,
+                    headers={
+                        "X-API-Key": API_KEY,
+                        "Content-Type": "application/json"
+                    }
+                )
+                
+                print(f"Processing response status: {response.status_code}")
+                print(f"Processing response: {response.text}")
+                
+                if response.status_code == 200:
+                    result = response.json()
+                    if result.get("success") and result.get("data", {}).get("success"):
+                        extracted_data = result["data"]["data"].get("extracted_data", {})
+                        print("✅ File processed successfully")
+                        print("📄 Extracted data:")
+                        print(f"   Vendor: {extracted_data.get('headerSection', {}).get('vendorName', 'N/A')}")
+                        print(f"   Invoice: {extracted_data.get('billingDetails', {}).get('invoiceNumber', 'N/A')}")
+                        print(f"   Amount: ${extracted_data.get('chargesSummary', {}).get('document_total', 'N/A')}")
+                    else:
+                        print(f"❌ Processing failed: {result.get('data', {}).get('error', 'Unknown error')}")
+                else:
+                    print(f"❌ Processing request failed: {response.status_code}")
+                    print(f"   Response: {response.text}")
+            else:
+                print(f"❌ Upload failed: {result.get('error')}")
+        else:
+            print(f"❌ Upload request failed: {response.status_code}")
+            print(f"   Response: {response.text}")
+            
+    except Exception as e:
+        print(f"❌ Upload/Processing error: {e}")
+    
+    print("\n🎉 Test completed!")
+
+if __name__ == "__main__":
+    test_remote_upload() 

memra/demos/etl_invoice_processing/test_schema_format.py

@@ -0,0 +1,39 @@
+import os
+import requests
+import json
+
+api_url = "https://api.memra.co"
+api_key = os.getenv("MEMRA_API_KEY", "test-secret-for-development")
+
+# Test with a more detailed schema format
+schema = {
+    "fields": [
+        {"name": "invoice_number", "type": "string", "required": True},
+        {"name": "vendor_name", "type": "string", "required": True},
+        {"name": "invoice_date", "type": "date", "required": True},
+        {"name": "total_amount", "type": "numeric", "required": True}
+    ]
+}
+
+print("Testing PDFProcessor with schema:")
+print(json.dumps(schema, indent=2))
+
+resp = requests.post(
+    f"{api_url}/tools/execute",
+    json={
+        "tool_name": "PDFProcessor",
+        "hosted_by": "memra",
+        "input_data": {
+            "file": "/uploads/6f4538c0-8fce-4488-be49-1a78afc58a4a.pdf",
+            "schema": schema
+        }
+    },
+    headers={"X-API-Key": api_key}
+)
+
+if resp.status_code == 200:
+    result = resp.json()
+    if result.get('success') and 'data' in result and 'data' in result['data']:
+        prompt = result['data']['data'].get('vision_prompt', '')
+        print("\nVision prompt generated:")
+        print(prompt[:500])

memra/demos/etl_invoice_processing/test_sql_executor.py

@@ -0,0 +1,58 @@
+#!/usr/bin/env python3
+"""
+Test SQLExecutor tool execution directly
+"""
+
+import sys
+import os
+sys.path.insert(0, '/Users/tarpus/memra')
+
+from memra.tool_registry import ToolRegistry
+import logging
+
+# Set up logging
+logging.basicConfig(level=logging.INFO, format='%(levelname)s:%(name)s:%(message)s')
+logger = logging.getLogger(__name__)
+
+def test_sql_executor():
+    """Test SQLExecutor tool execution"""
+    
+    # Create tool registry
+    registry = ToolRegistry()
+    
+    # Test configuration
+    config = {
+        "bridge_url": "http://localhost:8081",
+        "bridge_secret": "test-secret-for-development"
+    }
+    
+    # Test input data
+    input_data = {
+        "sql_query": "SELECT COUNT(*) as row_count FROM invoices"
+    }
+    
+    print("🔧 Testing SQLExecutor tool execution...")
+    print(f"Config: {config}")
+    print(f"Input: {input_data}")
+    
+    # Execute the tool
+    result = registry.execute_tool(
+        tool_name="SQLExecutor",
+        hosted_by="mcp",
+        input_data=input_data,
+        config=config
+    )
+    
+    print(f"\n📊 Result: {result}")
+    
+    if result.get("success"):
+        print("✅ SQLExecutor executed successfully!")
+        if "_mock" in result.get("data", {}):
+            print("⚠️  But returned mock data")
+        else:
+            print("🎉 Real data returned!")
+    else:
+        print(f"❌ SQLExecutor failed: {result.get('error')}")
+
+if __name__ == "__main__":
+    test_sql_executor() 

memra/demos/etl_invoice_processing/test_sql_executor_extra_fields.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python3
+"""
+Test SQLExecutor tool execution with extra fields (like agent_input)
+"""
+
+import sys
+import os
+sys.path.insert(0, '/Users/tarpus/memra')
+
+from memra.tool_registry import ToolRegistry
+import logging
+
+# Set up logging
+logging.basicConfig(level=logging.INFO, format='%(levelname)s:%(name)s:%(message)s')
+logger = logging.getLogger(__name__)
+
+def test_sql_executor_with_extra_fields():
+    """Test SQLExecutor tool execution with extra fields like agent_input"""
+    
+    # Create tool registry
+    registry = ToolRegistry()
+    
+    # Test configuration
+    config = {
+        "bridge_url": "http://localhost:8081",
+        "bridge_secret": "test-secret-for-development"
+    }
+    
+    # Test input data with extra fields (like agent_input)
+    input_data = {
+        "sql_query": "SELECT COUNT(*) as row_count FROM invoices",
+        "table_name": "invoices",
+        "connection": "postgresql://memra:memra123@localhost:5432/memra_invoice_db",
+        "monitoring_phase": "before"
+    }
+    
+    print("🔧 Testing SQLExecutor tool execution with extra fields...")
+    print(f"Config: {config}")
+    print(f"Input: {input_data}")
+    
+    # Execute the tool
+    result = registry.execute_tool(
+        tool_name="SQLExecutor",
+        hosted_by="mcp",
+        input_data=input_data,
+        config=config
+    )
+    
+    print(f"\n📊 Result: {result}")
+    
+    if result.get("success"):
+        print("✅ SQLExecutor executed successfully!")
+        if "_mock" in result.get("data", {}):
+            print("⚠️  But returned mock data")
+        else:
+            print("🎉 Real data returned!")
+    else:
+        print(f"❌ SQLExecutor failed: {result.get('error')}")
+
+if __name__ == "__main__":
+    test_sql_executor_with_extra_fields() 

memra/demos/etl_invoice_processing/test_sql_executor_fix.py

@@ -0,0 +1,40 @@
+#!/usr/bin/env python3
+"""
+Test SQLExecutor fix in execution engine
+"""
+
+import sys
+import os
+sys.path.insert(0, '/Users/tarpus/memra')
+
+from memra.execution import ExecutionEngine
+
+def test_sql_executor_real_work():
+    """Test that SQLExecutor is correctly identified as real work"""
+    
+    engine = ExecutionEngine()
+    
+    # Mock SQLExecutor result (what the MCP bridge returns)
+    sql_executor_result = {
+        "query": "SELECT COUNT(*) as row_count FROM invoices",
+        "results": [{"row_count": 3}],
+        "row_count": 1,
+        "columns": ["row_count"],
+        "success": True
+    }
+    
+    print("🔧 Testing SQLExecutor real work detection...")
+    print(f"SQLExecutor result: {sql_executor_result}")
+    
+    # Test the _is_real_work method
+    is_real = engine._is_real_work("SQLExecutor", sql_executor_result)
+    
+    print(f"✅ Is real work: {is_real}")
+    
+    if is_real:
+        print("🎉 SQLExecutor fix is working!")
+    else:
+        print("❌ SQLExecutor fix is not working!")
+
+if __name__ == "__main__":
+    test_sql_executor_real_work() 

memra/demos/etl_invoice_processing/test_updated_server.py

@@ -0,0 +1,50 @@
+import os
+import requests
+import json
+
+api_url = "https://api.memra.co"
+api_key = os.getenv("MEMRA_API_KEY", "test-secret-for-development")
+
+# Test with the schema format the client sends
+schema = [
+    {"column_name": "invoice_number", "data_type": "character varying", "is_nullable": "NO"},
+    {"column_name": "vendor_name", "data_type": "character varying", "is_nullable": "NO"},
+    {"column_name": "invoice_date", "data_type": "date", "is_nullable": "NO"},
+    {"column_name": "total_amount", "data_type": "numeric", "is_nullable": "NO"}
+]
+
+print("Testing updated server with client schema format:")
+print(json.dumps(schema[:2], indent=2))  # Show first 2 fields
+
+resp = requests.post(
+    f"{api_url}/tools/execute",
+    json={
+        "tool_name": "PDFProcessor",
+        "hosted_by": "memra",
+        "input_data": {
+            "file": "/uploads/6f4538c0-8fce-4488-be49-1a78afc58a4a.pdf",
+            "schema": schema
+        }
+    },
+    headers={"X-API-Key": api_key}
+)
+
+print(f"\nResponse status: {resp.status_code}")
+if resp.status_code == 200:
+    result = resp.json()
+    if result.get('success') and 'data' in result and 'data' in result['data']:
+        data = result['data']['data']
+        vision_prompt = data.get('vision_prompt', '')
+        print(f"\nVision prompt generated (first 500 chars):")
+        print(vision_prompt[:500])
+        
+        # Check if vendor_name is in the prompt
+        if 'vendor_name' in vision_prompt:
+            print("\n✅ SUCCESS: vendor_name is now in the vision prompt\!")
+        else:
+            print("\n❌ FAIL: vendor_name is NOT in the vision prompt")
+            
+        # Check vision response
+        vision_resp = data.get('vision_response', '')
+        if vision_resp:
+            print(f"\nVision response includes vendor info: {'vendor' in vision_resp.lower()}")

memra/demos/etl_invoice_processing/test_upload_functionality.py

@@ -0,0 +1,156 @@
+#!/usr/bin/env python3
+"""
+Test script to verify file upload functionality
+"""
+
+import os
+import base64
+import requests
+import json
+
+def test_upload_functionality():
+    """Test the complete upload and processing workflow"""
+    
+    # Test server URL (change this to api.memra.co when implementing on remote)
+    API_BASE = "http://localhost:8000"
+    
+    print("🧪 Testing File Upload Functionality")
+    print("=" * 50)
+    
+    # Step 1: Check server health
+    print("\n1️⃣ Checking server health...")
+    try:
+        response = requests.get(f"{API_BASE}/health")
+        if response.status_code == 200:
+            print("✅ Server is healthy")
+        else:
+            print(f"❌ Server health check failed: {response.status_code}")
+            return
+    except Exception as e:
+        print(f"❌ Cannot connect to server: {e}")
+        print("💡 Make sure to start the test server: python3 test_upload_server.py")
+        return
+    
+    # Step 2: Discover available tools
+    print("\n2️⃣ Discovering available tools...")
+    try:
+        response = requests.get(f"{API_BASE}/tools/discover")
+        if response.status_code == 200:
+            tools = response.json().get("tools", [])
+            print(f"✅ Found {len(tools)} tools:")
+            for tool in tools:
+                print(f"   - {tool['name']}: {tool['description']}")
+        else:
+            print(f"❌ Tool discovery failed: {response.status_code}")
+            return
+    except Exception as e:
+        print(f"❌ Tool discovery error: {e}")
+        return
+    
+    # Step 3: Create a test PDF file
+    print("\n3️⃣ Creating test PDF file...")
+    test_pdf_path = "test_invoice.pdf"
+    
+    # Create a simple test PDF (this is just for testing - in real usage you'd use actual PDFs)
+    try:
+        # For testing, we'll create a simple text file and pretend it's a PDF
+        with open(test_pdf_path, 'w') as f:
+            f.write("Test Invoice\nVendor: Test Corp\nAmount: $1234.56\n")
+        print(f"✅ Created test file: {test_pdf_path}")
+    except Exception as e:
+        print(f"❌ Failed to create test file: {e}")
+        return
+    
+    # Step 4: Upload the file
+    print("\n4️⃣ Uploading file to server...")
+    try:
+        with open(test_pdf_path, 'rb') as f:
+            file_content = f.read()
+        
+        file_b64 = base64.b64encode(file_content).decode('utf-8')
+        
+        upload_data = {
+            "filename": os.path.basename(test_pdf_path),
+            "content": file_b64,
+            "content_type": "application/pdf"
+        }
+        
+        response = requests.post(
+            f"{API_BASE}/upload",
+            json=upload_data,
+            headers={"Content-Type": "application/json"}
+        )
+        
+        if response.status_code == 200:
+            result = response.json()
+            if result.get("success"):
+                remote_path = result["data"]["remote_path"]
+                print(f"✅ File uploaded successfully")
+                print(f"   Remote path: {remote_path}")
+                print(f"   File ID: {result['data']['file_id']}")
+            else:
+                print(f"❌ Upload failed: {result.get('error')}")
+                return
+        else:
+            print(f"❌ Upload request failed: {response.status_code}")
+            print(f"   Response: {response.text}")
+            return
+            
+    except Exception as e:
+        print(f"❌ Upload error: {e}")
+        return
+    
+    # Step 5: Process the uploaded file
+    print("\n5️⃣ Processing uploaded file...")
+    try:
+        process_data = {
+            "tool_name": "PDFProcessor",
+            "hosted_by": "memra",
+            "input_data": {
+                "file": remote_path
+            }
+        }
+        
+        response = requests.post(
+            f"{API_BASE}/tools/execute",
+            json=process_data,
+            headers={"Content-Type": "application/json"}
+        )
+        
+        if response.status_code == 200:
+            result = response.json()
+            if result.get("success"):
+                extracted_data = result["data"]["extracted_data"]
+                print("✅ File processed successfully")
+                print("📄 Extracted data:")
+                print(f"   Vendor: {extracted_data['headerSection']['vendorName']}")
+                print(f"   Invoice: {extracted_data['billingDetails']['invoiceNumber']}")
+                print(f"   Amount: ${extracted_data['chargesSummary']['document_total']}")
+            else:
+                print(f"❌ Processing failed: {result.get('error')}")
+                return
+        else:
+            print(f"❌ Processing request failed: {response.status_code}")
+            print(f"   Response: {response.text}")
+            return
+            
+    except Exception as e:
+        print(f"❌ Processing error: {e}")
+        return
+    
+    # Step 6: Cleanup
+    print("\n6️⃣ Cleaning up...")
+    try:
+        os.remove(test_pdf_path)
+        print("✅ Test file removed")
+    except:
+        pass
+    
+    print("\n🎉 All tests passed! File upload functionality is working correctly.")
+    print("\n📋 Next Steps:")
+    print("1. Implement the upload endpoint on the remote API (api.memra.co)")
+    print("2. Update the demo to use the real remote API")
+    print("3. Test with actual PDF files")
+
+if __name__ == "__main__":
+    test_upload_functionality()