memra 0.2.12__py3-none-any.whl → 0.2.14__py3-none-any.whl

memra/demos/etl_invoice_processing/modify_database.py

@@ -0,0 +1,65 @@
+#!/usr/bin/env python3
+"""
+Modify database through MCP bridge
+"""
+
+import requests
+import json
+
+def modify_database():
+    """Remove constraint and show table contents"""
+    
+    bridge_url = "http://localhost:8081"
+    bridge_secret = "test-secret-for-development"
+    
+    headers = {
+        "Content-Type": "application/json",
+        "X-Bridge-Secret": bridge_secret
+    }
+    
+    print("🔧 Modifying database through MCP bridge...")
+    
+    # 1. Show current table contents
+    print("\n📊 Current table contents:")
+    payload = {
+        "tool_name": "SQLExecutor",
+        "input_data": {
+            "sql_query": "SELECT * FROM invoices;"
+        }
+    }
+    
+    response = requests.post(f"{bridge_url}/execute_tool", json=payload, headers=headers)
+    if response.status_code == 200:
+        result = response.json()
+        if result.get("success"):
+            data = result["data"]
+            print(f"Query: {data['query']}")
+            print(f"Rows: {data['row_count']}")
+            for row in data['results']:
+                print(f"  {row}")
+        else:
+            print(f"Error: {result.get('error')}")
+    else:
+        print(f"HTTP Error: {response.status_code}")
+    
+    # 2. Remove the unique constraint
+    print("\n🔧 Removing unique constraint on invoice_number...")
+    payload = {
+        "tool_name": "SQLExecutor",
+        "input_data": {
+            "sql_query": "ALTER TABLE invoices DROP CONSTRAINT IF EXISTS invoices_invoice_number_key;"
+        }
+    }
+    
+    response = requests.post(f"{bridge_url}/execute_tool", json=payload, headers=headers)
+    if response.status_code == 200:
+        result = response.json()
+        if result.get("success"):
+            print("✅ Constraint removed successfully!")
+        else:
+            print(f"Error: {result.get('error')}")
+    else:
+        print(f"HTTP Error: {response.status_code}")
+
+if __name__ == "__main__":
+    modify_database() 

memra/demos/etl_invoice_processing/run_etl_batch.py

@@ -0,0 +1,60 @@
+#!/usr/bin/env python3
+"""
+Run ETL workflow in batch mode
+"""
+
+import os
+import sys
+from pathlib import Path
+
+# Add the parent directory to the path so we can import memra
+sys.path.insert(0, str(Path(__file__).parent.parent.parent))
+
+from memra import Agent, Department, LLM, check_api_health, get_api_status
+from memra.execution import ExecutionEngine
+from etl_invoice_demo import etl_department, upload_file_to_api
+
+# Set API key for authentication
+os.environ['MEMRA_API_KEY'] = 'test-secret-for-development'
+os.environ['MEMRA_API_URL'] = 'https://api.memra.co'
+
+# Check API health
+print("🔍 Checking Memra API status...")
+api_status = get_api_status()
+print(f"API Health: {'✅ Healthy' if api_status['api_healthy'] else '❌ Unavailable'}")
+
+if not api_status['api_healthy']:
+    print("❌ Cannot proceed - Memra API is not available")
+    sys.exit(1)
+
+# Process one invoice
+invoice_path = "data/invoices/10352259401.PDF"  # Superior Propane invoice
+print(f"\n📄 Processing invoice: {invoice_path}")
+
+# Upload file to remote API
+remote_path = upload_file_to_api(invoice_path)
+if not remote_path:
+    print("❌ Failed to upload file")
+    sys.exit(1)
+
+# Database connection info
+connection = "postgresql://postgres:postgres@localhost:5433/local_workflow"
+
+# Prepare input for department
+input_data = {
+    "file": remote_path,
+    "file_path": invoice_path,
+    "table_name": "invoices",
+    "connection": connection,
+    "sql_query": "SELECT column_name, data_type, is_nullable, column_default FROM information_schema.columns WHERE table_name = 'invoices' ORDER BY ordinal_position"
+}
+
+# Execute department
+engine = ExecutionEngine()
+result = engine.execute_department(etl_department, input_data)
+
+if result.success:
+    print("\n✅ ETL workflow completed successfully!")
+    print(f"Data written to database: {result.data.get('write_confirmation', {})}")
+else:
+    print(f"\n❌ ETL workflow failed: {result.error}")

memra/demos/etl_invoice_processing/setup_demo_data.py

@@ -0,0 +1,154 @@
+#!/usr/bin/env python3
+"""
+Setup Demo Data Script
+Helps users set up invoice files for the ETL demo
+"""
+
+import os
+import shutil
+import sys
+from pathlib import Path
+
+def create_demo_structure():
+    """Create the demo data directory structure"""
+    
+    # Define paths
+    demo_dir = Path(__file__).parent
+    data_dir = demo_dir / "data"
+    invoices_dir = data_dir / "invoices"
+    
+    # Create directories
+    invoices_dir.mkdir(parents=True, exist_ok=True)
+    
+    print("📁 Created demo data structure:")
+    print(f"  {data_dir}")
+    print(f"  {invoices_dir}")
+    
+    return invoices_dir
+
+def check_existing_files(invoices_dir):
+    """Check for existing invoice files"""
+    
+    pdf_files = list(invoices_dir.glob("*.pdf"))
+    
+    if pdf_files:
+        print(f"\n📄 Found {len(pdf_files)} existing invoice files:")
+        for pdf_file in pdf_files:
+            size_mb = pdf_file.stat().st_size / (1024 * 1024)
+            print(f"  - {pdf_file.name} ({size_mb:.1f} MB)")
+        return True
+    else:
+        print(f"\n📄 No invoice files found in {invoices_dir}")
+        return False
+
+def copy_sample_files(invoices_dir):
+    """Copy sample files from external locations if available"""
+    
+    # Common locations where users might have invoice files
+    sample_locations = [
+        Path.home() / "Downloads",
+        Path.home() / "Desktop",
+        Path.home() / "Documents",
+        Path.cwd() / "invoices",  # If they have an invoices folder in current directory
+    ]
+    
+    print("\n🔍 Looking for sample invoice files...")
+    
+    for location in sample_locations:
+        if location.exists():
+            pdf_files = list(location.glob("*.pdf"))
+            if pdf_files:
+                print(f"  Found {len(pdf_files)} PDF files in {location}")
+                
+                # Copy first few files (up to 5)
+                copied = 0
+                for pdf_file in pdf_files[:5]:
+                    if copied >= 5:
+                        break
+                    
+                    dest_file = invoices_dir / f"invoice_{copied+1:03d}.pdf"
+                    try:
+                        shutil.copy2(pdf_file, dest_file)
+                        size_mb = pdf_file.stat().st_size / (1024 * 1024)
+                        print(f"    ✅ Copied {pdf_file.name} -> {dest_file.name} ({size_mb:.1f} MB)")
+                        copied += 1
+                    except Exception as e:
+                        print(f"    ❌ Failed to copy {pdf_file.name}: {e}")
+                
+                if copied > 0:
+                    return True
+    
+    return False
+
+def create_placeholder_files(invoices_dir):
+    """Create placeholder files for testing"""
+    
+    print("\n📝 Creating placeholder files for testing...")
+    
+    # Create a simple text file as placeholder
+    placeholder_content = """
+This is a placeholder file for the ETL Invoice Processing demo.
+
+To use real invoice files:
+1. Place your PDF invoice files in this directory
+2. Rename them to invoice_001.pdf, invoice_002.pdf, etc.
+3. Run the demo: python etl_invoice_demo.py
+
+The demo will automatically discover and process all PDF files in this directory.
+"""
+    
+    placeholder_file = invoices_dir / "README.txt"
+    with open(placeholder_file, 'w') as f:
+        f.write(placeholder_content)
+    
+    print(f"  ✅ Created {placeholder_file}")
+    return False
+
+def main():
+    """Main setup function"""
+    
+    print("🚀 Setting up ETL Invoice Processing Demo Data")
+    print("=" * 50)
+    
+    # Create directory structure
+    invoices_dir = create_demo_structure()
+    
+    # Check for existing files
+    has_files = check_existing_files(invoices_dir)
+    
+    if not has_files:
+        # Try to copy sample files
+        copied = copy_sample_files(invoices_dir)
+        
+        if not copied:
+            # Create placeholder files
+            create_placeholder_files(invoices_dir)
+    
+    # Show next steps
+    print("\n🎯 Next Steps:")
+    print("1. Add your invoice PDF files to the data/invoices/ directory")
+    print("2. Run the demo: python etl_invoice_demo.py")
+    print("3. Check the demo output and database results")
+    
+    print("\n📚 For more information:")
+    print("- See data/README.md for detailed usage instructions")
+    print("- Check the main demo README for workflow details")
+    
+    # Show file size recommendations
+    print("\n💡 File Size Recommendations:")
+    print("- Individual files: 1-5 MB each")
+    print("- Total demo data: 20-50 MB")
+    print("- GitHub limit: 100 MB per file")
+    print("- Repository limit: 1 GB total")
+    
+    return True
+
+if __name__ == "__main__":
+    try:
+        main()
+    except KeyboardInterrupt:
+        print("\n\n⏹️  Setup cancelled by user")
+        sys.exit(1)
+    except Exception as e:
+        print(f"\n❌ Setup failed: {e}")
+        sys.exit(1) 

memra/demos/etl_invoice_processing/simple_pdf_processor.py

@@ -0,0 +1,181 @@
+#!/usr/bin/env python3
+"""
+Simple PDF Processor - Direct API call without complex post-processing
+"""
+
+import os
+import sys
+import json
+import requests
+import base64
+
+# Add the parent directory to the path to import memra modules
+sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..'))
+
+from memra.tool_registry_client import ToolRegistryClient
+
+# Set environment variables
+os.environ["MEMRA_API_URL"] = "https://api.memra.co"
+os.environ["MEMRA_API_KEY"] = "test-secret-for-development"
+
+def upload_file_to_api(file_path: str, api_url: str = "https://api.memra.co") -> str:
+    """Upload a file to the remote API"""
+    try:
+        print(f"📤 Uploading {os.path.basename(file_path)} to remote API")
+        
+        # Read the file and encode as base64
+        with open(file_path, 'rb') as f:
+            file_content = f.read()
+        
+        file_b64 = base64.b64encode(file_content).decode('utf-8')
+        
+        # Prepare upload data
+        upload_data = {
+            "filename": os.path.basename(file_path),
+            "content": file_b64,
+            "content_type": "application/pdf"
+        }
+        
+        # Upload to remote API
+        api_key = os.getenv("MEMRA_API_KEY")
+        response = requests.post(
+            f"{api_url}/upload",
+            json=upload_data,
+            headers={
+                "X-API-Key": api_key,
+                "Content-Type": "application/json"
+            }
+        )
+        
+        if response.status_code == 200:
+            result = response.json()
+            if result.get("success"):
+                remote_path = result["data"]["remote_path"]
+                print(f"✅ File uploaded successfully")
+                print(f"   Remote path: {remote_path}")
+                return remote_path
+            else:
+                print(f"❌ Upload failed: {result.get('error')}")
+                return file_path
+        else:
+            print(f"❌ Upload request failed: {response.status_code}")
+            return file_path
+                
+    except Exception as e:
+        print(f"⚠️  Upload error: {e}")
+        return file_path
+
+def process_pdf_simple(file_path: str) -> dict:
+    """Simple PDF processing - direct API call, no post-processing"""
+    print(f"\n🔍 Processing PDF: {file_path}")
+    print("=" * 50)
+    
+    # Step 1: Upload file to API
+    remote_path = upload_file_to_api(file_path)
+    
+    # Step 2: Call PDFProcessor directly
+    client = ToolRegistryClient()
+    
+    print(f"\n📄 Calling PDFProcessor with remote path: {remote_path}")
+    
+    result = client.execute_tool(
+        tool_name="PDFProcessor",
+        hosted_by="memra",
+        input_data={"file": remote_path},
+        config=None
+    )
+    
+    print(f"\n📊 API Response:")
+    print(f"Success: {result.get('success')}")
+    print(f"Error: {result.get('error')}")
+    
+    # Debug: Print the full response structure
+    print(f"\n🔍 DEBUG: Full API Response Structure:")
+    print(f"Result keys: {list(result.keys())}")
+    print(f"Result: {json.dumps(result, indent=2, default=str)}")
+    
+    if result.get('success') and 'data' in result:
+        data = result['data']
+        print(f"\n🎯 Raw JSON Response from Vision Model:")
+        
+        if 'vision_response' in data:
+            vision_response = data['vision_response']
+            print(f"📝 Vision Response (raw):")
+            print(vision_response)
+            
+            # Parse the JSON response
+            try:
+                # Clean the response - remove markdown code blocks if present
+                cleaned_response = vision_response.strip()
+                if cleaned_response.startswith('```json'):
+                    cleaned_response = cleaned_response[7:]  # Remove ```json
+                if cleaned_response.endswith('```'):
+                    cleaned_response = cleaned_response[:-3]  # Remove ```
+                cleaned_response = cleaned_response.strip()
+                
+                parsed_json = json.loads(cleaned_response)
+                print(f"\n✅ Parsed JSON Response:")
+                print(json.dumps(parsed_json, indent=2))
+                
+                return {
+                    "success": True,
+                    "raw_vision_response": vision_response,
+                    "parsed_json": parsed_json,
+                    "extracted_data": data.get('extracted_data', {})
+                }
+                
+            except json.JSONDecodeError as e:
+                print(f"❌ JSON parsing error: {e}")
+                return {
+                    "success": False,
+                    "error": f"JSON parsing failed: {e}",
+                    "raw_response": vision_response
+                }
+        else:
+            print("❌ No vision_response in data")
+            return {
+                "success": False,
+                "error": "No vision_response in API response",
+                "data": data
+            }
+    else:
+        print(f"❌ API call failed: {result.get('error')}")
+        return {
+            "success": False,
+            "error": result.get('error', 'Unknown error'),
+            "result": result
+        }
+
+def main():
+    """Main function to process a PDF file"""
+    if len(sys.argv) != 2:
+        print("Usage: python3 simple_pdf_processor.py <pdf_file_path>")
+        sys.exit(1)
+    
+    file_path = sys.argv[1]
+    
+    if not os.path.exists(file_path):
+        print(f"❌ File not found: {file_path}")
+        sys.exit(1)
+    
+    # Process the PDF
+    result = process_pdf_simple(file_path)
+    
+    if result.get('success'):
+        print(f"\n🎉 SUCCESS! PDF processed successfully")
+        print(f"📄 Invoice Number: {result['parsed_json'].get('InvoiceNumber', 'N/A')}")
+        print(f"💰 Total Amount: ${result['parsed_json'].get('InvoiceTotal', 'N/A')}")
+        print(f"📅 Date: {result['parsed_json'].get('InvoiceDate', 'N/A')}")
+        
+        # Save the result to a JSON file for later use
+        output_file = f"{os.path.splitext(os.path.basename(file_path))[0]}_result.json"
+        with open(output_file, 'w') as f:
+            json.dump(result, f, indent=2)
+        print(f"\n💾 Result saved to: {output_file}")
+        
+    else:
+        print(f"\n❌ FAILED: {result.get('error')}")
+        sys.exit(1)
+
+if __name__ == "__main__":
+    main() 

memra/demos/etl_invoice_processing/test_agent3.py

@@ -0,0 +1,56 @@
+import os
+import requests
+import base64
+
+# Test the PDFProcessor API directly
+api_url = "https://api.memra.co"
+api_key = os.getenv("MEMRA_API_KEY", "test-secret-for-development")
+
+# Upload a test file
+with open("data/invoices/10352260169.PDF", "rb") as f:
+    content = base64.b64encode(f.read()).decode('utf-8')
+
+upload_resp = requests.post(
+    f"{api_url}/upload",
+    json={
+        "filename": "test.pdf",
+        "content": content,
+        "content_type": "application/pdf"
+    },
+    headers={"X-API-Key": api_key}
+)
+
+if upload_resp.status_code == 200:
+    remote_path = upload_resp.json()["data"]["remote_path"]
+    print(f"✅ Uploaded to: {remote_path}")
+    
+    # Try different parameter formats
+    print("\n1. Testing with input_data format:")
+    resp = requests.post(
+        f"{api_url}/tools/execute",
+        json={
+            "tool_name": "PDFProcessor",
+            "hosted_by": "memra",
+            "input_data": {"file_path": remote_path}
+        },
+        headers={"X-API-Key": api_key}
+    )
+    print(f"Status: {resp.status_code}")
+    print(f"Response: {resp.json()}")
+    
+    print("\n2. Testing with parameters format:")
+    resp = requests.post(
+        f"{api_url}/tools/execute",
+        json={
+            "tool_name": "PDFProcessor",
+            "hosted_by": "memra",
+            "parameters": {"file_path": remote_path}
+        },
+        headers={"X-API-Key": api_key}
+    )
+    print(f"Status: {resp.status_code}")
+    if resp.status_code == 200:
+        import json
+        print(f"Response: {json.dumps(resp.json(), indent=2)}")
+    else:
+        print(f"Response: {resp.text}")

memra/demos/etl_invoice_processing/test_agent3_v2.py

@@ -0,0 +1,32 @@
+import os
+import requests
+
+api_url = "https://api.memra.co"
+api_key = os.getenv("MEMRA_API_KEY", "test-secret-for-development")
+
+# Try with 'file' instead of 'file_path'
+remote_path = "/uploads/6f4538c0-8fce-4488-be49-1a78afc58a4a.pdf"
+
+print("Testing with 'file' parameter:")
+resp = requests.post(
+    f"{api_url}/tools/execute",
+    json={
+        "tool_name": "PDFProcessor",
+        "hosted_by": "memra",
+        "input_data": {"file": remote_path}
+    },
+    headers={"X-API-Key": api_key}
+)
+print(f"Status: {resp.status_code}")
+if resp.status_code == 200:
+    import json
+    result = resp.json()
+    print(f"Success: {result.get('success')}")
+    if result.get('success') and 'data' in result:
+        data = result['data']
+        if 'data' in data:
+            print("Found nested data structure")
+            inner_data = data['data']
+            if 'vision_response' in inner_data:
+                print("✅ Found vision_response\!")
+                print(inner_data['vision_response'][:200])

memra/demos/etl_invoice_processing/test_api.py

@@ -0,0 +1,28 @@
+#!/usr/bin/env python3
+"""Test API connectivity"""
+
+import os
+import requests
+import json
+
+def test_api():
+    # Test remote API
+    print("Testing remote API...")
+    try:
+        response = requests.get("https://api.memra.co/health", timeout=10)
+        print(f"Remote API Status: {response.status_code}")
+        print(f"Response: {response.text}")
+    except Exception as e:
+        print(f"Remote API Error: {e}")
+    
+    # Test local API
+    print("\nTesting local API...")
+    try:
+        response = requests.get("http://127.0.0.1:8081/health", timeout=5)
+        print(f"Local API Status: {response.status_code}")
+        print(f"Response: {response.text}")
+    except Exception as e:
+        print(f"Local API Error: {e}")
+
+if __name__ == "__main__":
+    test_api() 

memra/demos/etl_invoice_processing/test_api_client_direct.py

@@ -0,0 +1,89 @@
+#!/usr/bin/env python3
+"""
+Test API client directly to see what PDFProcessor returns
+"""
+
+import os
+import sys
+import json
+
+# Add the parent directory to the path to import memra modules
+sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..'))
+
+from memra.tool_registry_client import ToolRegistryClient
+
+# Set environment variables
+os.environ["MEMRA_API_URL"] = "https://api.memra.co"
+os.environ["MEMRA_API_KEY"] = "test-secret-for-development"
+
+def test_api_client_direct():
+    """Test API client directly"""
+    print("🧪 Testing API Client Direct Call")
+    print("=" * 50)
+    
+    # Create API client
+    client = ToolRegistryClient()
+    
+    # Test with the same file path that the ETL workflow uses
+    file_path = "/uploads/22526d2e-dfcf-45eb-9e1a-47f093cd05ab.PDF"  # From the latest ETL upload
+    
+    print(f"📄 Testing with file: {file_path}")
+    
+    # Execute the PDFProcessor tool
+    result = client.execute_tool(
+        tool_name="PDFProcessor",
+        hosted_by="memra",
+        input_data={"file": file_path},
+        config=None
+    )
+    
+    print(f"\n📊 Raw API Client Result:")
+    print(f"Success: {result.get('success')}")
+    print(f"Error: {result.get('error')}")
+    
+    if result.get('success'):
+        print(f"\n📄 Data keys: {list(result.keys())}")
+        
+        if 'data' in result:
+            data = result['data']
+            print(f"\n🔧 Data field:")
+            print(f"Type: {type(data)}")
+            print(f"Keys: {list(data.keys()) if isinstance(data, dict) else 'not a dict'}")
+            
+            if isinstance(data, dict):
+                print(f"\n📄 Data content:")
+                print(json.dumps(data, indent=2, default=str))
+                
+                # Check for nested data
+                if 'data' in data:
+                    nested_data = data['data']
+                    print(f"\n🔧 Nested data:")
+                    print(f"Type: {type(nested_data)}")
+                    print(f"Keys: {list(nested_data.keys()) if isinstance(nested_data, dict) else 'not a dict'}")
+                    
+                    if isinstance(nested_data, dict):
+                        print(f"\n📄 Nested data content:")
+                        print(json.dumps(nested_data, indent=2, default=str))
+                        
+                        # Check for vision_response and extracted_data
+                        if 'vision_response' in nested_data:
+                            print(f"\n📝 Vision Response found!")
+                            print(f"Length: {len(nested_data['vision_response'])} characters")
+                            try:
+                                parsed = json.loads(nested_data['vision_response'].replace('```json','').replace('```','').strip())
+                                print("✅ Valid JSON response:")
+                                print(json.dumps(parsed, indent=2))
+                            except Exception as e:
+                                print(f"❌ JSON parsing error: {e}")
+                                print(f"Raw response: {nested_data['vision_response'][:500]}...")
+                        
+                        if 'extracted_data' in nested_data:
+                            print(f"\n🎯 Extracted Data found!")
+                            print(json.dumps(nested_data['extracted_data'], indent=2))
+        else:
+            print("❌ No 'data' field in result")
+    else:
+        print(f"❌ API call failed: {result.get('error')}")
+
+if __name__ == "__main__":
+    test_api_client_direct()