connectonion 0.4.11__py3-none-any.whl → 0.5.0__py3-none-any.whl

tests/real_api/test_real_multi_llm.py

@@ -1,454 +0,0 @@
-"""Pytest tests for multi-LLM model support across OpenAI, Google, and Anthropic."""
-
-import os
-import time
-from unittest.mock import Mock, patch, MagicMock
-from dotenv import load_dotenv
-import pytest
-from connectonion import Agent
-from connectonion.llm import LLMResponse, ToolCall
-
-# Load environment variables from .env file
-load_dotenv()
-
-
-# Test tools that will work across all models
-def simple_calculator(a: int, b: int) -> int:
-    """Add two numbers together."""
-    return a + b
-
-
-def get_greeting(name: str) -> str:
-    """Generate a greeting for a person."""
-    return f"Hello, {name}!"
-
-
-def process_data(data: str, uppercase: bool = False) -> str:
-    """Process text data with optional uppercase conversion."""
-    if uppercase:
-        return data.upper()
-    return data.lower()
-
-
-def _available_providers():
-    providers = []
-    if os.getenv("OPENAI_API_KEY"):
-        providers.append("openai")
-    if os.getenv("GEMINI_API_KEY") or os.getenv("GOOGLE_API_KEY"):
-        providers.append("google")
-    if os.getenv("ANTHROPIC_API_KEY"):
-        providers.append("anthropic")
-    return providers
-
-
-def _tools():
-    return [simple_calculator, get_greeting, process_data]
-    
-    # -------------------------------------------------------------------------
-    # Model Detection Tests
-    # -------------------------------------------------------------------------
-    
-def test_model_detection_openai():
-    models = ["o4-mini", "gpt-4o", "gpt-4o-mini", "gpt-3.5-turbo", "o1", "o1-mini"]
-    for model in models:
-        assert model.startswith("gpt") or model.startswith("o")
-    
-def test_model_detection_google():
-    models = [
-        "gemini-2.5-pro",
-        "gemini-2.0-flash-exp",
-        "gemini-2.0-flash-thinking-exp",
-        "gemini-2.5-flash",
-        "gemini-1.5-flash-8b",  # Note: 2.5-flash-8b doesn't exist, use 1.5-flash-8b
-    ]
-    for model in models:
-        assert model.startswith("gemini")
-    
-def test_model_detection_anthropic():
-    models = [
-        "claude-opus-4.1",
-        "claude-opus-4",
-        "claude-sonnet-4",
-        "claude-3-5-sonnet-latest",
-        "claude-3-5-haiku-latest",
-    ]
-    for model in models:
-        assert model.startswith("claude")
-    
-    # -------------------------------------------------------------------------
-    # Agent Creation Tests (Using Mock)
-    # -------------------------------------------------------------------------
-    
-@patch('connectonion.llm.OpenAILLM')
-def test_create_agent_with_openai_flagship(mock_openai):
-        """Test creating an agent with OpenAI flagship model."""
-        mock_instance = Mock()
-        # mock_instance.model = "gpt-5"  # GPT-5 requires passport verification
-        mock_instance.model = "o4-mini"  # Using o4-mini for testing
-        mock_openai.return_value = mock_instance
-        
-        # agent = Agent("test_gpt5", model="gpt-5")  # Original GPT-5 test
-        agent = Agent("test_o4", model="o4-mini")  # Using o4-mini
-        assert agent.name == "test_o4"
-        # Will work once multi-LLM is implemented
-        # self.assertEqual(agent.llm.model, "o4-mini")
-    
-@patch('connectonion.llm.GeminiLLM')
-def test_create_agent_with_gemini25(mock_gemini):
-        """Test creating an agent with Gemini 2.5 Pro model."""
-        mock_instance = Mock()
-        mock_instance.model = "gemini-2.5-pro"
-        mock_gemini.return_value = mock_instance
-        
-        # Will work once GeminiLLM is implemented
-        # agent = Agent("test_gemini", model="gemini-2.5-pro")
-        # self.assertEqual(agent.llm.model, "gemini-2.5-pro")
-        pytest.skip("GeminiLLM not yet implemented")
-    
-@patch('connectonion.llm.AnthropicLLM')
-def test_create_agent_with_claude_opus4(mock_anthropic):
-        """Test creating an agent with Claude Opus 4.1 model."""
-        mock_instance = Mock()
-        mock_instance.model = "claude-opus-4.1"
-        mock_anthropic.return_value = mock_instance
-        
-        # Will work once AnthropicLLM is implemented
-        # agent = Agent("test_claude", model="claude-opus-4.1")
-        # self.assertEqual(agent.llm.model, "claude-opus-4.1")
-        pytest.skip("AnthropicLLM not yet implemented")
-    
-    # -------------------------------------------------------------------------
-    # Tool Compatibility Tests
-    # -------------------------------------------------------------------------
-    
-def test_tools_work_across_all_models():
-        """Test that the same tools work with all model providers."""
-        test_cases = []
-        
-        # Use actual models from our docs
-        available_providers = _available_providers()
-        tools = _tools()
-        if "openai" in available_providers:
-            # test_cases.append(("gpt-5-nano", "openai"))  # GPT-5 requires passport
-            test_cases.append(("gpt-4o-mini", "openai"))  # Use available model for testing
-        if "google" in available_providers:
-            test_cases.append(("gemini-2.5-flash", "google"))
-        if "anthropic" in available_providers:
-            test_cases.append(("claude-3-5-haiku-latest", "anthropic"))
-        
-        if not test_cases:
-            pytest.skip("No API keys available for testing")
-        
-        for model, provider in test_cases:
-            try:
-                agent = Agent(f"test_{provider}", model=model, tools=tools)
-                assert len(agent.tools) == 3
-                assert "simple_calculator" in agent.tools
-                assert "get_greeting" in agent.tools
-                assert "process_data" in agent.tools
-                for tool in agent.tools:
-                    schema = tool.to_function_schema()
-                    assert "name" in schema
-                    assert "description" in schema
-                    assert "parameters" in schema
-            except Exception as e:
-                if "Unknown model" in str(e) or "not yet implemented" in str(e):
-                    pytest.skip(f"Model {model} not yet implemented")
-                raise
-    
-    # -------------------------------------------------------------------------
-    # Model Registry Tests
-    # -------------------------------------------------------------------------
-    
-def test_model_registry_mapping():
-        """Test that models map to correct providers."""
-        # This will be the expected mapping when implemented
-        expected_mapping = {
-            # OpenAI models
-            "o4-mini": "openai",  # Testing model
-            "gpt-4o": "openai",
-            "gpt-4o-mini": "openai",
-            "gpt-3.5-turbo": "openai",
-            "o1": "openai",
-            "o1-mini": "openai",
-
-            # Google Gemini series
-            "gemini-2.5-pro": "google",
-            "gemini-2.0-flash-exp": "google",
-            "gemini-2.0-flash-thinking-exp": "google",
-            "gemini-2.5-flash": "google",
-            "gemini-1.5-flash-8b": "google",  # Fixed: 2.5-flash-8b doesn't exist
-
-            # Anthropic Claude series
-            "claude-opus-4.1": "anthropic",
-            "claude-opus-4": "anthropic",
-            "claude-sonnet-4": "anthropic",
-            "claude-3-5-sonnet-latest": "anthropic",
-            "claude-3-5-haiku-latest": "anthropic"
-        }
-        
-        # When MODEL_REGISTRY is implemented, test it
-        try:
-            from connectonion.llm import MODEL_REGISTRY
-            for model, expected_provider in expected_mapping.items():
-                assert MODEL_REGISTRY.get(model) == expected_provider, (
-                    f"Model {model} should map to {expected_provider}"
-                )
-        except ImportError:
-            pytest.skip("MODEL_REGISTRY not yet implemented")
-    
-    # -------------------------------------------------------------------------
-    # Error Handling Tests
-    # -------------------------------------------------------------------------
-    
-def test_missing_api_key_error():
-        """Test appropriate error when API key is missing."""
-        # Temporarily remove API key
-        original_key = os.environ.get("OPENAI_API_KEY")
-        if original_key:
-            del os.environ["OPENAI_API_KEY"]
-        
-        try:
-            with pytest.raises(ValueError) as context:
-                Agent("test", model="o4-mini")
-            assert "API key" in str(context.value)
-        finally:
-            # Restore API key
-            if original_key:
-                os.environ["OPENAI_API_KEY"] = original_key
-    
-def test_unknown_model_error():
-        """Test error handling for unknown model names."""
-        # This should raise an error once model validation is implemented
-        try:
-            with pytest.raises(ValueError) as context:
-                Agent("test", model="unknown-model-xyz")
-            assert "Unknown model" in str(context.value)
-        except Exception:
-            pytest.skip("Model validation not yet implemented")
-    
-    # -------------------------------------------------------------------------
-    # Integration Tests (require actual API keys)
-    # -------------------------------------------------------------------------
-    
-def test_openai_flagship_real_call():
-        """Test actual API call with OpenAI flagship model."""
-        # Testing with o4-mini (GPT-5 requires passport verification)
-        tools = _tools()
-        try:
-            agent = Agent("test", model="o4-mini", tools=tools)
-            response = agent.input("Use the simple_calculator tool to add 5 and 3")
-            assert "8" in response
-        except Exception as e:
-            if "model not found" in str(e).lower() or "o4-mini" in str(e).lower():
-                # o4-mini not available yet, try with current model
-                agent = Agent("test", model="gpt-4o-mini", tools=tools)
-                response = agent.input("Use the simple_calculator tool to add 5 and 3")
-                assert "8" in response
-            else:
-                raise
-    
-def test_google_gemini_real_call():
-        """Test actual API call with Gemini model."""
-        # Try Gemini 2.5 Pro first, fallback to 1.5 if not available
-        models_to_try = ["gemini-2.5-pro", "gemini-2.5-flash", "gemini-2.5-pro"]
-        
-        tools = _tools()
-        for model in models_to_try:
-            try:
-                agent = Agent("test", model=model, tools=tools)
-                response = agent.input("Use the get_greeting tool to greet 'Alice'")
-                assert "Alice" in response
-                break
-            except Exception as e:
-                if model == models_to_try[-1]:
-                    pytest.skip(f"No Gemini models available: {e}")
-                continue
-    
-def test_anthropic_claude_real_call():
-        """Test actual API call with Claude model."""
-        # Try Claude Opus 4.1 first, fallback to available models
-        models_to_try = ["claude-opus-4.1", "claude-3-5-sonnet-latest", "claude-3-5-haiku-latest"]
-        
-        tools = _tools()
-        for model in models_to_try:
-            try:
-                agent = Agent("test", model=model, tools=tools)
-                response = agent.input("Use the process_data tool to convert 'Hello' to uppercase")
-                assert "HELLO" in response
-                break
-            except Exception as e:
-                if model == models_to_try[-1]:
-                    pytest.skip(f"No Claude models available: {e}")
-                continue
-    
-    # -------------------------------------------------------------------------
-    # Model Comparison Tests
-    # -------------------------------------------------------------------------
-    
-def test_flagship_model_comparison():
-        """Test that flagship models from each provider can handle the same prompt."""
-        prompt = "What is 2 + 2?"
-        results = {}
-        
-        flagship_models = []
-        available_providers = _available_providers()
-        if "openai" in available_providers:
-            # Use gpt-4o-mini as fallback since GPT-5 isn't available yet
-            flagship_models.append(("gpt-4o-mini", "openai"))
-        if "google" in available_providers:
-            flagship_models.append(("gemini-2.5-flash", "google"))
-        if "anthropic" in available_providers:
-            flagship_models.append(("claude-3-5-haiku-latest", "anthropic"))
-        
-        if len(flagship_models) < 2:
-            pytest.skip("Need at least 2 providers for comparison test")
-        
-        for model, provider in flagship_models:
-            try:
-                agent = Agent(f"compare_{provider}", model=model)
-                response = agent.input(prompt)
-                results[model] = response
-                
-                # All should mention "4" in their response
-                assert "4" in response.lower()
-            except Exception as e:
-                print(f"Failed with {model}: {e}")
-                continue
-    
-    # -------------------------------------------------------------------------
-    # Fallback Chain Tests
-    # -------------------------------------------------------------------------
-    
-def test_fallback_chain_with_new_models():
-        """Test fallback chain with new model hierarchy."""
-        # Priority order from docs/models.md
-        fallback_models = [
-            # "gpt-5",              # Best overall (requires passport verification)
-            "o4-mini",            # Testing flagship model
-            "claude-opus-4.1",    # Strong alternative (might not be available)
-            "gemini-2.5-pro",     # Multimodal option (might not be available)
-            # "gpt-5-mini",         # Faster fallback (requires passport)
-            "gpt-4o",             # Current best available
-            "gpt-4o-mini"         # Fallback (should work)
-        ]
-        
-        agent = None
-        successful_model = None
-        
-        for model in fallback_models:
-            try:
-                agent = Agent("fallback_test", model=model)
-                successful_model = model
-                break
-            except Exception:
-                continue
-        
-        if agent is None:
-            pytest.skip("No models available for fallback test")
-        
-        assert agent is not None
-        assert successful_model is not None
-    
-    # -------------------------------------------------------------------------
-    # Model Feature Tests
-    # -------------------------------------------------------------------------
-    
-def test_context_window_sizes():
-        """Test that models have correct context window sizes."""
-        # Based on docs/models.md specifications
-        context_windows = {
-            # "gpt-5": 200000,  # Requires passport
-            # "gpt-5-mini": 200000,  # Requires passport
-            # "gpt-5-nano": 128000,  # Requires passport
-            # "gpt-4.1": 128000,  # Not yet available
-            "o4-mini": 128000,  # Testing model
-            "gpt-4o": 128000,
-            "gpt-4o-mini": 128000,
-            "gpt-3.5-turbo": 16385,
-            "gemini-2.5-pro": 2000000,  # 2M tokens
-            "gemini-2.5-pro": 2000000,  # 2M tokens
-            "gemini-2.5-flash": 1000000,  # 1M tokens
-            "claude-opus-4.1": 200000,
-            "claude-opus-4": 200000,
-            "claude-sonnet-4": 200000
-        }
-        
-        # This will be tested once model metadata is implemented
-        pytest.skip("Model metadata not yet implemented")
-    
-def test_multimodal_capabilities():
-        """Test which models support multimodal input."""
-        # Based on docs/models.md
-        multimodal_models = [
-            # "gpt-5",  # Requires passport verification
-            "o4-mini",  # Testing model
-            "gpt-4o",
-            "gpt-4o-mini",
-            "gemini-2.5-pro",  # Supports audio, video, images, PDF
-            "gemini-2.0-flash-exp",
-            "gemini-2.5-pro",
-            "gemini-2.5-flash",
-            "claude-opus-4.1",
-            "claude-opus-4",
-            "claude-sonnet-4"
-        ]
-        
-        # Will be tested once multimodal support is implemented
-        pytest.skip("Multimodal support not yet implemented")
-    
-# -------------------------------------------------------------------------
-# Performance Tests
-# -------------------------------------------------------------------------
-
-@pytest.mark.benchmark
-def test_fast_model_performance():
-        """Test that fast models initialize quickly."""
-        available_providers = _available_providers()
-        if not available_providers:
-            pytest.skip("No API keys available")
-
-        # Use the fastest model from each provider
-        fast_models = []
-        if "openai" in available_providers:
-            fast_models.append("gpt-4o-mini")  # Fastest available
-        if "google" in available_providers:
-            fast_models.append("gemini-2.5-flash")
-        if "anthropic" in available_providers:
-            fast_models.append("claude-3-5-haiku-latest")
-
-        for model in fast_models:
-            try:
-                start_time = time.time()
-                agent = Agent("perf_test", model=model)
-                end_time = time.time()
-
-                initialization_time = end_time - start_time
-
-                # Should initialize in less than 2 seconds
-                assert initialization_time < 2.0, f"Model {model} initialization took {initialization_time:.2f}s"
-            except Exception as e:
-                # Model might not be available yet
-                if "not found" in str(e).lower() or "not available" in str(e).lower():
-                    continue
-                raise
-
-
-def test_select_model_for_code_generation():
-    """Test smart model selection based on use case."""
-    def select_model_for_task(task_type):
-        if task_type == "code":
-            return "o4-mini"
-        elif task_type == "reasoning":
-            return "gemini-2.5-pro"
-        elif task_type == "fast":
-            return "gpt-4o-mini"
-        elif task_type == "long_context":
-            return "gemini-2.5-pro"
-        else:
-            return "o4-mini"
-
-    assert select_model_for_task("code") == "o4-mini"
-    assert select_model_for_task("reasoning") == "gemini-2.5-pro"
-    assert select_model_for_task("fast") == "gpt-4o-mini"
-    assert select_model_for_task("long_context") == "gemini-2.5-pro"

tests/real_api/test_real_openai.py

@@ -1,100 +0,0 @@
-"""
-Real OpenAI API tests.
-
-These tests make actual API calls to OpenAI and cost real money.
-Run with: pytest test_real_openai.py -v
-
-Requires: OPENAI_API_KEY environment variable
-"""
-
-import os
-import pytest
-from pathlib import Path
-from dotenv import load_dotenv
-from connectonion import Agent
-from connectonion.llm import OpenAILLM
-
-# Load environment variables from tests/.env
-env_path = Path(__file__).parent / ".env"
-if env_path.exists():
-    load_dotenv(env_path)
-
-
-def calculator(expression: str) -> str:
-    """Simple calculator for testing."""
-    try:
-        result = eval(expression)
-        return f"Result: {result}"
-    except:
-        return "Error in calculation"
-
-
-class TestRealOpenAI:
-    """Test real OpenAI API integration."""
-
-    def test_openai_basic_completion(self):
-        """Test basic completion with OpenAI."""
-        llm = OpenAILLM(model="gpt-4o-mini")
-        agent = Agent(name="openai_test", llm=llm)
-
-        response = agent.input("Say 'Hello from OpenAI' exactly")
-        assert response is not None
-        assert "Hello from OpenAI" in response
-
-    def test_openai_with_tools(self):
-        """Test OpenAI with tool calling."""
-        agent = Agent(
-            name="openai_tools",
-            model="gpt-4o-mini",
-            tools=[calculator]
-        )
-
-        response = agent.input("Calculate 42 * 2")
-        assert response is not None
-        assert "84" in response
-
-    def test_openai_multi_turn(self):
-        """Test multi-turn conversation with OpenAI."""
-        agent = Agent(
-            name="openai_conversation",
-            model="gpt-4o-mini"
-        )
-
-        # First turn
-        response = agent.input("My name is Alice. Remember this.")
-        assert response is not None
-
-        # Second turn - should remember context
-        response = agent.input("What's my name?")
-        assert response is not None
-        assert "Alice" in response
-
-    def test_openai_streaming(self):
-        """Test streaming responses from OpenAI."""
-        agent = Agent(
-            name="openai_streaming",
-            model="gpt-4o-mini"
-        )
-
-        response = agent.input("Count from 1 to 5")
-        assert response is not None
-        # Should contain numbers 1 through 5
-        for num in ["1", "2", "3", "4", "5"]:
-            assert num in response
-
-    def test_openai_different_models(self):
-        """Test different OpenAI models."""
-        models = ["gpt-4o-mini", "gpt-4o"]
-
-        for model in models:
-            if model == "gpt-4o" and not os.getenv("TEST_EXPENSIVE_MODELS"):
-                continue  # Skip expensive models unless explicitly enabled
-
-            agent = Agent(
-                name=f"openai_{model.replace('-', '_')}",
-                model=model
-            )
-
-            response = agent.input("Reply with OK")
-            assert response is not None
-            assert len(response) > 0

tests/real_api/test_responses_parse.py

@@ -1,88 +0,0 @@
-"""Test for issue #9 - structured output with managed keys using responses.parse endpoint"""
-import os
-import pytest
-import time
-import requests
-from pydantic import BaseModel
-from nacl.signing import SigningKey
-from nacl.encoding import HexEncoder
-from connectonion import llm_do
-
-
-class EmailDraft(BaseModel):
-    """Email draft with subject and body"""
-    subject: str
-    body: str
-
-
-def _is_localhost_available():
-    """Check if localhost:8000 is reachable."""
-    try:
-        requests.get("http://localhost:8000/health", timeout=2)
-        return True
-    except (requests.exceptions.ConnectionError, requests.exceptions.Timeout):
-        return False
-
-
-def get_test_token():
-    """Generate test auth token for local backend"""
-    signing_key = SigningKey.generate()
-    public_key = "0x" + signing_key.verify_key.encode(encoder=HexEncoder).decode()
-    timestamp = int(time.time())
-    message = f"ConnectOnion-Auth-{public_key}-{timestamp}"
-    signature = signing_key.sign(message.encode()).signature.hex()
-
-    # Authenticate with local backend
-    response = requests.post(
-        "http://localhost:8000/api/v1/auth",
-        json={
-            "public_key": public_key,
-            "message": message,
-            "signature": signature
-        }
-    )
-
-    if response.status_code != 200:
-        raise Exception(f"Auth failed: {response.status_code} - {response.text}")
-
-    return response.json()["token"]
-
-
-@pytest.mark.skipif(
-    os.getenv("CI") or os.getenv("GITHUB_ACTIONS") or not _is_localhost_available(),
-    reason="Skipped in CI or when localhost:8000 is not available"
-)
-def test_structured_output_with_managed_keys():
-    """Test that llm_do() with Pydantic output works with co/ models.
-
-    This tests the /v1/responses/parse endpoint on the backend.
-    """
-    # Use local backend for testing
-    os.environ["OPENONION_DEV"] = "1"
-
-    # Get auth token
-    token = get_test_token()
-    os.environ["OPENONION_API_KEY"] = token
-
-    draft = llm_do(
-        "Write a friendly hello email to a new colleague",
-        output=EmailDraft,
-        temperature=0.7,
-        model="co/gpt-4o-mini"
-    )
-
-    # Verify we got a valid Pydantic model
-    assert isinstance(draft, EmailDraft)
-    assert isinstance(draft.subject, str)
-    assert isinstance(draft.body, str)
-    assert len(draft.subject) > 0
-    assert len(draft.body) > 0
-
-    print(f"\n✓ Structured output test passed!")
-    print(f"Subject: {draft.subject}")
-    print(f"Body: {draft.body[:100]}...")
-
-
-if __name__ == "__main__":
-    # Allow running directly for quick testing
-    test_structured_output_with_managed_keys()