massgen 0.0.3__py3-none-any.whl → 0.1.0__py3-none-any.whl

massgen/tests/test_azure_openai_backend.py

@@ -0,0 +1,156 @@
+# -*- coding: utf-8 -*-
+"""
+Test Azure OpenAI backend functionality.
+"""
+
+import os
+import sys
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+
+# Add the parent directory to sys.path to allow relative imports
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
+
+from backend.azure_openai import AzureOpenAIBackend  # noqa: E402
+
+
+class TestAzureOpenAIBackend:
+    """Test Azure OpenAI backend functionality."""
+
+    def test_init_with_env_vars(self):
+        """Test initialization with environment variables."""
+        with patch.dict(
+            os.environ,
+            {
+                "AZURE_OPENAI_API_KEY": "test-key",
+                "AZURE_OPENAI_ENDPOINT": "https://test.openai.azure.com/",
+                "AZURE_OPENAI_API_VERSION": "2024-02-15-preview",
+            },
+        ):
+            backend = AzureOpenAIBackend()
+            assert backend.api_key == "test-key"
+            assert backend.azure_endpoint == "https://test.openai.azure.com"
+            assert backend.api_version == "2024-02-15-preview"
+
+    def test_init_with_kwargs(self):
+        """Test initialization with keyword arguments."""
+        backend = AzureOpenAIBackend(
+            api_key="custom-key",
+            base_url="https://custom.openai.azure.com/",
+            api_version="2024-01-01",
+        )
+        assert backend.api_key == "custom-key"
+        assert backend.azure_endpoint == "https://custom.openai.azure.com"
+        assert backend.api_version == "2024-01-01"
+
+    def test_init_missing_api_key(self):
+        """Test initialization fails without API key."""
+        with patch.dict(os.environ, {}, clear=True):
+            with pytest.raises(ValueError, match="Azure OpenAI endpoint URL is required"):
+                AzureOpenAIBackend()
+
+    def test_init_missing_endpoint(self):
+        """Test initialization fails without endpoint."""
+        with patch.dict(os.environ, {"AZURE_OPENAI_API_KEY": "test-key"}, clear=True):
+            with pytest.raises(ValueError, match="Azure OpenAI endpoint URL is required"):
+                AzureOpenAIBackend()
+
+    def test_init_missing_api_key_with_endpoint(self):
+        """Test initialization fails without API key when endpoint is provided."""
+        with patch.dict(os.environ, {}, clear=True):
+            with pytest.raises(ValueError, match="Azure OpenAI API key is required"):
+                AzureOpenAIBackend(base_url="https://test.openai.azure.com/")
+
+    def test_base_url_normalization(self):
+        """Test base URL is properly normalized."""
+        backend = AzureOpenAIBackend(api_key="test-key", base_url="https://test.openai.azure.com")
+        assert backend.azure_endpoint == "https://test.openai.azure.com"
+
+        backend2 = AzureOpenAIBackend(api_key="test-key", base_url="https://test2.openai.azure.com/")
+        assert backend2.azure_endpoint == "https://test2.openai.azure.com"
+
+    def test_get_provider_name(self):
+        """Test provider name is correct."""
+        backend = AzureOpenAIBackend(api_key="test-key", base_url="https://test.openai.azure.com/")
+        assert backend.get_provider_name() == "Azure OpenAI"
+
+    def test_estimate_tokens(self):
+        """Test token estimation."""
+        backend = AzureOpenAIBackend(api_key="test-key", base_url="https://test.openai.azure.com/")
+        text = "This is a test message with several words."
+        estimated = backend.estimate_tokens(text)
+        assert estimated > 0
+        assert isinstance(estimated, (int, float))
+
+    def test_calculate_cost(self):
+        """Test cost calculation."""
+        backend = AzureOpenAIBackend(api_key="test-key", base_url="https://test.openai.azure.com/")
+
+        # Test GPT-4 cost calculation
+        cost = backend.calculate_cost(1000, 500, "gpt-4o")
+        assert cost > 0
+        assert isinstance(cost, float)
+
+        # Test GPT-3.5 cost calculation
+        cost2 = backend.calculate_cost(1000, 500, "gpt-3.5-turbo")
+        assert cost2 > 0
+        assert cost2 < cost  # GPT-3.5 should be cheaper than GPT-4
+
+    @pytest.mark.asyncio
+    async def test_stream_with_tools_missing_model(self):
+        """Test stream_with_tools fails without model parameter."""
+        backend = AzureOpenAIBackend(api_key="test-key", base_url="https://test.openai.azure.com/")
+
+        messages = [{"role": "user", "content": "Hello"}]
+        tools = []
+
+        # The validation happens at the beginning of the method, before any API calls
+        # So we don't need to mock the client for this test
+        try:
+            async for chunk in backend.stream_with_tools(messages, tools):
+                # If we get here, the validation didn't work as expected
+                # Check if it's an error chunk
+                if chunk.type == "error" and "deployment name" in chunk.error:
+                    # This is the expected behavior - validation error is yielded as a chunk
+                    return
+                else:
+                    # Unexpected - validation should have failed
+                    pytest.fail(f"Expected validation error, but got chunk: {chunk}")
+        except ValueError as e:
+            # This is the expected behavior - validation error is raised
+            if "deployment name" in str(e):
+                return
+            else:
+                pytest.fail(f"Unexpected ValueError: {e}")
+        except Exception as e:
+            pytest.fail(f"Unexpected exception: {e}")
+
+    @pytest.mark.asyncio
+    async def test_stream_with_tools_with_model(self):
+        """Test stream_with_tools works with model parameter."""
+        backend = AzureOpenAIBackend(api_key="test-key", base_url="https://test.openai.azure.com/")
+
+        messages = [{"role": "user", "content": "Hello"}]
+        tools = []
+
+        # Mock the client and create a mock stream response
+        mock_chunk = MagicMock()
+        mock_chunk.choices = [MagicMock()]
+        mock_chunk.choices[0].delta = MagicMock()
+        mock_chunk.choices[0].delta.content = "Hello"
+        mock_chunk.choices[0].finish_reason = "stop"
+
+        mock_stream = [mock_chunk]
+
+        with patch.object(backend, "client") as mock_client:
+            mock_client.chat.completions.create = AsyncMock(return_value=mock_stream)
+
+            # Test that it doesn't raise an error with model parameter
+            try:
+                async for chunk in backend.stream_with_tools(messages, tools, model="gpt-4"):
+                    # Just consume the stream
+                    pass
+            except Exception as e:
+                # If there's an error, it should not be about missing model
+                assert "deployment name" not in str(e)

massgen/tests/test_backend_capabilities.py

@@ -0,0 +1,262 @@
+# -*- coding: utf-8 -*-
+"""
+Tests for backend capabilities registry.
+
+These tests ensure the capabilities registry is consistent and valid.
+Run with: uv run pytest massgen/tests/test_backend_capabilities.py -v
+"""
+
+import pytest
+
+from massgen.backend.capabilities import (
+    BACKEND_CAPABILITIES,
+    get_all_backend_types,
+    get_backends_with_capability,
+    get_capabilities,
+    has_capability,
+    validate_backend_config,
+)
+
+
+class TestBackendCapabilitiesRegistry:
+    """Test the capabilities registry structure and validity."""
+
+    def test_all_backends_have_required_fields(self):
+        """Ensure all backend entries have required fields."""
+        for backend_type, caps in BACKEND_CAPABILITIES.items():
+            assert caps.backend_type == backend_type, f"{backend_type}: backend_type mismatch"
+            assert caps.provider_name, f"{backend_type}: provider_name is empty"
+            assert caps.supported_capabilities is not None, f"{backend_type}: supported_capabilities is None"
+            assert caps.builtin_tools is not None, f"{backend_type}: builtin_tools is None"
+            assert caps.filesystem_support in ["none", "native", "mcp"], f"{backend_type}: invalid filesystem_support"
+            assert caps.models, f"{backend_type}: models list is empty"
+            assert caps.default_model, f"{backend_type}: default_model is empty"
+
+    def test_default_model_in_models_list(self):
+        """Ensure default model exists in models list."""
+        for backend_type, caps in BACKEND_CAPABILITIES.items():
+            assert caps.default_model in caps.models, f"{backend_type}: default_model '{caps.default_model}' not in models list"
+
+    def test_filesystem_support_values(self):
+        """Ensure filesystem_support has valid values."""
+        valid_values = {"none", "native", "mcp"}
+        for backend_type, caps in BACKEND_CAPABILITIES.items():
+            assert caps.filesystem_support in valid_values, f"{backend_type}: filesystem_support '{caps.filesystem_support}' " f"not in {valid_values}"
+
+    def test_no_empty_backend_types(self):
+        """Ensure no backend has an empty backend_type."""
+        for backend_type, caps in BACKEND_CAPABILITIES.items():
+            assert backend_type, "Found backend with empty backend_type"
+            assert caps.backend_type, f"Backend {backend_type} has empty backend_type field"
+
+    def test_capability_strings_are_valid(self):
+        """Ensure capability strings follow conventions."""
+        valid_capabilities = {
+            "web_search",
+            "code_execution",
+            "bash",
+            "multimodal",  # Legacy - being phased out
+            "vision",  # Legacy - use image_understanding
+            "mcp",
+            "filesystem_native",
+            "filesystem_mcp",
+            "reasoning",
+            "image_generation",
+            "image_understanding",
+            "audio_generation",
+            "audio_understanding",
+            "video_generation",
+            "video_understanding",
+        }
+
+        for backend_type, caps in BACKEND_CAPABILITIES.items():
+            for cap in caps.supported_capabilities:
+                assert cap in valid_capabilities, f"{backend_type}: unknown capability '{cap}'. " f"Valid capabilities: {valid_capabilities}"
+
+
+class TestCapabilityQueries:
+    """Test capability query functions."""
+
+    def test_get_capabilities_existing_backend(self):
+        """Test getting capabilities for existing backends."""
+        caps = get_capabilities("openai")
+        assert caps is not None
+        assert caps.backend_type == "openai"
+        assert caps.provider_name == "OpenAI"
+
+    def test_get_capabilities_nonexistent_backend(self):
+        """Test getting capabilities for non-existent backend."""
+        caps = get_capabilities("nonexistent_backend")
+        assert caps is None
+
+    def test_has_capability_true(self):
+        """Test checking for existing capability."""
+        # OpenAI has web_search
+        assert has_capability("openai", "web_search") is True
+
+    def test_has_capability_false(self):
+        """Test checking for non-existent capability."""
+        # LM Studio doesn't have web_search
+        assert has_capability("lmstudio", "web_search") is False
+
+    def test_has_capability_nonexistent_backend(self):
+        """Test checking capability on non-existent backend."""
+        assert has_capability("nonexistent", "web_search") is False
+
+    def test_get_all_backend_types(self):
+        """Test getting all backend types."""
+        backend_types = get_all_backend_types()
+        assert len(backend_types) > 0
+        assert "openai" in backend_types
+        assert "claude" in backend_types
+        assert "gemini" in backend_types
+
+    def test_get_backends_with_capability(self):
+        """Test getting backends by capability."""
+        web_search_backends = get_backends_with_capability("web_search")
+        assert "openai" in web_search_backends
+        assert "gemini" in web_search_backends
+        assert "grok" in web_search_backends
+
+        # Backends without web search should not be included
+        assert "claude_code" not in web_search_backends
+
+
+class TestBackendValidation:
+    """Test backend configuration validation."""
+
+    def test_validate_valid_openai_config(self):
+        """Test validating a valid OpenAI config."""
+        config = {
+            "type": "openai",
+            "model": "gpt-4o",
+            "enable_web_search": True,
+            "enable_code_interpreter": True,
+        }
+        errors = validate_backend_config("openai", config)
+        assert len(errors) == 0
+
+    def test_validate_invalid_capability(self):
+        """Test validation catches unsupported capability."""
+        # Claude Code doesn't support web_search
+        config = {
+            "type": "claude_code",
+            "enable_web_search": True,
+        }
+        errors = validate_backend_config("claude_code", config)
+        assert len(errors) > 0
+        assert any("web_search" in error for error in errors)
+
+    def test_validate_invalid_backend_type(self):
+        """Test validation catches unknown backend."""
+        config = {"type": "nonexistent"}
+        errors = validate_backend_config("nonexistent", config)
+        assert len(errors) > 0
+        assert any("Unknown backend" in error for error in errors)
+
+    def test_validate_code_execution_variants(self):
+        """Test validation handles different code execution config keys."""
+        # OpenAI uses enable_code_interpreter
+        config_openai = {"type": "openai", "enable_code_interpreter": True}
+        errors = validate_backend_config("openai", config_openai)
+        assert len(errors) == 0
+
+        # Claude uses enable_code_execution
+        config_claude = {"type": "claude", "enable_code_execution": True}
+        errors = validate_backend_config("claude", config_claude)
+        assert len(errors) == 0
+
+    def test_validate_mcp_servers(self):
+        """Test validation of MCP server configuration."""
+        # Valid MCP config for backend that supports it
+        config = {
+            "type": "openai",
+            "mcp_servers": [
+                {
+                    "name": "weather",
+                    "command": "npx",
+                    "args": ["-y", "@fak111/weather-mcp"],
+                },
+            ],
+        }
+        errors = validate_backend_config("openai", config)
+        assert len(errors) == 0
+
+
+class TestSpecificBackends:
+    """Test specific backend configurations."""
+
+    def test_openai_capabilities(self):
+        """Test OpenAI backend capabilities."""
+        caps = get_capabilities("openai")
+        assert "web_search" in caps.supported_capabilities
+        assert "code_execution" in caps.supported_capabilities
+        assert "mcp" in caps.supported_capabilities
+        assert "reasoning" in caps.supported_capabilities
+        assert "image_generation" in caps.supported_capabilities
+        assert "image_understanding" in caps.supported_capabilities
+        assert "audio_generation" in caps.supported_capabilities
+        assert "video_generation" in caps.supported_capabilities
+        assert caps.filesystem_support == "mcp"
+        assert caps.env_var == "OPENAI_API_KEY"
+
+    def test_claude_capabilities(self):
+        """Test Claude backend capabilities."""
+        caps = get_capabilities("claude")
+        assert "web_search" in caps.supported_capabilities
+        assert "code_execution" in caps.supported_capabilities
+        assert "mcp" in caps.supported_capabilities
+        assert caps.filesystem_support == "mcp"
+        assert caps.env_var == "ANTHROPIC_API_KEY"
+
+    def test_claude_code_capabilities(self):
+        """Test Claude Code backend capabilities."""
+        caps = get_capabilities("claude_code")
+        assert "bash" in caps.supported_capabilities
+        assert "mcp" in caps.supported_capabilities
+        assert caps.filesystem_support == "native"
+        assert caps.env_var == "ANTHROPIC_API_KEY"
+        assert len(caps.builtin_tools) > 0
+
+    def test_gemini_capabilities(self):
+        """Test Gemini backend capabilities."""
+        caps = get_capabilities("gemini")
+        assert "web_search" in caps.supported_capabilities
+        assert "code_execution" in caps.supported_capabilities
+        assert "mcp" in caps.supported_capabilities
+        assert "image_understanding" in caps.supported_capabilities
+        assert caps.filesystem_support == "mcp"
+        assert caps.env_var == "GEMINI_API_KEY"
+
+    def test_local_backends_no_api_key(self):
+        """Test local backends don't require API keys."""
+        local_backends = ["lmstudio", "inference", "chatcompletion"]
+        for backend_type in local_backends:
+            caps = get_capabilities(backend_type)
+            # These backends may or may not require API keys depending on provider
+            # Just verify they're in the registry
+            assert caps is not None
+
+
+class TestConsistency:
+    """Test consistency between related fields."""
+
+    def test_filesystem_native_implies_capability(self):
+        """Backends with native filesystem should have filesystem capability."""
+        for backend_type, caps in BACKEND_CAPABILITIES.items():
+            if caps.filesystem_support == "native":
+                # Should have filesystem_native in capabilities
+                assert "filesystem_native" in caps.supported_capabilities or len(caps.builtin_tools) > 0, f"{backend_type}: native filesystem but no capability/tools"  # Or have filesystem tools
+
+    def test_mcp_capability_consistency(self):
+        """All backends should support MCP except where explicitly excluded."""
+        # Most backends support MCP
+        mcp_backends = get_backends_with_capability("mcp")
+        assert len(mcp_backends) > 0
+        assert "openai" in mcp_backends
+        assert "claude" in mcp_backends
+        assert "gemini" in mcp_backends
+
+
+if __name__ == "__main__":
+    pytest.main([__file__, "-v"])

massgen/tests/test_backend_event_loop_all.py

@@ -0,0 +1,179 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Event loop/resource cleanup tests for multiple backends without changing code.
+These tests mock SDK async clients and assert aclose() is awaited by backends.
+
+Backends covered:
+- ResponseBackend (OpenAI Responses API)
+- GrokBackend (xAI via OpenAI-compatible client)
+- ClaudeBackend (Anthropic Messages API)
+
+NOTE: Some tests may currently FAIL, revealing missing cleanup in backends.
+"""
+
+import asyncio
+from types import SimpleNamespace
+from typing import Any, List
+
+import pytest
+
+from massgen.backend import ClaudeBackend, GrokBackend, ResponseBackend
+
+
+# ---- Common fakes ----
+class _FakeStreamSingleStop:
+    """Async stream that yields once, then stops. Shape varies per backend needs."""
+
+    def __init__(self, item_factory):
+        self._yielded = False
+        self._item_factory = item_factory
+
+    def __aiter__(self):
+        return self
+
+    async def __anext__(self):
+        if self._yielded:
+            raise StopAsyncIteration
+        self._yielded = True
+        await asyncio.sleep(0)
+        return self._item_factory()
+
+
+class _FakeAsyncClientBase:
+    def __init__(self, *args: Any, **kwargs: Any):
+        self.args = args
+        self.kwargs = kwargs
+        self._closed = False
+
+    async def aclose(self) -> None:
+        await asyncio.sleep(0)
+        self._closed = True
+
+
+# ---- ResponseBackend test ----
+class _FakeResponses:
+    async def create(self, **kwargs: Any):
+        # Build a stream where each chunk has a 'type' attribute that ends the response quickly
+        def _item():
+            return SimpleNamespace(type="response.completed", response={"output": []})
+
+        return _FakeStreamSingleStop(_item)
+
+
+class _FakeOpenAIClient(_FakeAsyncClientBase):
+    def __init__(self, *args: Any, **kwargs: Any):
+        super().__init__(*args, **kwargs)
+        self.responses = _FakeResponses()
+
+
+@pytest.mark.asyncio
+async def test_response_backend_stream_closes_client(monkeypatch):
+    import sys
+
+    created: List[_FakeOpenAIClient] = []
+
+    def _factory(*args: Any, **kwargs: Any) -> _FakeOpenAIClient:
+        client = _FakeOpenAIClient(*args, **kwargs)
+        created.append(client)
+        return client
+
+    # Inject fake openai module so in-function import resolves to our factory
+    monkeypatch.setitem(sys.modules, "openai", SimpleNamespace(AsyncOpenAI=_factory))
+
+    backend = ResponseBackend()
+
+    messages = [{"role": "user", "content": "hi"}]
+
+    # Drain the stream
+    async for _ in backend.stream_with_tools(messages, tools=[], model="gpt-4o-mini"):
+        pass
+
+    assert len(created) == 1
+    # Expectation: backend should close client to avoid event-loop errors
+    assert created[0]._closed is True
+
+
+# ---- GrokBackend test ----
+class _FakeChatCompletions:
+    async def create(self, **kwargs: Any):
+        # Yield a single finishing chunk similar to Chat Completions
+        def _item():
+            choice = SimpleNamespace(delta=None, finish_reason="stop")
+            return SimpleNamespace(choices=[choice], usage=None)
+
+        return _FakeStreamSingleStop(_item)
+
+
+class _FakeOpenAIClientForGrok(_FakeAsyncClientBase):
+    def __init__(self, *args: Any, **kwargs: Any):
+        super().__init__(*args, **kwargs)
+        self.chat = SimpleNamespace(completions=_FakeChatCompletions())
+
+
+@pytest.mark.asyncio
+async def test_grok_backend_stream_closes_client(monkeypatch):
+    import sys
+
+    created: List[_FakeOpenAIClientForGrok] = []
+
+    def _factory(*args: Any, **kwargs: Any) -> _FakeOpenAIClientForGrok:
+        client = _FakeOpenAIClientForGrok(*args, **kwargs)
+        created.append(client)
+        return client
+
+    # Inject fake openai module for dynamic import inside function
+    monkeypatch.setitem(sys.modules, "openai", SimpleNamespace(AsyncOpenAI=_factory))
+
+    backend = GrokBackend()
+    messages = [{"role": "user", "content": "hi"}]
+
+    async for _ in backend.stream_with_tools(messages, tools=[], model="grok-2-mini"):
+        pass
+
+    assert len(created) == 1
+    # Expectation: backend should close client to avoid event-loop errors
+    assert created[0]._closed is True
+
+
+# ---- ClaudeBackend test ----
+class _FakeClaudeMessages:
+    async def create(self, **kwargs: Any):
+        # Stream that yields a single message_stop event
+        def _item():
+            return SimpleNamespace(type="message_stop")
+
+        return _FakeStreamSingleStop(_item)
+
+
+class _FakeAnthropicClient(_FakeAsyncClientBase):
+    def __init__(self, *args: Any, **kwargs: Any):
+        super().__init__(*args, **kwargs)
+        # Provide both .messages and .beta.messages to cover branches
+        self.messages = _FakeClaudeMessages()
+        self.beta = SimpleNamespace(messages=_FakeClaudeMessages())
+
+
+@pytest.mark.asyncio
+async def test_claude_backend_stream_closes_client(monkeypatch):
+    import sys
+
+    created: List[_FakeAnthropicClient] = []
+
+    def _factory(*args: Any, **kwargs: Any) -> _FakeAnthropicClient:
+        client = _FakeAnthropicClient(*args, **kwargs)
+        created.append(client)
+        return client
+
+    # Inject fake anthropic module for dynamic import inside function
+    monkeypatch.setitem(sys.modules, "anthropic", SimpleNamespace(AsyncAnthropic=_factory))
+
+    backend = ClaudeBackend()
+    messages = [{"role": "user", "content": "hi"}]
+
+    async for _ in backend.stream_with_tools(messages, tools=[], model="claude-3.7-sonnet"):
+        pass
+
+    assert len(created) == 1
+    # Expectation: backend should close client to avoid event-loop errors
+    assert created[0]._closed is True