composo 0.0.11__tar.gz

composo-0.0.11/MANIFEST.in

@@ -0,0 +1,22 @@
+prune tests
+global-exclude test_*
+global-exclude *test.py
+global-exclude *test_*.py
+global-exclude .pytest_cache/
+global-exclude .coverage
+global-exclude htmlcov/
+global-exclude .tox/
+global-exclude .mypy_cache/
+global-exclude .eggs/
+global-exclude *.egg-info/
+global-exclude build/
+global-exclude dist/
+global-exclude .git/
+global-exclude .gitignore
+global-exclude .editorconfig
+global-exclude .flake8
+global-exclude .isort.cfg
+global-exclude .mypy.ini
+global-exclude pyproject.toml.bak
+global-exclude build_and_publish.sh
+global-exclude requirements.txt 

composo-0.0.11/PKG-INFO

@@ -0,0 +1,162 @@
+Metadata-Version: 2.4
+Name: composo
+Version: 0.0.11
+Summary: Composo Python SDK
+Author-email: Your Name <your.email@example.com>
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+Requires-Dist: httpx>=0.21.0
+Requires-Dist: pydantic<3.0.0,>=1.9.2
+Requires-Dist: typing-extensions>=4.0.0
+Requires-Dist: deepdiff>=6.0.0
+Requires-Dist: openai>=1.0.0
+Requires-Dist: anthropic>=0.25.0
+Requires-Dist: tenacity>=8.0.0
+Provides-Extra: test
+Requires-Dist: pytest>=7.0.0; extra == "test"
+Requires-Dist: pytest-asyncio>=0.21.0; extra == "test"
+Requires-Dist: pytest-cov>=4.0.0; extra == "test"
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0.0; extra == "dev"
+Requires-Dist: pytest-asyncio>=0.21.0; extra == "dev"
+Requires-Dist: pytest-cov>=4.0.0; extra == "dev"
+Requires-Dist: black>=23.0.0; extra == "dev"
+Requires-Dist: isort>=5.12.0; extra == "dev"
+Requires-Dist: mypy>=1.0.0; extra == "dev"
+Requires-Dist: flake8>=6.0.0; extra == "dev"
+
+# Composo Python SDK
+
+A Python SDK for Composo evaluation services, providing both synchronous and asynchronous clients for evaluating LLM conversations with support for OpenAI and Anthropic formats.
+
+## Features
+
+- **Dual Client Support**: Both synchronous and asynchronous clients
+- **Multiple LLM Provider Support**: Native support for OpenAI and Anthropic formats
+- **Connection Pooling**: Optimized HTTP client with connection reuse
+- **Retry Logic**: Exponential backoff with jitter for robust API calls
+- **Type Safety**: Full type hints and Pydantic models
+- **Context Managers**: Proper resource management with context managers
+
+## Installation
+
+```bash
+pip install composo
+```
+
+## Quick Start
+
+### Basic Usage
+
+```python
+from composo import Composo, AsyncComposo
+
+# Initialize client
+client = Composo(api_key="your-api-key")
+
+# Evaluate messages
+messages = [
+    {"role": "user", "content": "What is machine learning?"},
+    {"role": "assistant", "content": "Machine learning is..."}
+]
+
+criteria = ["Reward responses that provide accurate technical explanations"]
+
+result = client.evaluate(messages=messages, criteria=criteria)
+print(f"Score: {result.score}")
+print(f"Explanation: {result.explanation}")
+```
+
+### Async Usage
+
+```python
+import asyncio
+from composo import AsyncComposo
+
+async def main():
+    async with AsyncComposo(api_key="your-api-key") as client:
+        result = await client.evaluate(
+            messages=messages,
+            criteria=criteria
+        )
+        print(f"Score: {result.score}")
+
+asyncio.run(main())
+```
+
+### With LLM Results
+
+```python
+import openai
+from composo import Composo
+
+# Get response from OpenAI
+openai_client = openai.OpenAI(api_key="your-openai-key")
+openai_result = openai_client.chat.completions.create(
+    model="gpt-4",
+    messages=[{"role": "user", "content": "What is machine learning?"}]
+)
+
+# Evaluate the response
+composo_client = Composo(api_key="your-composo-key")
+eval_result = composo_client.evaluate(
+    messages=[{"role": "user", "content": "What is machine learning?"}],
+    result=openai_result,
+    criteria=["Reward accurate technical explanations"]
+)
+```
+
+## Configuration
+
+### Client Options
+
+- `api_key` (required): Your Composo API key
+- `base_url` (optional): Custom API endpoint (default: https://platform.composo.ai)
+- `num_retries` (optional): Number of retry attempts (default: 1)
+- `model_core` (optional): Specific model core for evaluation
+
+### Logging
+
+The SDK uses Python's standard logging module. Configure logging level:
+
+```python
+import logging
+logging.getLogger("composo").setLevel(logging.INFO)
+```
+
+## Error Handling
+
+The SDK provides specific exception types:
+
+```python
+from composo import (
+    ComposoError,
+    RateLimitError,
+    MalformedError,
+    APIError,
+    AuthenticationError
+)
+
+try:
+    result = client.evaluate(messages=messages, criteria=criteria)
+except RateLimitError:
+    print("Rate limit exceeded")
+except AuthenticationError:
+    print("Invalid API key")
+except ComposoError as e:
+    print(f"Composo error: {e}")
+```
+
+## Performance Optimization
+
+- **Connection Pooling**: HTTP clients reuse connections for better performance
+- **Context Managers**: Use context managers to properly close connections
+- **Async Support**: Use async client for high-throughput scenarios

composo-0.0.11/README.md

@@ -0,0 +1,127 @@
+# Composo Python SDK
+
+A Python SDK for Composo evaluation services, providing both synchronous and asynchronous clients for evaluating LLM conversations with support for OpenAI and Anthropic formats.
+
+## Features
+
+- **Dual Client Support**: Both synchronous and asynchronous clients
+- **Multiple LLM Provider Support**: Native support for OpenAI and Anthropic formats
+- **Connection Pooling**: Optimized HTTP client with connection reuse
+- **Retry Logic**: Exponential backoff with jitter for robust API calls
+- **Type Safety**: Full type hints and Pydantic models
+- **Context Managers**: Proper resource management with context managers
+
+## Installation
+
+```bash
+pip install composo
+```
+
+## Quick Start
+
+### Basic Usage
+
+```python
+from composo import Composo, AsyncComposo
+
+# Initialize client
+client = Composo(api_key="your-api-key")
+
+# Evaluate messages
+messages = [
+    {"role": "user", "content": "What is machine learning?"},
+    {"role": "assistant", "content": "Machine learning is..."}
+]
+
+criteria = ["Reward responses that provide accurate technical explanations"]
+
+result = client.evaluate(messages=messages, criteria=criteria)
+print(f"Score: {result.score}")
+print(f"Explanation: {result.explanation}")
+```
+
+### Async Usage
+
+```python
+import asyncio
+from composo import AsyncComposo
+
+async def main():
+    async with AsyncComposo(api_key="your-api-key") as client:
+        result = await client.evaluate(
+            messages=messages,
+            criteria=criteria
+        )
+        print(f"Score: {result.score}")
+
+asyncio.run(main())
+```
+
+### With LLM Results
+
+```python
+import openai
+from composo import Composo
+
+# Get response from OpenAI
+openai_client = openai.OpenAI(api_key="your-openai-key")
+openai_result = openai_client.chat.completions.create(
+    model="gpt-4",
+    messages=[{"role": "user", "content": "What is machine learning?"}]
+)
+
+# Evaluate the response
+composo_client = Composo(api_key="your-composo-key")
+eval_result = composo_client.evaluate(
+    messages=[{"role": "user", "content": "What is machine learning?"}],
+    result=openai_result,
+    criteria=["Reward accurate technical explanations"]
+)
+```
+
+## Configuration
+
+### Client Options
+
+- `api_key` (required): Your Composo API key
+- `base_url` (optional): Custom API endpoint (default: https://platform.composo.ai)
+- `num_retries` (optional): Number of retry attempts (default: 1)
+- `model_core` (optional): Specific model core for evaluation
+
+### Logging
+
+The SDK uses Python's standard logging module. Configure logging level:
+
+```python
+import logging
+logging.getLogger("composo").setLevel(logging.INFO)
+```
+
+## Error Handling
+
+The SDK provides specific exception types:
+
+```python
+from composo import (
+    ComposoError,
+    RateLimitError,
+    MalformedError,
+    APIError,
+    AuthenticationError
+)
+
+try:
+    result = client.evaluate(messages=messages, criteria=criteria)
+except RateLimitError:
+    print("Rate limit exceeded")
+except AuthenticationError:
+    print("Invalid API key")
+except ComposoError as e:
+    print(f"Composo error: {e}")
+```
+
+## Performance Optimization
+
+- **Connection Pooling**: HTTP clients reuse connections for better performance
+- **Context Managers**: Use context managers to properly close connections
+- **Async Support**: Use async client for high-throughput scenarios

composo-0.0.11/composo/__init__.py

@@ -0,0 +1,69 @@
+"""
+Composo SDK - A Python SDK for Composo evaluation services
+
+This package provides both synchronous and asynchronous clients for evaluating
+LLM conversations with support for OpenAI and Anthropic formats.
+"""
+
+__version__ = "0.1.0"
+__author__ = "Composo Team"
+__email__ = "support@composo.ai"
+__description__ = "A Python SDK for Composo evaluation services"
+
+from .client import Composo, AsyncComposo
+from .exceptions import (
+    ComposoError,
+    RateLimitError,
+    MalformedError,
+    APIError,
+    AuthenticationError,
+)
+from .validation import validate_raw_chat_conforms_to_type
+from .chat_types import OpenAIChatSessionType, AnthropicChatSessionType, ChatSessionType
+from .models import CriteriaSet
+
+
+# Create a criteria module-like object for backward compatibility
+class CriteriaModule:
+    """Module-like object for accessing predefined criteria sets"""
+
+    @property
+    def basic(self):
+        return CriteriaSet.basic
+
+    @property
+    def rag(self):
+        return CriteriaSet.rag
+
+
+# Create a singleton instance
+criteria = CriteriaModule()
+
+
+# Package exports
+__all__ = [
+    # Main clients
+    "Composo",
+    "AsyncComposo",
+    # Exceptions
+    "ComposoError",
+    "RateLimitError",
+    "MalformedError",
+    "APIError",
+    "AuthenticationError",
+    # Validation
+    "validate_raw_chat_conforms_to_type",
+    # Types
+    "OpenAIChatSessionType",
+    "AnthropicChatSessionType",
+    "ChatSessionType",
+    # Criteria libraries
+    "CriteriaSet",
+    "criteria",
+    # Metadata
+    "__version__",
+]
+
+
+# Welcome message - removed for performance
+# print(f"🚀 Composo SDK v{__version__} loaded successfully!")

composo-0.0.11/composo/adapters/__init__.py

@@ -0,0 +1,15 @@
+"""
+Format adapters for different LLM providers
+"""
+
+from .base import FormatAdapter
+from .openai_adapter import OpenAIAdapter
+from .anthropic_adapter import AnthropicAdapter
+from .factory import AdapterFactory
+
+__all__ = [
+    "FormatAdapter",
+    "OpenAIAdapter",
+    "AnthropicAdapter",
+    "AdapterFactory",
+]

composo-0.0.11/composo/adapters/anthropic_adapter.py

@@ -0,0 +1,31 @@
+"""
+Anthropic format adapter
+"""
+
+from typing import List, Dict, Any, Optional
+from .base import FormatAdapter
+from ..client.types import MessagesType, ToolsType, ResultType
+
+# Import Anthropic types for type checking
+from anthropic.types.message import Message
+
+
+class AnthropicAdapter(FormatAdapter):
+    """Adapter for Anthropic format"""
+
+    def can_handle(self, result: ResultType) -> bool:
+        """Check if result is Anthropic Message using type checking"""
+        return isinstance(result, Message)
+
+    def process_result(
+        self,
+        messages: MessagesType,
+        result: ResultType,
+        system: Optional[str] = None,
+        tools: ToolsType = None,
+    ) -> tuple[List[Dict[str, Any]], Optional[str], Optional[List[Dict[str, Any]]]]:
+        if not self.can_handle(result):
+            return messages, system, tools
+        # Match notebook behavior: directly append without field filtering, excluding None values
+        messages.append(result.model_dump(exclude_none=True))
+        return messages, system, tools

composo-0.0.11/composo/adapters/base.py

@@ -0,0 +1,32 @@
+"""
+Base adapter interface
+"""
+
+from abc import ABC, abstractmethod
+from typing import List, Dict, Any, Optional
+from ..client.types import MessagesType, ToolsType, ResultType
+
+
+class FormatAdapter(ABC):
+    """Abstract base class for format adapters"""
+
+    @abstractmethod
+    def process_result(
+        self,
+        messages: MessagesType,
+        result: ResultType,
+        system: Optional[str] = None,
+        tools: ToolsType = None,
+    ) -> tuple[List[Dict[str, Any]], Optional[str], Optional[List[Dict[str, Any]]]]:
+        """
+        Process LLM result and append to messages
+
+        Returns:
+            tuple: (updated_messages, system_message, tools)
+        """
+        pass
+
+    @abstractmethod
+    def can_handle(self, result: ResultType) -> bool:
+        """Check if this adapter can handle the given result type"""
+        pass

composo-0.0.11/composo/adapters/factory.py

@@ -0,0 +1,35 @@
+"""
+Adapter factory for creating appropriate format adapters
+"""
+
+from typing import Any, Optional
+from .base import FormatAdapter
+from .openai_adapter import OpenAIAdapter
+from .anthropic_adapter import AnthropicAdapter
+
+
+class AdapterFactory:
+    """Factory for creating format adapters based on result type"""
+
+    _adapters = [
+        OpenAIAdapter(),
+        AnthropicAdapter(),
+    ]
+
+    @classmethod
+    def get_adapter(cls, result: Any) -> Optional[FormatAdapter]:
+        """Get appropriate adapter for the given result"""
+        for adapter in cls._adapters:
+            if adapter.can_handle(result):
+                return adapter
+        return None
+
+    @classmethod
+    def register_adapter(cls, adapter: FormatAdapter) -> None:
+        """Register a new adapter"""
+        cls._adapters.insert(0, adapter)  # Insert at beginning for priority
+
+    @classmethod
+    def get_all_adapters(cls) -> list[FormatAdapter]:
+        """Get all registered adapters"""
+        return cls._adapters.copy()

composo-0.0.11/composo/adapters/openai_adapter.py

@@ -0,0 +1,40 @@
+"""
+OpenAI format adapter
+"""
+
+from typing import List, Dict, Any, Optional
+from .base import FormatAdapter
+from ..client.types import MessagesType, ToolsType, ResultType
+
+# Import OpenAI types for type checking
+from openai.types.chat.chat_completion import ChatCompletion
+
+
+class OpenAIAdapter(FormatAdapter):
+    """Adapter for OpenAI format"""
+
+    def can_handle(self, result: ResultType) -> bool:
+        """Check if this adapter can handle the given result type"""
+        # Check if result is OpenAI ChatCompletion using type checking
+        return isinstance(result, ChatCompletion)
+
+    def process_result(
+        self,
+        messages: MessagesType,
+        result: ResultType,
+        system: Optional[str] = None,
+        tools: ToolsType = None,
+    ) -> tuple[List[Dict[str, Any]], Optional[str], Optional[List[Dict[str, Any]]]]:
+        """Process OpenAI format result by preserving original structure"""
+        if not self.can_handle(result):
+            return messages, system, tools
+
+        # Extract the message from OpenAI result and preserve its structure
+        if hasattr(result, "choices") and result.choices:
+            choice = result.choices[0]
+            if hasattr(choice, "message"):
+                # Use model_dump() to preserve the original structure, excluding None values
+                message_dict = choice.message.model_dump(exclude_none=True)
+                messages.append(message_dict)
+
+        return messages, system, tools

composo-0.0.11/composo/chat_types.py

@@ -0,0 +1,46 @@
+"""
+Type definitions for chat session types
+"""
+
+from typing import List, Union
+from typing_extensions import TypedDict, Required
+
+# Import the actual types if available, otherwise use generic types
+try:
+    from openai.types.chat.chat_completion_message_param import (
+        ChatCompletionMessageParam,
+    )
+    from openai.types.chat.chat_completion_tool_param import ChatCompletionToolParam
+except ImportError:
+    # Fallback to generic types if OpenAI types are not available
+    ChatCompletionMessageParam = dict
+    ChatCompletionToolParam = dict
+
+try:
+    from anthropic.types.message_param import MessageParam
+    from anthropic.types.text_block_param import TextBlockParam
+    from anthropic.types.tool_union_param import ToolUnionParam
+except ImportError:
+    # Fallback to generic types if Anthropic types are not available
+    MessageParam = dict
+    TextBlockParam = dict
+    ToolUnionParam = dict
+
+
+class OpenAIChatSessionType(TypedDict):
+    """Container for OpenAI messages with 'messages' key."""
+
+    messages: Required[List[ChatCompletionMessageParam]]
+    tools: List[ChatCompletionToolParam]
+
+
+class AnthropicChatSessionType(TypedDict, total=False):
+    """Container for Anthropic messages with 'messages' key."""
+
+    messages: Required[List[MessageParam]]
+    system: Union[str, List[TextBlockParam]]
+    tools: List[ToolUnionParam]
+
+
+# Union type for when you don't know the type of the chat session
+ChatSessionType = Union[OpenAIChatSessionType, AnthropicChatSessionType]

composo-0.0.11/composo/client/__init__.py

@@ -0,0 +1,17 @@
+"""
+Client implementations for Composo SDK
+"""
+
+from .sync import Composo
+from .async_client import AsyncComposo
+from .types import MessagesType, ToolsType, ResultType, MessageType, ToolType
+
+__all__ = [
+    "Composo",
+    "AsyncComposo",
+    "MessagesType",
+    "ToolsType",
+    "ResultType",
+    "MessageType",
+    "ToolType",
+]