camel-ai 0.2.21__py3-none-any.whl → 0.2.23a0__py3-none-any.whl

camel/utils/commons.py

@@ -19,6 +19,7 @@ import platform
 import re
 import socket
 import subprocess
+import threading
 import time
 import zipfile
 from functools import wraps
@@ -306,10 +307,55 @@ def api_keys_required(
                 if not value or value.strip() == "":
                     missing_keys.append(env_var_name)
 
+            key_way = "the official website"
+            if env_var_name == 'ANTHROPIC_API_KEY':
+                key_way = (
+                    "https://docs.anthropic.com/zh-CN/api/getting-started"
+                )
+            elif env_var_name == 'AIML_API_KEY':
+                key_way = "https://aimlapi.com/"
+            elif env_var_name == 'COHERE_API_KEY':
+                key_way = "https://cohere.com/"
+            elif env_var_name == 'DEEPSEEK_API_KEY':
+                key_way = "https://www.deepseek.com/"
+            elif env_var_name == 'AZURE_OPENAI_API_KEY':
+                key_way = "https://portal.azure.com/"
+            elif env_var_name == 'OPENAI_API_KEY':
+                key_way = "https://platform.openai.com/docs/overview"
+            elif env_var_name == 'FISHAUDIO_API_KEY':
+                key_way = "https://fish.audio/"
+            elif env_var_name == 'GEMINI_API_KEY':
+                key_way = "https://gemini.google.com/"
+            elif env_var_name == 'INTERNLM_API_KEY':
+                key_way = "https://internlm-chat.intern-ai.org.cn/puyu/api/v1"
+            elif env_var_name == 'GROQ_API_KEY':
+                key_way = "https://api.groq.com/openai/v1"
+            elif env_var_name == 'MISTRAL_API_KEY':
+                key_way = "https://mistral.ai/"
+            elif env_var_name == 'MOONSHOT_API_KEY':
+                key_way = "https://api.moonshot.cn/v1"
+            elif env_var_name == 'NVIDIA_API_KEY':
+                key_way = "https://integrate.api.nvidia.com/"
+            elif env_var_name == 'OPENAI_COMPATIBILIY_API_KEY':
+                key_way = "https://platform.openai.com/docs/overview"
+            elif env_var_name == 'QWEN_API_KEY':
+                key_way = "https://tongyi.aliyun.com/"
+            elif env_var_name == 'REKA_API_KEY':
+                key_way = "https://docs.reka.ai/quick-start"
+            elif env_var_name == 'SAMBA_API_KEY':
+                key_way = "https://community.sambanova.ai/t/looking-for-api-key-and-url-for-sambanova/576"
+            elif env_var_name == 'TOGETHER_API_KEY':
+                key_way = "https://docs.together.ai/docs/quickstart"
+            elif env_var_name == 'YI_API_KEY':
+                key_way = "https://platform.lingyiwanwu.com/docs"
+            elif env_var_name == 'ZHIPUAI_API_KEY':
+                key_way = "https://www.zhipuai.cn/"
+
             if missing_keys:
                 raise ValueError(
                     "Missing or empty required API keys in "
-                    f"environment variables: {', '.join(missing_keys)}"
+                    f"environment variables: {', '.join(missing_keys)}.\n"
+                    f"You can obtain the API key from {key_way}"
                 )
             return func(*args, **kwargs)
 
@@ -905,3 +951,70 @@ def generate_prompt_for_structured_output(
     {user_prompt}
     """
     return final_prompt
+
+
+def with_timeout(timeout=None):
+    r"""Decorator that adds timeout functionality to functions.
+
+    Executes functions with a specified timeout value. Returns a timeout
+    message if execution time is exceeded.
+
+    Args:
+        timeout (float, optional): The timeout duration in seconds. If None,
+            will try to get timeout from the instance's timeout attribute.
+            (default: :obj:`None`)
+
+    Example:
+        >>> @with_timeout(5)
+        ... def my_function():
+        ...     return "Success"
+        >>> my_function()
+
+        >>> class MyClass:
+        ...     timeout = 5
+        ...     @with_timeout()
+        ...     def my_method(self):
+        ...         return "Success"
+    """
+
+    def decorator(func):
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs):
+            # Determine the effective timeout value
+            effective_timeout = timeout
+            if effective_timeout is None and args:
+                effective_timeout = getattr(args[0], 'timeout', None)
+
+            # If no timeout value is provided, execute function normally
+            if effective_timeout is None:
+                return func(*args, **kwargs)
+
+            # Container to hold the result of the function call
+            result_container = []
+
+            def target():
+                result_container.append(func(*args, **kwargs))
+
+            # Start the function in a new thread
+            thread = threading.Thread(target=target)
+            thread.start()
+            thread.join(effective_timeout)
+
+            # Check if the thread is still alive after the timeout
+            if thread.is_alive():
+                return (
+                    f"Function `{func.__name__}` execution timed out, "
+                    f"exceeded {effective_timeout} seconds."
+                )
+            else:
+                return result_container[0]
+
+        return wrapper
+
+    # Handle both @with_timeout and @with_timeout() usage
+    if callable(timeout):
+        # If timeout is passed as a function, apply it to the decorator
+        func, timeout = timeout, None
+        return decorator(func)
+
+    return decorator

camel/verifiers/__init__.py

@@ -0,0 +1,23 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+from .base import BaseVerifier
+from .models import VerificationOutcome, VerifierInput
+from .python_verifier import PythonVerifier
+
+__all__ = [
+    "BaseVerifier",
+    "VerificationOutcome",
+    "VerifierInput",
+    "PythonVerifier",
+]

camel/verifiers/base.py

@@ -0,0 +1,340 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+import asyncio
+import time
+from abc import ABC, abstractmethod
+from typing import List, Optional
+
+from camel.logger import get_logger
+from camel.utils import BatchProcessor
+
+from .models import (
+    VerificationOutcome,
+    VerificationResult,
+    VerifierInput,
+)
+
+logger = get_logger(__name__)
+
+
+class BaseVerifier(ABC):
+    r"""Base class for all verifiers.
+
+    Example:
+        ```python
+        verifier = MyVerifier()
+        await verifier.setup()
+        result = await verifier.verify(response)
+        await verifier.cleanup()
+        ```
+
+    Key Features:
+    - Async verification with retry logic
+    - Comprehensive error handling and logging
+    - Configurable batch processing
+    - Resource monitoring for adaptive scaling
+    """
+
+    def __init__(
+        self,
+        max_parallel: Optional[int] = None,
+        timeout: Optional[float] = None,
+        max_retries: int = 3,
+        retry_delay: float = 1.0,
+        initial_batch_size: Optional[int] = None,
+        cpu_threshold: float = 80.0,
+        memory_threshold: float = 85.0,
+        **kwargs,
+    ):
+        r"""Initialize the verifier with configuration parameters.
+
+        Args:
+            max_parallel: Maximum number of parallel verifications. If None,
+                determined dynamically based on system resources.
+                (default: :obj:`None`)
+            timeout: Timeout in seconds for each verification. (default:
+                :obj:`None`)
+            max_retries: Maximum number of retry attempts. (default: :obj:`3`)
+            retry_delay: Delay between retries in seconds. (default:
+                :obj:`1.0`)
+            initial_batch_size: Initial size for batch processing. If None,
+                defaults to 10. (default: :obj:`None`)
+            cpu_threshold: CPU usage percentage threshold for scaling down.
+                (default: :obj:`80.0`)
+            memory_threshold: Memory usage percentage threshold for scaling
+                down. (default: :obj:`85.0`)
+            **kwargs: Additional verifier parameters.
+        """
+        self._is_setup: bool = False
+        self._max_parallel: Optional[int] = max_parallel
+        self._timeout: Optional[float] = timeout
+        self._max_retries: int = max_retries
+        self._retry_delay: float = retry_delay
+        self._initial_batch_size: Optional[int] = initial_batch_size
+        self._cpu_threshold: float = cpu_threshold
+        self._memory_threshold: float = memory_threshold
+        self._batch_processor: BatchProcessor = BatchProcessor()
+
+    async def setup(self) -> None:
+        r"""Set up the verifier with necessary resources.
+
+        Initializes:
+        1. Batch processor with validated parameters
+        2. Any verifier-specific resources
+
+        Raises:
+            RuntimeError: If setup fails or resources cannot be initialized.
+        """
+        if self._is_setup:
+            logger.debug(f"{self.__class__.__name__} already initialized")
+            return
+
+        try:
+            batch_size = max(1, self._initial_batch_size or 10)
+            max_parallel = max(1, self._max_parallel or 1)
+            self._batch_processor = BatchProcessor()
+
+            logger.info(
+                f"{self.__class__.__name__} initialized with "
+                f"batch_size={batch_size}, max_parallel={max_parallel}"
+            )
+
+            await self._setup()
+            self._is_setup = True
+
+        except Exception as e:
+            error_msg = (
+                f"Failed to initialize {self.__class__.__name__}: {e!s}"
+            )
+            logger.error(error_msg, exc_info=True)
+            await self.cleanup()
+            raise RuntimeError(error_msg) from e
+
+    @abstractmethod
+    async def _setup(self) -> None:
+        r"""Implement verifier-specific setup logic."""
+        pass
+
+    async def cleanup(self) -> None:
+        r"""Clean up verifier resources.
+
+        Ensures:
+        1. Batch processor is reset
+        2. All internal states are cleared
+
+        Raises:
+            RuntimeError: If cleanup fails.
+        """
+        if not self._is_setup:
+            return
+
+        try:
+            self._batch_processor = BatchProcessor()
+            await self._cleanup()
+            logger.info(f"{self.__class__.__name__} cleaned up successfully")
+
+        except Exception as e:
+            error_msg = f"Failed to cleanup {self.__class__.__name__}: {e!s}"
+            logger.error(error_msg, exc_info=True)
+            raise RuntimeError(error_msg) from e
+
+        finally:
+            self._is_setup = False
+
+    @abstractmethod
+    async def _cleanup(self) -> None:
+        r"""Implement verifier-specific cleanup logic."""
+        pass
+
+    async def verify(self, result: VerifierInput) -> VerificationResult:
+        r"""Perform verification with full error handling.
+
+        Verifies correctness, expected output, reasoning, and symbolic
+        consistency.
+
+        Args:
+            result: The response to verify.
+
+        Returns:
+            VerificationResult: Structured result containing:
+                - status: SUCCESS/FAILURE/ERROR/TIMEOUT
+                - result: Verification outcome description
+                - duration: Time taken for verification
+                - metadata: Additional details
+                - error_message: Error description if applicable
+
+        Raises:
+            RuntimeError: If verification fails unexpectedly.
+            asyncio.TimeoutError: If verification times out.
+        """
+        if not self._is_setup:
+            logger.warning(
+                f"{self.__class__.__name__} not set up, calling setup()"
+            )
+            await self.setup()
+
+        attempt = 0
+        start_time = time.time()
+
+        while attempt < self._max_retries:
+            try:
+                verification_result = (
+                    await asyncio.wait_for(
+                        self._verify_implementation(result),
+                        timeout=self._timeout,
+                    )
+                    if self._timeout
+                    else await self._verify_implementation(result)
+                )
+
+                verification_result.duration = time.time() - start_time
+                verification_result.metadata["attempt"] = attempt + 1
+                return verification_result
+
+            except asyncio.TimeoutError:
+                attempt += 1
+                if attempt == self._max_retries:
+                    return VerificationResult(
+                        status=VerificationOutcome.TIMEOUT,
+                        result="",
+                        error_message="Verification timed out "
+                        "after all retries.",
+                        duration=time.time() - start_time,
+                        metadata={"attempt": attempt},
+                    )
+                logger.warning(
+                    f"Verification timeout on attempt {attempt}, retrying..."
+                )
+                await asyncio.sleep(self._retry_delay)
+
+            except Exception as e:
+                attempt += 1
+                if attempt == self._max_retries:
+                    return VerificationResult(
+                        status=VerificationOutcome.ERROR,
+                        result="",
+                        error_message=f"Verification failed: {e!s}",
+                        duration=time.time() - start_time,
+                        metadata={"attempt": attempt},
+                    )
+                await asyncio.sleep(self._retry_delay)
+
+        return VerificationResult(
+            status=VerificationOutcome.ERROR,
+            result="",
+            error_message="Unexpected code path reached",
+            duration=time.time() - start_time,
+            metadata={"attempt": attempt},
+        )
+
+    @abstractmethod
+    async def _verify_implementation(
+        self, result: VerifierInput
+    ) -> VerificationResult:
+        r"""Implement the actual verification logic.
+
+        Args:
+            result: The response to verify.
+
+        Returns:
+            VerificationResult: Containing the verification outcome.
+
+        Raises:
+            NotImplementedError: Must be implemented in subclasses.
+        """
+        raise NotImplementedError(
+            "Subclasses must implement _verify_implementation()"
+        )
+
+
+async def verify_batch(
+    self, results: List[VerifierInput], raise_on_error: bool = False
+) -> List[VerificationResult]:
+    r"""Verify multiple results in parallel with controlled concurrency.
+
+    Args:
+        results: List of responses to verify.
+        raise_on_error: Whether to raise an exception if any verification
+            fails. (default: :obj:`False`)
+
+    Returns:
+        List[VerificationResult]: One for each input response.
+
+    Raises:
+        RuntimeError: If any verification fails and raise_on_error is True.
+        asyncio.TimeoutError: If verifications time out and max retries
+            exceeded.
+    """
+    if not self._is_setup:
+        logger.warning(
+            f"{self.__class__.__name__} not set up, calling setup()"
+        )
+        await self.setup()
+
+    # Get current batch parameters from processor with defaults if not
+    #  present
+    max_workers = getattr(
+        self._batch_processor, 'max_workers', self._max_parallel or 1
+    )
+    batch_size = getattr(
+        self._batch_processor, 'batch_size', self._initial_batch_size or 10
+    )
+    semaphore = asyncio.Semaphore(max(1, max_workers))
+
+    async def _verify_with_semaphore(
+        response: VerifierInput,
+    ) -> VerificationResult:
+        start_time = time.time()
+        try:
+            async with semaphore:
+                verification_result = await self.verify(response)
+            processing_time = time.time() - start_time
+            success = verification_result.status == VerificationOutcome.SUCCESS
+            self._batch_processor.adjust_batch_size(success, processing_time)
+            return verification_result
+        except Exception as e:
+            processing_time = time.time() - start_time
+            self._batch_processor.adjust_batch_size(False, processing_time)
+            logger.error(f"Verification failed: {e!s}", exc_info=True)
+            return VerificationResult(
+                status=VerificationOutcome.ERROR,
+                result="",
+                error_message=str(e),
+                metadata={"error_type": type(e).__name__},
+            )
+
+    # Process in batches
+    all_results: List[VerificationResult] = []
+    for i in range(0, len(results), batch_size):
+        batch = results[i : i + batch_size]
+        verification_tasks = [
+            _verify_with_semaphore(result) for result in batch
+        ]
+        try:
+            batch_results = await asyncio.gather(*verification_tasks)
+            all_results.extend(batch_results)
+        except Exception as e:
+            logger.error(f"Batch verification failed: {e!s}", exc_info=True)
+            if raise_on_error:
+                raise RuntimeError(f"Batch verification failed: {e!s}") from e
+
+    if raise_on_error and any(
+        r.status in {VerificationOutcome.ERROR, VerificationOutcome.TIMEOUT}
+        for r in all_results
+    ):
+        error_msg = "One or more verifications failed"
+        logger.error(error_msg)
+        raise RuntimeError(error_msg)
+
+    return all_results

camel/verifiers/models.py

@@ -0,0 +1,82 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+from datetime import datetime
+from enum import Enum
+from typing import Any, Dict, Optional
+
+from pydantic import BaseModel, Field
+
+
+class VerifierInput(BaseModel):
+    r"""Structured input to the verifier"""
+
+    llm_response: str = Field(
+        description="The LLM response to be verified."
+        "Needs to be in a format that the verifier can handle."
+    )
+    ground_truth: Optional[str] = Field(
+        None, description="The ground truth data, if available."
+    )
+
+
+class VerificationOutcome(Enum):
+    r"""Enum representing the status of a verification."""
+
+    SUCCESS = "success"
+    FAILURE = "failure"
+    ERROR = "error"
+    TIMEOUT = "timeout"
+
+    def __bool__(self):
+        r"""Only VerificationOutcome.SUCCESS is truthy; others are falsy."""
+        return self is VerificationOutcome.SUCCESS
+
+
+class VerificationResult(BaseModel):
+    r"""Structured result from a verification."""
+
+    status: VerificationOutcome = Field(
+        description="Status of the verification"
+    )
+    result: str = Field(description="Verification result")
+    duration: float = Field(
+        default=0.0, description="Duration of verification in seconds"
+    )
+    timestamp: datetime = Field(
+        default_factory=datetime.now,
+        description="When the verification was performed",
+    )
+    metadata: Dict[str, Any] = Field(
+        default_factory=dict,
+        description="Additional metadata about the verification",
+    )
+    error_message: Optional[str] = Field(
+        default=None, description="Error message if verification failed"
+    )
+
+
+class VerifierConfig(BaseModel):
+    r"""Configuration for verifier behavior."""
+
+    enabled: bool = Field(True, description="Whether verification is enabled")
+    strict_mode: bool = Field(
+        False, description="Whether to fail on any validation error"
+    )
+    timeout: Optional[float] = Field(
+        None, description="Verification timeout in seconds"
+    )
+    max_retries: int = Field(3, description="Maximum number of retry attempts")
+    retry_delay: float = Field(
+        1.0, description="Delay between retries in seconds"
+    )