camel-ai 0.2.22__py3-none-any.whl → 0.2.23__py3-none-any.whl

camel/verifiers/base.py

@@ -0,0 +1,340 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+import asyncio
+import time
+from abc import ABC, abstractmethod
+from typing import List, Optional
+
+from camel.logger import get_logger
+from camel.utils import BatchProcessor
+
+from .models import (
+    VerificationOutcome,
+    VerificationResult,
+    VerifierInput,
+)
+
+logger = get_logger(__name__)
+
+
+class BaseVerifier(ABC):
+    r"""Base class for all verifiers.
+
+    Example:
+        ```python
+        verifier = MyVerifier()
+        await verifier.setup()
+        result = await verifier.verify(response)
+        await verifier.cleanup()
+        ```
+
+    Key Features:
+    - Async verification with retry logic
+    - Comprehensive error handling and logging
+    - Configurable batch processing
+    - Resource monitoring for adaptive scaling
+    """
+
+    def __init__(
+        self,
+        max_parallel: Optional[int] = None,
+        timeout: Optional[float] = None,
+        max_retries: int = 3,
+        retry_delay: float = 1.0,
+        initial_batch_size: Optional[int] = None,
+        cpu_threshold: float = 80.0,
+        memory_threshold: float = 85.0,
+        **kwargs,
+    ):
+        r"""Initialize the verifier with configuration parameters.
+
+        Args:
+            max_parallel: Maximum number of parallel verifications. If None,
+                determined dynamically based on system resources.
+                (default: :obj:`None`)
+            timeout: Timeout in seconds for each verification. (default:
+                :obj:`None`)
+            max_retries: Maximum number of retry attempts. (default: :obj:`3`)
+            retry_delay: Delay between retries in seconds. (default:
+                :obj:`1.0`)
+            initial_batch_size: Initial size for batch processing. If None,
+                defaults to 10. (default: :obj:`None`)
+            cpu_threshold: CPU usage percentage threshold for scaling down.
+                (default: :obj:`80.0`)
+            memory_threshold: Memory usage percentage threshold for scaling
+                down. (default: :obj:`85.0`)
+            **kwargs: Additional verifier parameters.
+        """
+        self._is_setup: bool = False
+        self._max_parallel: Optional[int] = max_parallel
+        self._timeout: Optional[float] = timeout
+        self._max_retries: int = max_retries
+        self._retry_delay: float = retry_delay
+        self._initial_batch_size: Optional[int] = initial_batch_size
+        self._cpu_threshold: float = cpu_threshold
+        self._memory_threshold: float = memory_threshold
+        self._batch_processor: BatchProcessor = BatchProcessor()
+
+    async def setup(self) -> None:
+        r"""Set up the verifier with necessary resources.
+
+        Initializes:
+        1. Batch processor with validated parameters
+        2. Any verifier-specific resources
+
+        Raises:
+            RuntimeError: If setup fails or resources cannot be initialized.
+        """
+        if self._is_setup:
+            logger.debug(f"{self.__class__.__name__} already initialized")
+            return
+
+        try:
+            batch_size = max(1, self._initial_batch_size or 10)
+            max_parallel = max(1, self._max_parallel or 1)
+            self._batch_processor = BatchProcessor()
+
+            logger.info(
+                f"{self.__class__.__name__} initialized with "
+                f"batch_size={batch_size}, max_parallel={max_parallel}"
+            )
+
+            await self._setup()
+            self._is_setup = True
+
+        except Exception as e:
+            error_msg = (
+                f"Failed to initialize {self.__class__.__name__}: {e!s}"
+            )
+            logger.error(error_msg, exc_info=True)
+            await self.cleanup()
+            raise RuntimeError(error_msg) from e
+
+    @abstractmethod
+    async def _setup(self) -> None:
+        r"""Implement verifier-specific setup logic."""
+        pass
+
+    async def cleanup(self) -> None:
+        r"""Clean up verifier resources.
+
+        Ensures:
+        1. Batch processor is reset
+        2. All internal states are cleared
+
+        Raises:
+            RuntimeError: If cleanup fails.
+        """
+        if not self._is_setup:
+            return
+
+        try:
+            self._batch_processor = BatchProcessor()
+            await self._cleanup()
+            logger.info(f"{self.__class__.__name__} cleaned up successfully")
+
+        except Exception as e:
+            error_msg = f"Failed to cleanup {self.__class__.__name__}: {e!s}"
+            logger.error(error_msg, exc_info=True)
+            raise RuntimeError(error_msg) from e
+
+        finally:
+            self._is_setup = False
+
+    @abstractmethod
+    async def _cleanup(self) -> None:
+        r"""Implement verifier-specific cleanup logic."""
+        pass
+
+    async def verify(self, result: VerifierInput) -> VerificationResult:
+        r"""Perform verification with full error handling.
+
+        Verifies correctness, expected output, reasoning, and symbolic
+        consistency.
+
+        Args:
+            result: The response to verify.
+
+        Returns:
+            VerificationResult: Structured result containing:
+                - status: SUCCESS/FAILURE/ERROR/TIMEOUT
+                - result: Verification outcome description
+                - duration: Time taken for verification
+                - metadata: Additional details
+                - error_message: Error description if applicable
+
+        Raises:
+            RuntimeError: If verification fails unexpectedly.
+            asyncio.TimeoutError: If verification times out.
+        """
+        if not self._is_setup:
+            logger.warning(
+                f"{self.__class__.__name__} not set up, calling setup()"
+            )
+            await self.setup()
+
+        attempt = 0
+        start_time = time.time()
+
+        while attempt < self._max_retries:
+            try:
+                verification_result = (
+                    await asyncio.wait_for(
+                        self._verify_implementation(result),
+                        timeout=self._timeout,
+                    )
+                    if self._timeout
+                    else await self._verify_implementation(result)
+                )
+
+                verification_result.duration = time.time() - start_time
+                verification_result.metadata["attempt"] = attempt + 1
+                return verification_result
+
+            except asyncio.TimeoutError:
+                attempt += 1
+                if attempt == self._max_retries:
+                    return VerificationResult(
+                        status=VerificationOutcome.TIMEOUT,
+                        result="",
+                        error_message="Verification timed out "
+                        "after all retries.",
+                        duration=time.time() - start_time,
+                        metadata={"attempt": attempt},
+                    )
+                logger.warning(
+                    f"Verification timeout on attempt {attempt}, retrying..."
+                )
+                await asyncio.sleep(self._retry_delay)
+
+            except Exception as e:
+                attempt += 1
+                if attempt == self._max_retries:
+                    return VerificationResult(
+                        status=VerificationOutcome.ERROR,
+                        result="",
+                        error_message=f"Verification failed: {e!s}",
+                        duration=time.time() - start_time,
+                        metadata={"attempt": attempt},
+                    )
+                await asyncio.sleep(self._retry_delay)
+
+        return VerificationResult(
+            status=VerificationOutcome.ERROR,
+            result="",
+            error_message="Unexpected code path reached",
+            duration=time.time() - start_time,
+            metadata={"attempt": attempt},
+        )
+
+    @abstractmethod
+    async def _verify_implementation(
+        self, result: VerifierInput
+    ) -> VerificationResult:
+        r"""Implement the actual verification logic.
+
+        Args:
+            result: The response to verify.
+
+        Returns:
+            VerificationResult: Containing the verification outcome.
+
+        Raises:
+            NotImplementedError: Must be implemented in subclasses.
+        """
+        raise NotImplementedError(
+            "Subclasses must implement _verify_implementation()"
+        )
+
+
+async def verify_batch(
+    self, results: List[VerifierInput], raise_on_error: bool = False
+) -> List[VerificationResult]:
+    r"""Verify multiple results in parallel with controlled concurrency.
+
+    Args:
+        results: List of responses to verify.
+        raise_on_error: Whether to raise an exception if any verification
+            fails. (default: :obj:`False`)
+
+    Returns:
+        List[VerificationResult]: One for each input response.
+
+    Raises:
+        RuntimeError: If any verification fails and raise_on_error is True.
+        asyncio.TimeoutError: If verifications time out and max retries
+            exceeded.
+    """
+    if not self._is_setup:
+        logger.warning(
+            f"{self.__class__.__name__} not set up, calling setup()"
+        )
+        await self.setup()
+
+    # Get current batch parameters from processor with defaults if not
+    #  present
+    max_workers = getattr(
+        self._batch_processor, 'max_workers', self._max_parallel or 1
+    )
+    batch_size = getattr(
+        self._batch_processor, 'batch_size', self._initial_batch_size or 10
+    )
+    semaphore = asyncio.Semaphore(max(1, max_workers))
+
+    async def _verify_with_semaphore(
+        response: VerifierInput,
+    ) -> VerificationResult:
+        start_time = time.time()
+        try:
+            async with semaphore:
+                verification_result = await self.verify(response)
+            processing_time = time.time() - start_time
+            success = verification_result.status == VerificationOutcome.SUCCESS
+            self._batch_processor.adjust_batch_size(success, processing_time)
+            return verification_result
+        except Exception as e:
+            processing_time = time.time() - start_time
+            self._batch_processor.adjust_batch_size(False, processing_time)
+            logger.error(f"Verification failed: {e!s}", exc_info=True)
+            return VerificationResult(
+                status=VerificationOutcome.ERROR,
+                result="",
+                error_message=str(e),
+                metadata={"error_type": type(e).__name__},
+            )
+
+    # Process in batches
+    all_results: List[VerificationResult] = []
+    for i in range(0, len(results), batch_size):
+        batch = results[i : i + batch_size]
+        verification_tasks = [
+            _verify_with_semaphore(result) for result in batch
+        ]
+        try:
+            batch_results = await asyncio.gather(*verification_tasks)
+            all_results.extend(batch_results)
+        except Exception as e:
+            logger.error(f"Batch verification failed: {e!s}", exc_info=True)
+            if raise_on_error:
+                raise RuntimeError(f"Batch verification failed: {e!s}") from e
+
+    if raise_on_error and any(
+        r.status in {VerificationOutcome.ERROR, VerificationOutcome.TIMEOUT}
+        for r in all_results
+    ):
+        error_msg = "One or more verifications failed"
+        logger.error(error_msg)
+        raise RuntimeError(error_msg)
+
+    return all_results

camel/verifiers/models.py

@@ -0,0 +1,82 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+from datetime import datetime
+from enum import Enum
+from typing import Any, Dict, Optional
+
+from pydantic import BaseModel, Field
+
+
+class VerifierInput(BaseModel):
+    r"""Structured input to the verifier"""
+
+    llm_response: str = Field(
+        description="The LLM response to be verified."
+        "Needs to be in a format that the verifier can handle."
+    )
+    ground_truth: Optional[str] = Field(
+        None, description="The ground truth data, if available."
+    )
+
+
+class VerificationOutcome(Enum):
+    r"""Enum representing the status of a verification."""
+
+    SUCCESS = "success"
+    FAILURE = "failure"
+    ERROR = "error"
+    TIMEOUT = "timeout"
+
+    def __bool__(self):
+        r"""Only VerificationOutcome.SUCCESS is truthy; others are falsy."""
+        return self is VerificationOutcome.SUCCESS
+
+
+class VerificationResult(BaseModel):
+    r"""Structured result from a verification."""
+
+    status: VerificationOutcome = Field(
+        description="Status of the verification"
+    )
+    result: str = Field(description="Verification result")
+    duration: float = Field(
+        default=0.0, description="Duration of verification in seconds"
+    )
+    timestamp: datetime = Field(
+        default_factory=datetime.now,
+        description="When the verification was performed",
+    )
+    metadata: Dict[str, Any] = Field(
+        default_factory=dict,
+        description="Additional metadata about the verification",
+    )
+    error_message: Optional[str] = Field(
+        default=None, description="Error message if verification failed"
+    )
+
+
+class VerifierConfig(BaseModel):
+    r"""Configuration for verifier behavior."""
+
+    enabled: bool = Field(True, description="Whether verification is enabled")
+    strict_mode: bool = Field(
+        False, description="Whether to fail on any validation error"
+    )
+    timeout: Optional[float] = Field(
+        None, description="Verification timeout in seconds"
+    )
+    max_retries: int = Field(3, description="Maximum number of retry attempts")
+    retry_delay: float = Field(
+        1.0, description="Delay between retries in seconds"
+    )

camel/verifiers/python_verifier.py

@@ -0,0 +1,202 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+
+import asyncio
+import os
+import shutil
+import subprocess
+import tempfile
+import venv
+from typing import List, Optional
+
+from camel.logger import get_logger
+from camel.verifiers import BaseVerifier
+
+from .models import VerificationOutcome, VerificationResult, VerifierInput
+
+logger = get_logger(__name__)
+
+
+class PythonVerifier(BaseVerifier):
+    r"""The PythonVerifier class verifies Python-based implementations
+    by executing them in an isolated virtual environment.
+
+    Features:
+    - Creates a virtual environment with a specified Python version.
+    - Installs required packages before executing the provided script.
+    - Executes the script and compares the output against a ground truth,
+      if supplied.
+    - Automatically cleans up the virtual environment after execution.
+
+    The verification process ensures that the code runs in a controlled
+    environment, minimizing external dependencies and conflicts.
+    """
+
+    def __init__(
+        self,
+        timeout: Optional[float] = 30.0,
+        required_packages: Optional[List[str]] = None,
+    ):
+        r"""Initializes the PythonVerifier.
+
+        Args:
+            timeout (Optional[float], optional): The execution timeout in
+                seconds. (default: :obj:`30.0`)
+            required_packages (Optional[List[str]], optional): A list of
+                packages to install in the virtual environment.
+                (default: :obj:`None`)
+        """
+        # TODO: Use CAMEL's Interpreter to execute the code
+        super().__init__(timeout=timeout)
+        self.venv_path: Optional[str] = None
+        self.required_packages = required_packages or []
+
+        if os.name == 'nt':  # Windows
+            self.bin_dir = 'Scripts'
+        else:  # Unix-like systems
+            self.bin_dir = 'bin'
+
+    async def _setup(self) -> None:
+        r"""Set up a virtual environment for execution
+        and install required packages.
+        """
+        self.venv_path = tempfile.mkdtemp()
+        venv.create(self.venv_path, with_pip=True)
+        logger.info(f"Virtual environment created at {self.venv_path}")
+
+        venv_pip = os.path.join(self.venv_path, self.bin_dir, "pip")
+
+        if self.required_packages:
+            try:
+                subprocess.run(
+                    [venv_pip, "install", *self.required_packages],
+                    check=True,
+                    capture_output=True,
+                )
+                logger.info(
+                    "Installed required packages:"
+                    f"{', '.join(self.required_packages)}"
+                )
+            except subprocess.CalledProcessError as e:
+                logger.error(
+                    "Failed to install required packages: "
+                    f"{e.stderr.decode().strip()}"
+                )
+
+    async def _cleanup(self) -> None:
+        r"""Clean up the virtual environment."""
+        if self.venv_path:
+            shutil.rmtree(self.venv_path)
+            logger.info(f"Virtual environment at {self.venv_path} removed")
+            self.venv_path = None
+
+    async def _verify_implementation(
+        self, result: VerifierInput
+    ) -> VerificationResult:
+        r"""Executes the LLM-generated response in a Python virtual
+        environment.
+
+        Args:
+            result (VerifierInput): Contains the LLM-generated Python code to
+                execute and optional ground truth for comparison.
+
+        Returns:
+            VerificationResult: Contains verification status (SUCCESS/FAILURE/
+                ERROR), execution output, error messages if any, and execution
+                duration.
+
+        Raises:
+            asyncio.TimeoutError: If execution exceeds the configured timeout.
+            Exception: Any unexpected errors during execution are caught and
+                converted to an ERROR verification result.
+        """
+        if not self.venv_path:
+            return VerificationResult(
+                status=VerificationOutcome.ERROR,
+                result="",
+                error_message="Virtual environment is not set up.",
+            )
+
+        script = result.llm_response.strip()
+        venv_python = os.path.join(self.venv_path, self.bin_dir, "python")
+
+        if not os.path.exists(venv_python):
+            return VerificationResult(
+                status=VerificationOutcome.ERROR,
+                result="",
+                error_message="Python binary not found in virtual environment",
+            )
+
+        try:
+            process = await asyncio.create_subprocess_exec(
+                venv_python,
+                "-c",
+                script,
+                stdout=asyncio.subprocess.PIPE,
+                stderr=asyncio.subprocess.PIPE,
+            )
+
+            stdout, stderr = await asyncio.wait_for(
+                process.communicate(), timeout=self._timeout
+            )
+
+            output_result = stdout.decode().strip()
+            error_output = stderr.decode().strip()
+
+            if process.returncode == 0:
+                # If ground truth is provided, compare it with the result
+                if result.ground_truth is not None:
+                    # Normalize both strings by removing extra whitespace
+                    normalized_output = ' '.join(output_result.strip().split())
+                    normalized_truth = ' '.join(
+                        str(result.ground_truth).strip().split()
+                    )
+
+                    if normalized_output == normalized_truth:
+                        return VerificationResult(
+                            status=VerificationOutcome.SUCCESS,
+                            result=output_result,
+                        )
+                    else:
+                        return VerificationResult(
+                            status=VerificationOutcome.FAILURE,
+                            error_message="Output doesn't match ground truth",
+                            result=output_result,
+                        )
+                else:
+                    return VerificationResult(
+                        status=VerificationOutcome.SUCCESS,
+                        result=output_result,
+                    )
+
+            else:
+                return VerificationResult(
+                    status=VerificationOutcome.ERROR,
+                    error_message=error_output,
+                    result=output_result,
+                )
+
+        except asyncio.TimeoutError:
+            return VerificationResult(
+                status=VerificationOutcome.TIMEOUT,
+                result="",
+                error_message="Execution timed out.",
+            )
+
+        except Exception as e:
+            return VerificationResult(
+                status=VerificationOutcome.ERROR,
+                result="",
+                error_message=f"Execution error: {e}",
+            )