camel-ai 0.2.29__py3-none-any.whl → 0.2.30__py3-none-any.whl

camel/environments/multi_step.py

@@ -0,0 +1,271 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+
+from abc import ABC, abstractmethod
+from typing import Any, Dict, List, Optional, Tuple
+
+from camel.extractors.base import BaseExtractor
+from camel.logger import get_logger
+
+from .models import Action, Observation, StepResult
+
+logger = get_logger(__name__)
+
+
+class MultiStepEnv(ABC):
+    r"""A multi-step environment for reinforcement learning with LLMs."""
+
+    def __init__(
+        self,
+        extractor: BaseExtractor,
+        max_steps: Optional[int] = None,
+        **kwargs,
+    ) -> None:
+        r"""Initialize the environment.
+
+        Args:
+            extractor: Extractor to process LLM responses.
+            max_steps: Maximum steps per episode.
+            **kwargs: Additional environment parameters.
+        """
+        self.extractor = extractor
+        self.max_steps = max_steps
+        self._metadata = kwargs
+
+        # State tracking
+        self._is_setup: bool = False
+        self._current_step: int = 0
+        self._episode_ended: bool = False
+        self._state: Dict[str, Any] = self._get_initial_state()
+        self._last_observation: Optional[Observation] = None
+        self._episode_history: List[Tuple[Observation, Action]] = []
+
+    async def setup(self) -> None:
+        r"""Set up the environment by initializing the verifier and extractor.
+
+        This method ensures that the environment is ready for interaction.
+        It sets up necessary components, including the verifier and extractor.
+
+        Raises:
+            Exception: If setup fails due to an internal error.
+        """
+
+        if self._is_setup:
+            return
+
+        try:
+            await self.extractor.setup()
+            await self._setup()
+            self._is_setup = True
+            logger.info('Environment setup completed successfully')
+        except Exception as e:
+            logger.error(f'Failed to setup environment: {e}')
+            raise
+
+    @abstractmethod
+    async def _setup(self) -> None:
+        pass
+
+    async def close(self) -> None:
+        r"""Clean up and close all resources used by the environment.
+        This method shuts down the verifier, calls the internal
+        close function that is implemented in any MultiStepEnv,
+        and ensures that the environment is properly closed.
+
+        Raises:
+            Exception: If an error occurs while closing the environment.
+        """
+        if not self._is_setup:
+            return
+
+        try:
+            await self.extractor.cleanup()
+
+            await self._close()
+
+            self._is_setup = False
+            logger.info('Environment teardown completed successfully')
+        except Exception as e:
+            logger.error(f'Failed to teardown environment: {e}')
+            raise
+
+    @abstractmethod
+    async def _close(self) -> None:
+        pass
+
+    async def reset(self) -> Observation:
+        r"""Reset the environment to an initial state.
+
+        Returns:
+            Observation: The initial observation for the episode.
+
+        Raises:
+            RuntimeError: If we fail to get the initial observation.
+        """
+
+        if not self._is_setup:
+            await self.setup()
+
+        # Reset state
+        self._current_step = 0
+        self._episode_ended = False
+        self._episode_history = []
+        self._state = self._get_initial_state()
+
+        # Get initial observation
+        observation = self._get_next_observation()
+        if observation is None:
+            raise RuntimeError("Failed to get initial observation")
+
+        self._last_observation = observation
+
+        return observation
+
+    async def step(self, action: Action) -> StepResult:
+        r"""Take a step in the environment using the given action.
+
+        This method updates the environment state based on the LLM's response,
+        computes rewards, checks if the episode is done, and based on that
+        gets the next or final observation.
+
+        Args:
+            action (Action): The action containing the LLM response.
+
+        Returns:
+            StepResult containing next observation, total reward, a dictionary
+                of rewards, done flag, and info.
+
+        Raises:
+            RuntimeError: If the environment is not set up, the episode has
+                ended, or there is no valid current observation.
+        """
+        if self.max_steps and self._current_step >= self.max_steps:
+            return StepResult(
+                observation=self._get_terminal_observation(),
+                reward=0,
+                rewards_dict={},
+                done=True,
+                info={"reason": "max_steps_reached"},
+            )
+
+        if not self._is_setup:
+            raise RuntimeError("Environment not set up. Call setup() first.")
+        if self._episode_ended:
+            raise RuntimeError("Episode has ended. Call reset() first.")
+        if self._last_observation is None:
+            raise RuntimeError("No current observation. Call reset() first.")
+
+        self._current_step += 1
+
+        current_obs: Observation = self._last_observation
+        self._episode_history.append((current_obs, action))
+
+        # Update the environment state based on the action
+        await self._update_state(action)
+
+        # Compute rewards
+        total_reward, rewards_dict = await self.compute_reward()
+
+        # Check termination
+        done = self.is_done()
+
+        # Get next observation based on the updated state
+        next_obs = (
+            self._get_terminal_observation()
+            if done
+            else self._get_next_observation()
+        )
+
+        self._last_observation = next_obs
+        self._episode_ended = done
+
+        return StepResult(
+            observation=next_obs,
+            reward=total_reward,
+            rewards_dict=rewards_dict,
+            done=done,
+            info={
+                "extraction_result": self.extractor.extract(
+                    action.llm_response
+                ),
+                "step": self._current_step,
+                "state": self._state,  # Updated state
+            },
+        )
+
+    @abstractmethod
+    def _get_initial_state(self) -> Dict[str, Any]:
+        pass
+
+    @abstractmethod
+    async def _update_state(self, action: Action) -> None:
+        pass
+
+    @abstractmethod
+    def _get_next_observation(self) -> Observation:
+        pass
+
+    @abstractmethod
+    def _get_terminal_observation(self) -> Observation:
+        pass
+
+    @abstractmethod
+    async def compute_reward(
+        self,
+    ) -> Tuple[float, Dict[str, float]]:
+        pass
+
+    def is_done(self) -> bool:
+        r"""Check if the episode should terminate.
+
+        This function terminates the episode if the maximum number of
+        steps is reached or if any other terminating criterion is met.
+
+        Returns:
+            bool: A boolean flag.
+        """
+
+        # After too many steps
+        if self.max_steps and self._current_step >= self.max_steps:
+            return True
+
+        # Further termination logic can be implemented in subclass
+        if self._is_done():
+            return True
+
+        return False
+
+    @abstractmethod
+    def _is_done(self) -> bool:
+        pass
+
+    @property
+    def metadata(self) -> Dict[str, Any]:
+        r"""Retrieve the metadata of the environment.
+
+        This provides additional parameters and configuration details.
+
+        Returns:
+            Dict[str, Any]: A copy of the environment's metadata.
+        """
+        return self._metadata.copy()
+
+    @property
+    def current_step(self) -> int:
+        r"""Get the current step number.
+
+        Returns:
+            int: The number of the step we are currently in.
+        """
+        return self._current_step

camel/environments/single_step.py

@@ -0,0 +1,293 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+
+
+from abc import abstractmethod
+from typing import Any, Dict, Optional, Tuple, Union
+
+from camel.datasets import BaseGenerator, DataPoint, StaticDataset
+from camel.extractors.base import BaseExtractor
+from camel.logger import get_logger
+from camel.verifiers.base import (
+    BaseVerifier,
+    VerificationResult,
+)
+from camel.verifiers.models import (
+    VerifierInput,
+)
+
+from .models import Action, Observation, StepResult
+
+logger = get_logger(__name__)
+
+
+class SingleStepEnv:
+    r"""A single-step environment for reinforcement learning with LLMs.
+
+    Key Features:
+    - Samples questions from a dataset and asks the LLM
+    - Extracts verifiable information from model responses.
+    - Verifies extracted responses against ground truth.
+    - Computes and assigns rewards based on correctness.
+    - Supports async setup, teardown, and cleanup of resources.
+
+    This class is intended as a foundation for RL experiments involving
+    LLM-based policies, ensuring structured interactions between model
+    actions and verification mechanisms.
+    """
+
+    PLACEHOLDER_OBS = Observation(
+        question="Episode ended. This is just a placeholder."
+    )
+
+    ACCURACY_REWARD = 10
+
+    def __init__(
+        self,
+        dataset: Union[StaticDataset, BaseGenerator],
+        verifier: BaseVerifier,
+        extractor: BaseExtractor,
+        **kwargs,
+    ) -> None:
+        r"""Initialize the environment.
+
+        Args:
+            dataset: Dataset to sample questions from.
+            verifier: Verifier to check responses.
+            extractor: Extractor to process LLM responses.
+            **kwargs: Additional environment parameters.
+        """
+        self.dataset = dataset
+        self.verifier = verifier
+        self.extractor = extractor
+        self._metadata = kwargs
+
+        # State tracking
+        self._is_setup: bool = False
+        self._state: Optional[DataPoint] = None
+        self._episode_ended: bool = False
+
+    async def setup(self) -> None:
+        r"""Set up the environment by initializing the verifier and extractor.
+
+        This method ensures that the environment is ready for interaction.
+        It sets up necessary components, including the verifier and extractor.
+
+        Raises:
+            Exception: If setup fails due to an internal error.
+        """
+
+        if self._is_setup:
+            return
+
+        try:
+            await self.verifier.setup()
+            await self.extractor.setup()
+
+            self._is_setup = True
+            logger.info('Environment setup completed successfully')
+        except Exception as e:
+            logger.error(f'Failed to setup environment: {e}')
+            raise
+
+    async def close(self) -> None:
+        r"""Clean up and close all resources used by the environment.
+
+        This method shuts down the verifier and extractor, resets the internal
+        state, and ensures that the environment is properly closed.
+
+        Raises:
+            Exception: If an error occurs while closing the environment.
+        """
+
+        if not self._is_setup:
+            return
+
+        try:
+            self._is_setup = False
+            await self.verifier.cleanup()
+            await self.extractor.cleanup()
+            self._state = None
+            self._episode_ended = False
+            logger.info('Environment closed successfully')
+        except Exception as e:
+            logger.error(f'Failed to close environment: {e}')
+            raise
+
+    async def reset(self) -> Observation:
+        r"""Reset the environment and start a new episode.
+
+        This method samples a new data point from the dataset and returns the
+        initial observation.
+
+        Returns:
+            Observation: The first observation of the new episode, including
+                the question.
+
+        Raises:
+            Exception: If the environment is not set up properly.
+        """
+
+        if not self._is_setup:
+            await self.setup()
+
+        self._episode_ended = False
+
+        # Sample a datapoint
+
+        self._state = self.dataset.sample()
+
+        observation = Observation(
+            question=self._state.question, context={}, metadata={}
+        )
+
+        return observation
+
+    async def step(self, action: Action) -> StepResult:
+        r"""Take a step in the environment using the given action.
+
+        This method processes the LLM response, extracts verifiable content,
+        verifies correctness, computes rewards, and ends the episode.
+
+        Args:
+            action (Action): The action containing the LLM response to
+                evaluate.
+
+        Returns:
+            StepResult: Contains the next observation (placeholder), total
+                reward, reward breakdown, completion flag, and additional
+                information.
+
+        Raises:
+            RuntimeError: If the environment is not set up, the episode has
+                ended, or there is no valid current observation.
+        """
+
+        if not self._is_setup:
+            raise RuntimeError("Environment not set up. Call setup() first.")
+        if self._episode_ended:
+            raise RuntimeError("Episode has ended. Call reset() first.")
+        if self._state is None:
+            raise RuntimeError("No current observation. Call reset() first.")
+
+        # extract verifiable part from llm response
+        extraction_result = await self.extractor.extract(action.llm_response)
+
+        if not extraction_result:
+            raise RuntimeError(f"Couldn't extract from {action.llm_response}")
+
+        # verify the extracted
+        verification_result = await self.verifier.verify(
+            VerifierInput(
+                llm_response=extraction_result,
+                ground_truth=self._state.final_answer,
+            )
+        )
+
+        # compute rewards
+        total_reward, rewards_dict = await self._compute_reward(
+            action, extraction_result, verification_result
+        )
+
+        self._episode_ended = True
+
+        return StepResult(
+            observation=self.PLACEHOLDER_OBS,
+            reward=total_reward,
+            rewards_dict=rewards_dict,
+            done=True,
+            info={
+                "extraction_result": extraction_result,
+                "verification_result": verification_result,
+                "state": self._state,
+            },
+        )
+
+    async def _compute_reward(
+        self,
+        action: Action,
+        extraction_result: str,
+        verification_result: VerificationResult,
+    ) -> Tuple[float, Dict[str, float]]:
+        r"""Compute reward scores based on verification results.
+
+        This method calculates the reward based on correctness and any
+        additional custom reward components.
+
+        Args:
+            action (Action): The action taken in the environment.
+            extraction_result (str): The extracted verifiable content from the
+                LLM response.
+            verification_result (VerificationResult): The result of verifying
+                the extracted response.
+
+        Returns:
+            Tuple[float, Dict[str, float]]: A tuple containing:
+                - Total reward (float)
+                - Dictionary of individual reward components.
+
+        Raises:
+            Exception: If an error occurs while computing rewards.
+        """
+
+        rewards: Dict[str, float] = {}
+
+        rewards["correctness"] = (
+            self.ACCURACY_REWARD if verification_result.status else 0.0
+        )
+
+        further_rewards = await self._compute_custom_reward(
+            action, extraction_result, verification_result
+        )
+
+        rewards = rewards | further_rewards
+
+        return sum(rewards.values()), rewards
+
+    @abstractmethod
+    async def _compute_custom_reward(
+        self,
+        action: Action,
+        extraction_result: str,
+        verification_result: VerificationResult,
+    ) -> Dict[str, float]:
+        r"""Compute additional custom reward components.
+
+        This method should be implemented by subclasses to define
+        domain-specific reward calculations.
+
+        Args:
+            action (Action): The action taken in the environment.
+            extraction_result (str): The extracted verifiable content from the
+                LLM response.
+            verification_result (VerificationResult): The result of verifying
+                the extracted response.
+
+        Returns:
+            Dict[str, float]: A dictionary mapping custom reward categories
+                to their values.
+        """
+        pass
+
+    @property
+    def metadata(self) -> Dict[str, Any]:
+        r"""Retrieve the metadata of the environment.
+
+        This provides additional parameters and configuration details.
+
+        Returns:
+            Dict[str, Any]: A copy of the environment's metadata.
+        """
+
+        return self._metadata.copy()

camel/logger.py

@@ -12,6 +12,7 @@
 # limitations under the License.
 # ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
 
+
 import logging
 import os
 import sys
@@ -42,9 +43,51 @@ def _configure_library_logging():
         _logger.debug("Existing logger configuration found, using that.")
 
 
+def set_log_file(file_path):
+    r"""Set a file handler for the CAMEL library logging.
+
+    Args:
+        file_path (str): Path to the log file. If the directory doesn't exist,
+            it will be created.
+
+    Returns:
+        logging.FileHandler: The file handler that was added to the logger.
+    """
+    # Check for existing handlers to the same file
+    for handler in _logger.handlers:
+        if isinstance(handler, logging.FileHandler) and os.path.abspath(
+            handler.baseFilename
+        ) == os.path.abspath(file_path):
+            _logger.info(f"File handler already exists for: {file_path}")
+            return handler
+
+    # Create directory if it doesn't exist
+    log_dir = os.path.dirname(file_path)
+    if log_dir and not os.path.exists(log_dir):
+        os.makedirs(log_dir)
+
+    # Create file handler
+    file_handler = logging.FileHandler(file_path)
+    file_handler.setFormatter(
+        logging.Formatter(
+            '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+        )
+    )
+
+    # Set the same level as the logger
+    file_handler.setLevel(_logger.getEffectiveLevel())
+
+    # Add the handler to the logger
+    _logger.addHandler(file_handler)
+    _logger.info(f"Log file configured at: {file_path}")
+
+    return file_handler
+
+
 def disable_logging():
     r"""Disable all logging for the CAMEL library.
 
+
     This function sets the log level to a value higher than CRITICAL,
     effectively disabling all log messages, and adds a NullHandler to
     suppress any potential warnings about no handlers being found.
@@ -63,6 +106,7 @@ def disable_logging():
 def enable_logging():
     r"""Enable logging for the CAMEL library.
 
+
     This function re-enables logging if it was previously disabled,
     and configures the library logging using the default settings.
     If the logging is already configured,
@@ -75,12 +119,14 @@ def enable_logging():
 def set_log_level(level):
     r"""Set the logging level for the CAMEL library.
 
+
     Args:
         level (Union[str, int]): The logging level to set. This can be a string
             (e.g., 'INFO') or a logging level constant (e.g., logging.INFO,
             logging.DEBUG).
             See https://docs.python.org/3/library/logging.html#levels
 
+
     Raises:
         ValueError: If the provided level is not a valid logging level.
     """
@@ -98,15 +144,25 @@ def set_log_level(level):
         )
 
     _logger.setLevel(level)
+
+    # Update level for all handlers
+    for handler in _logger.handlers:
+        try:
+            handler.setLevel(level)
+        except Exception as e:
+            _logger.warning(f"Failed to set level on handler {handler}: {e}")
+
     _logger.debug(f"Logging level set to: {logging.getLevelName(level)}")
 
 
 def get_logger(name):
     r"""Get a logger with the specified name, prefixed with 'camel.'.
 
+
     Args:
         name (str): The name to be appended to 'camel.' to create the logger.
 
+
     Returns:
         logging.Logger: A logger instance with the name 'camel.{name}'.
     """

camel/models/openai_compatible_model.py

@@ -56,8 +56,10 @@ class OpenAICompatibleModel(BaseModelBackend):
         url: Optional[str] = None,
         token_counter: Optional[BaseTokenCounter] = None,
     ) -> None:
-        self.api_key = api_key or os.environ.get("OPENAI_COMPATIBILIY_API_KEY")
-        self.url = url or os.environ.get("OPENAI_COMPATIBILIY_API_BASE_URL")
+        self.api_key = api_key or os.environ.get(
+            "OPENAI_COMPATIBILITY_API_KEY"
+        )
+        self.url = url or os.environ.get("OPENAI_COMPATIBILITY_API_BASE_URL")
         super().__init__(
             model_type, model_config_dict, api_key, url, token_counter
         )