camel-ai 0.2.36__py3-none-any.whl → 0.2.38__py3-none-any.whl

camel/loaders/{panda_reader.py → pandas_reader.py}

@@ -11,13 +11,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
-import os
 from functools import wraps
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Union
 
-import pandas as pd
-
 if TYPE_CHECKING:
     from pandas import DataFrame
     from pandasai import SmartDataframe
@@ -50,25 +47,18 @@ def check_suffix(valid_suffixs: List[str]) -> Callable:
     return decorator
 
 
-class PandaReader:
+class PandasReader:
     def __init__(self, config: Optional[Dict[str, Any]] = None) -> None:
-        r"""Initializes the PandaReader class.
+        r"""Initializes the PandasReader class.
 
         Args:
             config (Optional[Dict[str, Any]], optional): The configuration
                 dictionary that can include LLM API settings for LLM-based
-                processing. If not provided, it will use OpenAI with the API
-                key from the OPENAI_API_KEY environment variable. You can
-                customize the LLM configuration by providing a 'llm' key in
-                the config dictionary. (default: :obj:`None`)
+                processing. If not provided, no LLM will be configured by
+                default. You can customize the LLM configuration by providing
+                a 'llm' key in the config dictionary. (default: :obj:`None`)
         """
-        from pandasai.llm import OpenAI  # type: ignore[import-untyped]
-
         self.config = config or {}
-        if "llm" not in self.config:
-            self.config["llm"] = OpenAI(
-                api_token=os.getenv("OPENAI_API_KEY"),
-            )
 
         self.__LOADER = {
             ".csv": self.read_csv,
@@ -91,8 +81,13 @@ class PandaReader:
         data: Union["DataFrame", str],
         *args: Any,
         **kwargs: Dict[str, Any],
-    ) -> "SmartDataframe":
-        r"""Loads a file or DataFrame and returns a SmartDataframe object.
+    ) -> Union["DataFrame", "SmartDataframe"]:
+        r"""Loads a file or DataFrame and returns a DataFrame or
+        SmartDataframe object.
+
+        If an LLM is configured in the config dictionary, a SmartDataframe
+        will be returned, otherwise a regular pandas DataFrame will be
+        returned.
 
         args:
             data (Union[DataFrame, str]): The data to load.
@@ -100,24 +95,32 @@ class PandaReader:
             **kwargs (Dict[str, Any]): Additional keyword arguments.
 
         Returns:
-            SmartDataframe: The SmartDataframe object.
+            Union[DataFrame, SmartDataframe]: The DataFrame or SmartDataframe
+                object.
         """
         from pandas import DataFrame
-        from pandasai import SmartDataframe
 
+        # Load the data into a pandas DataFrame
         if isinstance(data, DataFrame):
-            return SmartDataframe(data, config=self.config)
-        file_path = str(data)
-        path = Path(file_path)
-        if not file_path.startswith("http") and not path.exists():
-            raise FileNotFoundError(f"File {file_path} not found")
-        if path.suffix in self.__LOADER:
-            return SmartDataframe(
-                self.__LOADER[path.suffix](file_path, *args, **kwargs),  # type: ignore[operator]
-                config=self.config,
-            )
+            df = data
         else:
-            raise ValueError(f"Unsupported file format: {path.suffix}")
+            file_path = str(data)
+            path = Path(file_path)
+            if not file_path.startswith("http") and not path.exists():
+                raise FileNotFoundError(f"File {file_path} not found")
+            if path.suffix in self.__LOADER:
+                df = self.__LOADER[path.suffix](file_path, *args, **kwargs)  # type: ignore[operator]
+            else:
+                raise ValueError(f"Unsupported file format: {path.suffix}")
+
+        # If an LLM is configured, return a SmartDataframe, otherwise return a
+        # regular DataFrame
+        if "llm" in self.config:
+            from pandasai import SmartDataframe
+
+            return SmartDataframe(df, config=self.config)
+        else:
+            return df
 
     @check_suffix([".csv"])
     def read_csv(
@@ -133,6 +136,8 @@ class PandaReader:
         Returns:
             DataFrame: The DataFrame object.
         """
+        import pandas as pd
+
         return pd.read_csv(file_path, *args, **kwargs)
 
     @check_suffix([".xlsx", ".xls"])
@@ -149,6 +154,8 @@ class PandaReader:
         Returns:
             DataFrame: The DataFrame object.
         """
+        import pandas as pd
+
         return pd.read_excel(file_path, *args, **kwargs)
 
     @check_suffix([".json"])
@@ -165,6 +172,8 @@ class PandaReader:
         Returns:
             DataFrame: The DataFrame object.
         """
+        import pandas as pd
+
         return pd.read_json(file_path, *args, **kwargs)
 
     @check_suffix([".parquet"])
@@ -181,6 +190,8 @@ class PandaReader:
         Returns:
             DataFrame: The DataFrame object.
         """
+        import pandas as pd
+
         return pd.read_parquet(file_path, *args, **kwargs)
 
     def read_sql(self, *args: Any, **kwargs: Dict[str, Any]) -> "DataFrame":
@@ -193,6 +204,8 @@ class PandaReader:
         Returns:
             DataFrame: The DataFrame object.
         """
+        import pandas as pd
+
         return pd.read_sql(*args, **kwargs)
 
     def read_table(
@@ -208,6 +221,8 @@ class PandaReader:
         Returns:
             DataFrame: The DataFrame object.
         """
+        import pandas as pd
+
         return pd.read_table(file_path, *args, **kwargs)
 
     def read_clipboard(
@@ -222,6 +237,8 @@ class PandaReader:
         Returns:
             DataFrame: The DataFrame object.
         """
+        import pandas as pd
+
         return pd.read_clipboard(*args, **kwargs)
 
     @check_suffix([".html"])
@@ -238,6 +255,8 @@ class PandaReader:
         Returns:
             DataFrame: The DataFrame object.
         """
+        import pandas as pd
+
         return pd.read_html(file_path, *args, **kwargs)
 
     @check_suffix([".feather"])
@@ -254,6 +273,8 @@ class PandaReader:
         Returns:
             DataFrame: The DataFrame object.
         """
+        import pandas as pd
+
         return pd.read_feather(file_path, *args, **kwargs)
 
     @check_suffix([".dta"])
@@ -270,6 +291,8 @@ class PandaReader:
         Returns:
             DataFrame: The DataFrame object.
         """
+        import pandas as pd
+
         return pd.read_stata(file_path, *args, **kwargs)
 
     @check_suffix([".sas"])
@@ -286,6 +309,8 @@ class PandaReader:
         Returns:
             DataFrame: The DataFrame object.
         """
+        import pandas as pd
+
         return pd.read_sas(file_path, *args, **kwargs)
 
     @check_suffix([".pkl"])
@@ -302,6 +327,8 @@ class PandaReader:
         Returns:
             DataFrame: The DataFrame object.
         """
+        import pandas as pd
+
         return pd.read_pickle(file_path, *args, **kwargs)
 
     @check_suffix([".h5"])
@@ -318,6 +345,8 @@ class PandaReader:
         Returns:
             DataFrame: The DataFrame object.
         """
+        import pandas as pd
+
         return pd.read_hdf(file_path, *args, **kwargs)
 
     @check_suffix([".orc"])
@@ -334,4 +363,6 @@ class PandaReader:
         Returns:
             DataFrame: The DataFrame object.
         """
+        import pandas as pd
+
         return pd.read_orc(file_path, *args, **kwargs)

camel/memories/context_creators/score_based.py

@@ -11,7 +11,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
-from typing import List, Tuple
+from typing import List, Optional, Tuple
 
 from pydantic import BaseModel
 
@@ -19,6 +19,7 @@ from camel.logger import get_logger
 from camel.memories.base import BaseContextCreator
 from camel.memories.records import ContextRecord
 from camel.messages import OpenAIMessage
+from camel.types.enums import OpenAIBackendRole
 from camel.utils import BaseTokenCounter
 
 logger = get_logger(__name__)
@@ -64,96 +65,226 @@ class ScoreBasedContextCreator(BaseContextCreator):
         self,
         records: List[ContextRecord],
     ) -> Tuple[List[OpenAIMessage], int]:
-        r"""Creates conversational context from chat history while respecting
+        r"""Constructs conversation context from chat history while respecting
         token limits.
 
-        Constructs the context from provided records and ensures that the total
-        token count does not exceed the specified limit by pruning the least
-        score messages if necessary.
+        Key strategies:
+        1. System message is always prioritized and preserved
+        2. Truncation removes low-score messages first
+        3. Final output maintains chronological order and in history memory,
+           the score of each message decreases according to keep_rate. The
+           newer the message, the higher the score.
 
         Args:
-            records (List[ContextRecord]): A list of message records from which
-                to generate the context.
+            records (List[ContextRecord]): List of context records with scores
+                and timestamps.
 
         Returns:
-            Tuple[List[OpenAIMessage], int]: A tuple containing the constructed
-                context in OpenAIMessage format and the total token count.
+            Tuple[List[OpenAIMessage], int]:
+            - Ordered list of OpenAI messages
+            - Total token count of the final context
 
         Raises:
-            RuntimeError: If it's impossible to create a valid context without
-                exceeding the token limit.
+            RuntimeError: If system message alone exceeds token limit
         """
-        # Create unique context units list
-        uuid_set = set()
-        context_units = []
+        # ======================
+        # 1. System Message Handling
+        # ======================
+        system_unit, regular_units = self._extract_system_message(records)
+        system_tokens = system_unit.num_tokens if system_unit else 0
+
+        # Check early if system message alone exceeds token limit
+        if system_tokens > self.token_limit:
+            raise RuntimeError(
+                f"System message alone exceeds token limit"
+                f": {system_tokens} > {self.token_limit}",
+                system_tokens,
+            )
+
+        # ======================
+        # 2. Deduplication & Initial Processing
+        # ======================
+        seen_uuids = set()
+        if system_unit:
+            seen_uuids.add(system_unit.record.memory_record.uuid)
+
+        # Process non-system messages with deduplication
         for idx, record in enumerate(records):
-            if record.memory_record.uuid not in uuid_set:
-                uuid_set.add(record.memory_record.uuid)
-                context_units.append(
-                    _ContextUnit(
-                        idx=idx,
-                        record=record,
-                        num_tokens=self.token_counter.count_tokens_from_messages(
-                            [record.memory_record.to_openai_message()]
-                        ),
-                    )
+            if record.memory_record.uuid in seen_uuids:
+                continue
+            seen_uuids.add(record.memory_record.uuid)
+
+            token_count = self.token_counter.count_tokens_from_messages(
+                [record.memory_record.to_openai_message()]
+            )
+            regular_units.append(
+                _ContextUnit(
+                    idx=idx,
+                    record=record,
+                    num_tokens=token_count,
                 )
+            )
 
-        # TODO: optimize the process, may give information back to memory
+        # ======================
+        # 3. Token Calculation
+        # ======================
+        total_tokens = system_tokens + sum(u.num_tokens for u in regular_units)
 
-        # If not exceed token limit, simply return
-        total_tokens = sum([unit.num_tokens for unit in context_units])
+        # ======================
+        # 4. Early Return if Within Limit
+        # ======================
         if total_tokens <= self.token_limit:
-            context_units = sorted(
-                context_units,
-                key=lambda unit: (unit.record.timestamp, unit.record.score),
+            sorted_units = sorted(
+                regular_units, key=self._conversation_sort_key
             )
-            return self._create_output(context_units)
+            return self._assemble_output(sorted_units, system_unit)
 
-        # Log warning about token limit being exceeded
+        # ======================
+        # 5. Truncation Logic
+        # ======================
         logger.warning(
-            f"Token limit reached ({total_tokens} > {self.token_limit}). "
-            f"Some messages will be pruned from memory to meet the limit."
+            f"Context truncation required "
+            f"({total_tokens} > {self.token_limit}), "
+            f"pruning low-score messages."
         )
 
-        # Sort by score
-        context_units = sorted(
-            context_units,
-            key=lambda unit: (unit.record.timestamp, unit.record.score),
+        # Sort for truncation: high scores first, older messages first at same
+        # score
+        sorted_for_truncation = sorted(
+            regular_units, key=self._truncation_sort_key
         )
 
-        # Remove the least score messages until total token number is smaller
-        # than token limit
-        truncate_idx = None
-        for i, unit in enumerate(context_units):
-            if i == len(context_units) - 1:
-                # If we reach the end of the list and still exceed the token
-                raise RuntimeError(
-                    "Cannot create context: exceed token limit.", total_tokens
-                )
-            total_tokens -= unit.num_tokens
-            if total_tokens <= self.token_limit:
-                truncate_idx = i
-                break
-        if truncate_idx is None:
+        # Reverse to process from lowest score (end of sorted list)
+        remaining_units = []
+        current_total = system_tokens
+
+        for unit in sorted_for_truncation:
+            potential_total = current_total + unit.num_tokens
+            if potential_total <= self.token_limit:
+                remaining_units.append(unit)
+                current_total = potential_total
+
+        # ======================
+        # 6. Output Assembly
+        # ======================
+
+        # Incase system message is the only message in memory when sorted units
+        # are empty, raise an error
+        if system_unit and len(remaining_units) == 0 and len(records) > 1:
             raise RuntimeError(
-                "Cannot create context: exceed token limit.", total_tokens
+                "System message and current message exceeds token limit ",
+                total_tokens,
             )
-        return self._create_output(context_units[truncate_idx + 1 :])
 
-    def _create_output(
-        self, context_units: List[_ContextUnit]
-    ) -> Tuple[List[OpenAIMessage], int]:
-        r"""Helper method to generate output from context units.
+        # Sort remaining units chronologically
+        final_units = sorted(remaining_units, key=self._conversation_sort_key)
+        return self._assemble_output(final_units, system_unit)
+
+    def _extract_system_message(
+        self, records: List[ContextRecord]
+    ) -> Tuple[Optional[_ContextUnit], List[_ContextUnit]]:
+        r"""Extracts the system message from records and validates it.
+
+        Args:
+            records (List[ContextRecord]): List of context records
+                representing conversation history.
 
-        This method converts the provided context units into a format suitable
-        for output, specifically a list of OpenAIMessages and an integer
-        representing the total token count.
+        Returns:
+            Tuple[Optional[_ContextUnit], List[_ContextUnit]]: containing:
+            - The system message as a `_ContextUnit`, if valid; otherwise,
+                `None`.
+            - An empty list, serving as the initial container for regular
+                messages.
         """
-        context_units = sorted(
-            context_units, key=lambda unit: unit.record.timestamp
+        if not records:
+            return None, []
+
+        first_record = records[0]
+        if (
+            first_record.memory_record.role_at_backend
+            != OpenAIBackendRole.SYSTEM
+        ):
+            return None, []
+
+        message = first_record.memory_record.to_openai_message()
+        tokens = self.token_counter.count_tokens_from_messages([message])
+        system_message_unit = _ContextUnit(
+            idx=0,
+            record=first_record,
+            num_tokens=tokens,
         )
-        return [
-            unit.record.memory_record.to_openai_message()
-            for unit in context_units
-        ], sum([unit.num_tokens for unit in context_units])
+        return system_message_unit, []
+
+    def _truncation_sort_key(self, unit: _ContextUnit) -> Tuple[float, float]:
+        r"""Defines the sorting key for the truncation phase.
+
+        Sorting priority:
+        - Primary: Sort by score in descending order (higher scores first).
+        - Secondary: Sort by timestamp in ascending order (older messages
+            first when scores are equal).
+
+        Args:
+            unit (_ContextUnit): A `_ContextUnit` representing a conversation
+                record.
+
+        Returns:
+            Tuple[float, float]:
+            - Negative score for descending order sorting.
+            - Timestamp for ascending order sorting.
+        """
+        return (-unit.record.score, unit.record.timestamp)
+
+    def _conversation_sort_key(
+        self, unit: _ContextUnit
+    ) -> Tuple[float, float]:
+        r"""Defines the sorting key for assembling the final output.
+
+        Sorting priority:
+        - Primary: Sort by timestamp in ascending order (chronological order).
+        - Secondary: Sort by score in descending order (higher scores first
+            when timestamps are equal).
+
+        Args:
+            unit (_ContextUnit): A `_ContextUnit` representing a conversation
+                record.
+
+        Returns:
+            Tuple[float, float]:
+            - Timestamp for chronological sorting.
+            - Negative score for descending order sorting.
+        """
+        return (unit.record.timestamp, -unit.record.score)
+
+    def _assemble_output(
+        self,
+        context_units: List[_ContextUnit],
+        system_unit: Optional[_ContextUnit],
+    ) -> Tuple[List[OpenAIMessage], int]:
+        r"""Assembles final message list with proper ordering and token count.
+
+        Args:
+            context_units (List[_ContextUnit]): Sorted list of regular message
+                units.
+            system_unit (Optional[_ContextUnit]): System message unit (if
+                present).
+
+        Returns:
+            Tuple[List[OpenAIMessage], int]: Tuple of (ordered messages, total
+                tokens)
+        """
+        messages = []
+        total_tokens = 0
+
+        # Add system message first if present
+        if system_unit:
+            messages.append(
+                system_unit.record.memory_record.to_openai_message()
+            )
+            total_tokens += system_unit.num_tokens
+
+        # Add sorted regular messages
+        for unit in context_units:
+            messages.append(unit.record.memory_record.to_openai_message())
+            total_tokens += unit.num_tokens
+
+        return messages, total_tokens

camel/models/aiml_model.py

@@ -52,6 +52,10 @@ class AIMLModel(BaseModelBackend):
             use for the model. If not provided, :obj:`OpenAITokenCounter(
             ModelType.GPT_4O_MINI)` will be used.
             (default: :obj:`None`)
+        timeout (Optional[float], optional): The timeout value in seconds for
+            API calls. If not provided, will fall back to the MODEL_TIMEOUT
+            environment variable or default to 180 seconds.
+            (default: :obj:`None`)
     """
 
     @api_keys_required([("api_key", "AIML_API_KEY")])
@@ -62,6 +66,7 @@ class AIMLModel(BaseModelBackend):
         api_key: Optional[str] = None,
         url: Optional[str] = None,
         token_counter: Optional[BaseTokenCounter] = None,
+        timeout: Optional[float] = None,
     ) -> None:
         if model_config_dict is None:
             model_config_dict = AIMLConfig().as_dict()
@@ -70,17 +75,18 @@ class AIMLModel(BaseModelBackend):
             "AIML_API_BASE_URL",
             "https://api.aimlapi.com/v1",
         )
+        timeout = timeout or float(os.environ.get("MODEL_TIMEOUT", 180))
         super().__init__(
-            model_type, model_config_dict, api_key, url, token_counter
+            model_type, model_config_dict, api_key, url, token_counter, timeout
         )
         self._client = OpenAI(
-            timeout=180,
+            timeout=self._timeout,
             max_retries=3,
             api_key=self._api_key,
             base_url=self._url,
         )
         self._async_client = AsyncOpenAI(
-            timeout=180,
+            timeout=self._timeout,
             max_retries=3,
             api_key=self._api_key,
             base_url=self._url,

camel/models/anthropic_model.py

@@ -45,6 +45,10 @@ class AnthropicModel(BaseModelBackend):
         token_counter (Optional[BaseTokenCounter], optional): Token counter to
             use for the model. If not provided, :obj:`AnthropicTokenCounter`
             will be used. (default: :obj:`None`)
+        timeout (Optional[float], optional): The timeout value in seconds for
+            API calls. If not provided, will fall back to the MODEL_TIMEOUT
+            environment variable or default to 180 seconds.
+            (default: :obj:`None`)
     """
 
     @api_keys_required(
@@ -60,6 +64,7 @@ class AnthropicModel(BaseModelBackend):
         api_key: Optional[str] = None,
         url: Optional[str] = None,
         token_counter: Optional[BaseTokenCounter] = None,
+        timeout: Optional[float] = None,
     ) -> None:
         from openai import AsyncOpenAI, OpenAI
 
@@ -71,13 +76,16 @@ class AnthropicModel(BaseModelBackend):
             or os.environ.get("ANTHROPIC_API_BASE_URL")
             or "https://api.anthropic.com/v1/"
         )
+        timeout = timeout or float(os.environ.get("MODEL_TIMEOUT", 180))
         super().__init__(
-            model_type, model_config_dict, api_key, url, token_counter
+            model_type, model_config_dict, api_key, url, token_counter, timeout
+        )
+        self.client = OpenAI(
+            base_url=self._url, api_key=self._api_key, timeout=self._timeout
         )
-        self.client = OpenAI(base_url=self._url, api_key=self._api_key)
 
         self.async_client = AsyncOpenAI(
-            api_key=self._api_key, base_url=self._url
+            api_key=self._api_key, base_url=self._url, timeout=self._timeout
         )
 
     @property

camel/models/azure_openai_model.py

@@ -49,6 +49,10 @@ class AzureOpenAIModel(BaseModelBackend):
         token_counter (Optional[BaseTokenCounter], optional): Token counter to
             use for the model. If not provided, :obj:`OpenAITokenCounter`
             will be used. (default: :obj:`None`)
+        timeout (Optional[float], optional): The timeout value in seconds for
+            API calls. If not provided, will fall back to the MODEL_TIMEOUT
+            environment variable or default to 180 seconds.
+            (default: :obj:`None`)
 
     References:
         https://learn.microsoft.com/en-us/azure/ai-services/openai/
@@ -60,6 +64,7 @@ class AzureOpenAIModel(BaseModelBackend):
         model_config_dict: Optional[Dict[str, Any]] = None,
         api_key: Optional[str] = None,
         url: Optional[str] = None,
+        timeout: Optional[float] = None,
         token_counter: Optional[BaseTokenCounter] = None,
         api_version: Optional[str] = None,
         azure_deployment_name: Optional[str] = None,
@@ -68,8 +73,9 @@ class AzureOpenAIModel(BaseModelBackend):
             model_config_dict = ChatGPTConfig().as_dict()
         api_key = api_key or os.environ.get("AZURE_OPENAI_API_KEY")
         url = url or os.environ.get("AZURE_OPENAI_BASE_URL")
+        timeout = timeout or float(os.environ.get("MODEL_TIMEOUT", 180))
         super().__init__(
-            model_type, model_config_dict, api_key, url, token_counter
+            model_type, model_config_dict, api_key, url, token_counter, timeout
         )
 
         self.api_version = api_version or os.environ.get("AZURE_API_VERSION")
@@ -92,7 +98,7 @@ class AzureOpenAIModel(BaseModelBackend):
             azure_deployment=self.azure_deployment_name,
             api_version=self.api_version,
             api_key=self._api_key,
-            timeout=180,
+            timeout=self._timeout,
             max_retries=3,
         )
 
@@ -101,7 +107,7 @@ class AzureOpenAIModel(BaseModelBackend):
             azure_deployment=self.azure_deployment_name,
             api_version=self.api_version,
             api_key=self._api_key,
-            timeout=180,
+            timeout=self._timeout,
             max_retries=3,
         )