camel-ai 0.2.65__py3-none-any.whl → 0.2.83a6__py3-none-any.whl

camel/memories/context_creators/score_based.py

@@ -1,4 +1,4 @@
-# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# ========= Copyright 2023-2026 @ CAMEL-AI.org. All Rights Reserved. =========
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
@@ -10,41 +10,30 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
-from typing import List, Optional, Tuple
+# ========= Copyright 2023-2026 @ CAMEL-AI.org. All Rights Reserved. =========
 
-from pydantic import BaseModel
+from typing import List, Optional, Tuple
 
-from camel.logger import get_logger
 from camel.memories.base import BaseContextCreator
 from camel.memories.records import ContextRecord
 from camel.messages import OpenAIMessage
 from camel.types.enums import OpenAIBackendRole
 from camel.utils import BaseTokenCounter
 
-logger = get_logger(__name__)
-
-
-class _ContextUnit(BaseModel):
-    idx: int
-    record: ContextRecord
-    num_tokens: int
-
 
 class ScoreBasedContextCreator(BaseContextCreator):
-    r"""A default implementation of context creation strategy, which inherits
-    from :obj:`BaseContextCreator`.
+    r"""A context creation strategy that orders records chronologically.
 
-    This class provides a strategy to generate a conversational context from
-    a list of chat history records while ensuring the total token count of
-    the context does not exceed a specified limit. It prunes messages based
-    on their score if the total token count exceeds the limit.
+    This class supports token count estimation to reduce expensive repeated
+    token counting. When a cached token count is available, it estimates
+    new message tokens using character-based approximation instead of
+    calling the token counter for every message.
 
     Args:
-        token_counter (BaseTokenCounter): An instance responsible for counting
-            tokens in a message.
-        token_limit (int): The maximum number of tokens allowed in the
-            generated context.
+        token_counter (BaseTokenCounter): Token counter instance used to
+            compute the combined token count of the returned messages.
+        token_limit (int): Retained for API compatibility. No longer used to
+            filter records.
     """
 
     def __init__(
@@ -52,6 +41,10 @@ class ScoreBasedContextCreator(BaseContextCreator):
     ) -> None:
         self._token_counter = token_counter
         self._token_limit = token_limit
+        # Token count cache: stores the known token count and corresponding
+        # message count from the last LLM response
+        self._cached_token_count: Optional[int] = None
+        self._cached_message_count: int = 0
 
     @property
     def token_counter(self) -> BaseTokenCounter:
@@ -61,230 +54,116 @@ class ScoreBasedContextCreator(BaseContextCreator):
     def token_limit(self) -> int:
         return self._token_limit
 
-    def create_context(
-        self,
-        records: List[ContextRecord],
-    ) -> Tuple[List[OpenAIMessage], int]:
-        r"""Constructs conversation context from chat history while respecting
-        token limits.
-
-        Key strategies:
-        1. System message is always prioritized and preserved
-        2. Truncation removes low-score messages first
-        3. Final output maintains chronological order and in history memory,
-           the score of each message decreases according to keep_rate. The
-           newer the message, the higher the score.
-
-        Args:
-            records (List[ContextRecord]): List of context records with scores
-                and timestamps.
-
-        Returns:
-            Tuple[List[OpenAIMessage], int]:
-            - Ordered list of OpenAI messages
-            - Total token count of the final context
-
-        Raises:
-            RuntimeError: If system message alone exceeds token limit
-        """
-        # ======================
-        # 1. System Message Handling
-        # ======================
-        system_unit, regular_units = self._extract_system_message(records)
-        system_tokens = system_unit.num_tokens if system_unit else 0
-
-        # Check early if system message alone exceeds token limit
-        if system_tokens > self.token_limit:
-            raise RuntimeError(
-                f"System message alone exceeds token limit"
-                f": {system_tokens} > {self.token_limit}",
-                system_tokens,
-            )
-
-        # ======================
-        # 2. Deduplication & Initial Processing
-        # ======================
-        seen_uuids = set()
-        if system_unit:
-            seen_uuids.add(system_unit.record.memory_record.uuid)
-
-        # Process non-system messages with deduplication
-        for idx, record in enumerate(records):
-            if record.memory_record.uuid in seen_uuids:
-                continue
-            seen_uuids.add(record.memory_record.uuid)
-
-            token_count = self.token_counter.count_tokens_from_messages(
-                [record.memory_record.to_openai_message()]
-            )
-            regular_units.append(
-                _ContextUnit(
-                    idx=idx,
-                    record=record,
-                    num_tokens=token_count,
-                )
-            )
-
-        # ======================
-        # 3. Token Calculation
-        # ======================
-        total_tokens = system_tokens + sum(u.num_tokens for u in regular_units)
-
-        # ======================
-        # 4. Early Return if Within Limit
-        # ======================
-        if total_tokens <= self.token_limit:
-            sorted_units = sorted(
-                regular_units, key=self._conversation_sort_key
-            )
-            return self._assemble_output(sorted_units, system_unit)
-
-        # ======================
-        # 5. Truncation Logic
-        # ======================
-        logger.warning(
-            f"Context truncation required "
-            f"({total_tokens} > {self.token_limit}), "
-            f"pruning low-score messages."
-        )
-
-        # Sort for truncation: high scores first, older messages first at same
-        # score
-        sorted_for_truncation = sorted(
-            regular_units, key=self._truncation_sort_key
-        )
-
-        # Reverse to process from lowest score (end of sorted list)
-        remaining_units = []
-        current_total = system_tokens
-
-        for unit in sorted_for_truncation:
-            potential_total = current_total + unit.num_tokens
-            if potential_total <= self.token_limit:
-                remaining_units.append(unit)
-                current_total = potential_total
-
-        # ======================
-        # 6. Output Assembly
-        # ======================
-
-        # In case system message is the only message in memory when sorted
-        # units are empty, raise an error
-        if system_unit and len(remaining_units) == 0 and len(records) > 1:
-            raise RuntimeError(
-                "System message and current message exceeds token limit ",
-                total_tokens,
-            )
-
-        # Sort remaining units chronologically
-        final_units = sorted(remaining_units, key=self._conversation_sort_key)
-        return self._assemble_output(final_units, system_unit)
-
-    def _extract_system_message(
-        self, records: List[ContextRecord]
-    ) -> Tuple[Optional[_ContextUnit], List[_ContextUnit]]:
-        r"""Extracts the system message from records and validates it.
+    def set_cached_token_count(
+        self, token_count: int, message_count: int
+    ) -> None:
+        r"""Set the cached token count from LLM response usage.
 
         Args:
-            records (List[ContextRecord]): List of context records
-                representing conversation history.
-
-        Returns:
-            Tuple[Optional[_ContextUnit], List[_ContextUnit]]: containing:
-            - The system message as a `_ContextUnit`, if valid; otherwise,
-                `None`.
-            - An empty list, serving as the initial container for regular
-                messages.
+            token_count (int): The total token count (prompt + completion)
+                from LLM response usage.
+            message_count (int): The number of messages including the
+                assistant response that will be added to memory.
         """
-        if not records:
-            return None, []
-
-        first_record = records[0]
-        if (
-            first_record.memory_record.role_at_backend
-            != OpenAIBackendRole.SYSTEM
-        ):
-            return None, []
-
-        message = first_record.memory_record.to_openai_message()
-        tokens = self.token_counter.count_tokens_from_messages([message])
-        system_message_unit = _ContextUnit(
-            idx=0,
-            record=first_record,
-            num_tokens=tokens,
-        )
-        return system_message_unit, []
-
-    def _truncation_sort_key(self, unit: _ContextUnit) -> Tuple[float, float]:
-        r"""Defines the sorting key for the truncation phase.
-
-        Sorting priority:
-        - Primary: Sort by score in descending order (higher scores first).
-        - Secondary: Sort by timestamp in ascending order (older messages
-            first when scores are equal).
-
-        Args:
-            unit (_ContextUnit): A `_ContextUnit` representing a conversation
-                record.
+        self._cached_token_count = token_count
+        self._cached_message_count = message_count
 
-        Returns:
-            Tuple[float, float]:
-            - Negative score for descending order sorting.
-            - Timestamp for ascending order sorting.
-        """
-        return (-unit.record.score, unit.record.timestamp)
+    def clear_cache(self) -> None:
+        r"""Clear the cached token count."""
+        self._cached_token_count = None
+        self._cached_message_count = 0
 
-    def _conversation_sort_key(
-        self, unit: _ContextUnit
-    ) -> Tuple[float, float]:
-        r"""Defines the sorting key for assembling the final output.
+    def _estimate_message_tokens(self, message: OpenAIMessage) -> int:
+        r"""Estimate token count for a single message.
 
-        Sorting priority:
-        - Primary: Sort by timestamp in ascending order (chronological order).
-        - Secondary: Sort by score in descending order (higher scores first
-            when timestamps are equal).
+        Uses ~2 chars/token as a conservative approximation to handle both
+        ASCII (~4 chars/token) and CJK text (~1-2 chars/token).
 
         Args:
-            unit (_ContextUnit): A `_ContextUnit` representing a conversation
-                record.
+            message: The OpenAI message to estimate.
 
         Returns:
-            Tuple[float, float]:
-            - Timestamp for chronological sorting.
-            - Negative score for descending order sorting.
+            Estimated token count (intentionally conservative).
         """
-        return (unit.record.timestamp, -unit.record.score)
+        content = message.get("content", "")
+        token_estimate = 4  # message overhead
+
+        if isinstance(content, list):
+            # Multimodal content
+            for part in content:
+                if isinstance(part, dict):
+                    if part.get("type") == "image_url":
+                        # Images: 85 (low) to ~1500 (high detail large image)
+                        # Use 1500 as conservative estimate
+                        token_estimate += 1500
+                    else:
+                        token_estimate += len(str(part)) // 2
+                else:
+                    token_estimate += len(str(part)) // 2
+        else:
+            token_estimate += len(str(content)) // 2
+
+        # Add tool_calls if present (already text, safe to serialize)
+        if message.get("tool_calls"):
+            token_estimate += len(str(message.get("tool_calls"))) // 2
+
+        return token_estimate
 
-    def _assemble_output(
+    def create_context(
         self,
-        context_units: List[_ContextUnit],
-        system_unit: Optional[_ContextUnit],
+        records: List[ContextRecord],
     ) -> Tuple[List[OpenAIMessage], int]:
-        r"""Assembles final message list with proper ordering and token count.
+        """Returns messages sorted by timestamp and their total token count."""
+
+        system_record: Optional[ContextRecord] = None
+        remaining_records: List[ContextRecord] = []
+
+        for record in records:
+            if (
+                system_record is None
+                and record.memory_record.role_at_backend
+                == OpenAIBackendRole.SYSTEM
+            ):
+                system_record = record
+                continue
+            remaining_records.append(record)
 
-        Args:
-            context_units (List[_ContextUnit]): Sorted list of regular message
-                units.
-            system_unit (Optional[_ContextUnit]): System message unit (if
-                present).
+        remaining_records.sort(key=lambda record: record.timestamp)
 
-        Returns:
-            Tuple[List[OpenAIMessage], int]: Tuple of (ordered messages, total
-                tokens)
-        """
-        messages = []
-        total_tokens = 0
+        messages: List[OpenAIMessage] = []
+        if system_record is not None:
+            messages.append(system_record.memory_record.to_openai_message())
 
-        # Add system message first if present
-        if system_unit:
-            messages.append(
-                system_unit.record.memory_record.to_openai_message()
-            )
-            total_tokens += system_unit.num_tokens
+        messages.extend(
+            record.memory_record.to_openai_message()
+            for record in remaining_records
+        )
+
+        if not messages:
+            return [], 0
 
-        # Add sorted regular messages
-        for unit in context_units:
-            messages.append(unit.record.memory_record.to_openai_message())
-            total_tokens += unit.num_tokens
+        current_count = len(messages)
+
+        # Use cache if available and valid
+        if (
+            self._cached_token_count is not None
+            and self._cached_message_count > 0
+        ):
+            if current_count == self._cached_message_count:
+                # Same message count, use cached value directly
+                return messages, self._cached_token_count
+            elif current_count > self._cached_message_count:
+                # New messages added, estimate incrementally
+                new_messages = messages[self._cached_message_count :]
+                estimated_new_tokens = sum(
+                    self._estimate_message_tokens(msg) for msg in new_messages
+                )
+                return (
+                    messages,
+                    self._cached_token_count + estimated_new_tokens,
+                )
+            # current_count < cached: messages were removed, cache invalid
 
+        # No cache or cache is stale - do full calculation
+        total_tokens = self.token_counter.count_tokens_from_messages(messages)
         return messages, total_tokens

camel/memories/records.py

@@ -1,4 +1,4 @@
-# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# ========= Copyright 2023-2026 @ CAMEL-AI.org. All Rights Reserved. =========
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
@@ -10,13 +10,13 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# ========= Copyright 2023-2026 @ CAMEL-AI.org. All Rights Reserved. =========
 
 # Enables postponed evaluation of annotations (for string-based type hints)
 from __future__ import annotations
 
+import inspect
 import time
-from dataclasses import asdict
 from typing import Any, ClassVar, Dict
 from uuid import UUID, uuid4
 
@@ -63,6 +63,20 @@ class MemoryRecord(BaseModel):
         "FunctionCallingMessage": FunctionCallingMessage,
     }
 
+    # Cache for constructor parameters (performance optimization)
+    _constructor_params_cache: ClassVar[Dict[str, set]] = {}
+
+    @classmethod
+    def _get_constructor_params(cls, message_cls) -> set:
+        """Get constructor parameters for a message class with caching."""
+        cls_name = message_cls.__name__
+        if cls_name not in cls._constructor_params_cache:
+            sig = inspect.signature(message_cls.__init__)
+            cls._constructor_params_cache[cls_name] = set(
+                sig.parameters.keys()
+            ) - {'self'}
+        return cls._constructor_params_cache[cls_name]
+
     @classmethod
     def from_dict(cls, record_dict: Dict[str, Any]) -> "MemoryRecord":
         r"""Reconstruct a :obj:`MemoryRecord` from the input dict.
@@ -70,14 +84,78 @@ class MemoryRecord(BaseModel):
         Args:
             record_dict(Dict[str, Any]): A dict generated by :meth:`to_dict`.
         """
+        from camel.types import OpenAIBackendRole, RoleType
+
         message_cls = cls._MESSAGE_TYPES[record_dict["message"]["__class__"]]
-        kwargs: Dict = record_dict["message"].copy()
-        kwargs.pop("__class__")
-        reconstructed_message = message_cls(**kwargs)
+        data = record_dict["message"].copy()
+        data.pop("__class__")
+
+        # Convert role_type string to enum
+        if "role_type" in data and isinstance(data["role_type"], str):
+            data["role_type"] = RoleType(data["role_type"])
+
+        # Deserialize image_list from base64 strings/URLs back to PIL Images/
+        # URLs
+        if "image_list" in data and data["image_list"] is not None:
+            import base64
+            from io import BytesIO
+
+            from PIL import Image
+
+            image_objects = []
+            for img_item in data["image_list"]:
+                if isinstance(img_item, dict):
+                    # New format with type indicator
+                    if img_item["type"] == "url":
+                        # URL string, keep as-is
+                        image_objects.append(img_item["data"])
+                    else:  # type == "base64"
+                        # Base64 encoded image, convert to PIL Image
+                        img_bytes = base64.b64decode(img_item["data"])
+                        img = Image.open(BytesIO(img_bytes))
+                        # Restore the format attribute if it was saved
+                        if "format" in img_item:
+                            img.format = img_item["format"]
+                        image_objects.append(img)
+                else:
+                    # Legacy format: assume it's a base64 string
+                    img_bytes = base64.b64decode(img_item)
+                    img = Image.open(BytesIO(img_bytes))
+                    image_objects.append(img)
+            data["image_list"] = image_objects
+
+        # Deserialize video_bytes from base64 string
+        if "video_bytes" in data and data["video_bytes"] is not None:
+            import base64
+
+            data["video_bytes"] = base64.b64decode(data["video_bytes"])
+
+        # Get valid constructor parameters (cached)
+        valid_params = cls._get_constructor_params(message_cls)
+
+        # Separate constructor args from extra fields
+        kwargs = {k: v for k, v in data.items() if k in valid_params}
+        extra_fields = {k: v for k, v in data.items() if k not in valid_params}
+
+        # Handle meta_dict properly: merge existing meta_dict with extra fields
+        existing_meta = kwargs.get("meta_dict", {}) or {}
+        if extra_fields:
+            # Extra fields take precedence, but preserve existing meta_dict
+            # structure
+            merged_meta = {**existing_meta, **extra_fields}
+            kwargs["meta_dict"] = merged_meta
+        elif not existing_meta:
+            kwargs["meta_dict"] = None
+
+        # Convert role_at_backend
+        role_at_backend = record_dict["role_at_backend"]
+        if isinstance(role_at_backend, str):
+            role_at_backend = OpenAIBackendRole(role_at_backend)
+
         return cls(
             uuid=UUID(record_dict["uuid"]),
-            message=reconstructed_message,
-            role_at_backend=record_dict["role_at_backend"],
+            message=message_cls(**kwargs),
+            role_at_backend=role_at_backend,
             extra_info=record_dict["extra_info"],
             timestamp=record_dict["timestamp"],
             agent_id=record_dict["agent_id"],
@@ -91,9 +169,11 @@ class MemoryRecord(BaseModel):
             "uuid": str(self.uuid),
             "message": {
                 "__class__": self.message.__class__.__name__,
-                **asdict(self.message),
+                **self.message.to_dict(),
             },
-            "role_at_backend": self.role_at_backend,
+            "role_at_backend": self.role_at_backend.value
+            if hasattr(self.role_at_backend, "value")
+            else self.role_at_backend,
             "extra_info": self.extra_info,
             "timestamp": self.timestamp,
             "agent_id": self.agent_id,

camel/messages/__init__.py

@@ -1,4 +1,4 @@
-# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# ========= Copyright 2023-2026 @ CAMEL-AI.org. All Rights Reserved. =========
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
@@ -10,7 +10,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# ========= Copyright 2023-2026 @ CAMEL-AI.org. All Rights Reserved. =========
 from typing import Union
 
 from camel.types import (