camel-ai 0.2.18__py3-none-any.whl → 0.2.20__py3-none-any.whl

camel/utils/commons.py

@@ -11,7 +11,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+import functools
 import importlib
+import logging
 import os
 import platform
 import re
@@ -47,6 +49,8 @@ from .constants import Constants
 
 F = TypeVar('F', bound=Callable[..., Any])
 
+logger = logging.getLogger(__name__)
+
 
 def print_text_animated(text, delay: float = 0.02, end: str = ""):
     r"""Prints the given text with an animated effect.
@@ -620,33 +624,206 @@ def handle_http_error(response: requests.Response) -> str:
         return "HTTP Error"
 
 
-def retry_request(
-    func: Callable, retries: int = 3, delay: int = 1, *args: Any, **kwargs: Any
-) -> Any:
-    r"""Retries a function in case of any errors.
+def retry_on_error(
+    max_retries: int = 3, initial_delay: float = 1.0
+) -> Callable:
+    r"""Decorator to retry function calls on exception with exponential
+    backoff.
 
     Args:
-        func (Callable): The function to be retried.
-        retries (int): Number of retry attempts. (default: :obj:`3`)
-        delay (int): Delay between retries in seconds. (default: :obj:`1`)
-        *args: Arguments to pass to the function.
-        **kwargs: Keyword arguments to pass to the function.
+        max_retries (int): Maximum number of retry attempts
+        initial_delay (float): Initial delay between retries in seconds
 
     Returns:
-        Any: The result of the function call if successful.
+        Callable: Decorated function with retry logic
+    """
 
-    Raises:
-        Exception: If all retry attempts fail.
+    def decorator(func: Callable) -> Callable:
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs):
+            delay = initial_delay
+            last_exception = None
+
+            for attempt in range(max_retries + 1):
+                try:
+                    return func(*args, **kwargs)
+                except Exception as e:
+                    last_exception = e
+                    if attempt == max_retries:
+                        logger.error(
+                            f"Failed after {max_retries} retries: {e!s}"
+                        )
+                        raise
+
+                    logger.warning(
+                        f"Attempt {attempt + 1} failed: {e!s}. "
+                        f"Retrying in {delay:.1f}s..."
+                    )
+                    time.sleep(delay)
+                    delay *= 2  # Exponential backoff
+
+            raise last_exception
+
+        return wrapper
+
+    return decorator
+
+
+class BatchProcessor:
+    r"""Handles batch processing with dynamic sizing and error handling based
+    on system load.
     """
-    for attempt in range(retries):
-        try:
-            return func(*args, **kwargs)
-        except Exception as e:
-            print(f"Attempt {attempt + 1}/{retries} failed: {e}")
-            if attempt < retries - 1:
-                time.sleep(delay)
-            else:
-                raise
+
+    def __init__(
+        self,
+        max_workers: Optional[int] = None,
+        initial_batch_size: Optional[int] = None,
+        monitoring_interval: float = 5.0,
+        cpu_threshold: float = 80.0,
+        memory_threshold: float = 85.0,
+    ):
+        r"""Initialize the BatchProcessor with dynamic worker allocation.
+
+        Args:
+            max_workers: Maximum number of workers. If None, will be
+                determined dynamically based on system resources.
+                (default: :obj:`None`)
+            initial_batch_size: Initial size of each batch. If `None`,
+                defaults to `10`. (default: :obj:`None`)
+            monitoring_interval: Interval in seconds between resource checks.
+                (default: :obj:`5.0`)
+            cpu_threshold: CPU usage percentage threshold for scaling down.
+                (default: :obj:`80.0`)
+            memory_threshold: Memory usage percentage threshold for scaling
+                down. (default: :obj:`85.0`)
+        """
+        import psutil
+
+        self.monitoring_interval = monitoring_interval
+        self.cpu_threshold = cpu_threshold
+        self.memory_threshold = memory_threshold
+        self.last_check_time = time.time()
+        self.psutil = psutil
+
+        # Initialize performance metrics
+        self.total_processed = 0
+        self.total_errors = 0
+        self.processing_times: List = []
+
+        if max_workers is None:
+            self.max_workers = self._calculate_optimal_workers()
+        else:
+            self.max_workers = max_workers
+
+        self.batch_size = (
+            10 if initial_batch_size is None else initial_batch_size
+        )
+        self.min_batch_size = 1
+        self.max_batch_size = 20
+        self.backoff_factor = 0.8
+        self.success_factor = 1.2
+
+        # Initial resource check
+        self._update_resource_metrics()
+
+    def _calculate_optimal_workers(self) -> int:
+        r"""Calculate optimal number of workers based on system resources."""
+        cpu_count = self.psutil.cpu_count()
+        cpu_percent = self.psutil.cpu_percent(interval=1)
+        memory = self.psutil.virtual_memory()
+
+        # Base number of workers on CPU count and current load
+        if cpu_percent > self.cpu_threshold:
+            workers = max(1, cpu_count // 4)
+        elif cpu_percent > 60:
+            workers = max(1, cpu_count // 2)
+        else:
+            workers = max(1, cpu_count - 1)
+
+        # Further reduce if memory is constrained
+        if memory.percent > self.memory_threshold:
+            workers = max(1, workers // 2)
+
+        return workers
+
+    def _update_resource_metrics(self) -> None:
+        r"""Update current resource usage metrics."""
+        self.current_cpu = self.psutil.cpu_percent()
+        self.current_memory = self.psutil.virtual_memory().percent
+        self.last_check_time = time.time()
+
+    def _should_check_resources(self) -> bool:
+        r"""Determine if it's time to check resource usage again."""
+        return time.time() - self.last_check_time >= self.monitoring_interval
+
+    def adjust_batch_size(
+        self, success: bool, processing_time: Optional[float] = None
+    ) -> None:
+        r"""Adjust batch size based on success/failure and system resources.
+
+        Args:
+            success (bool): Whether the last batch completed successfully
+            processing_time (Optional[float]): Time taken to process the last
+                batch. (default: :obj:`None`)
+        """
+        # Update metrics
+        self.total_processed += 1
+        if not success:
+            self.total_errors += 1
+        if processing_time is not None:
+            self.processing_times.append(processing_time)
+
+        # Check system resources if interval has elapsed
+        if self._should_check_resources():
+            self._update_resource_metrics()
+
+            # Adjust based on resource usage
+            if (
+                self.current_cpu > self.cpu_threshold
+                or self.current_memory > self.memory_threshold
+            ):
+                self.batch_size = max(
+                    int(self.batch_size * self.backoff_factor),
+                    self.min_batch_size,
+                )
+                self.max_workers = max(1, self.max_workers - 1)
+                return
+
+        # Adjust based on success/failure
+        if success:
+            self.batch_size = min(
+                int(self.batch_size * self.success_factor), self.max_batch_size
+            )
+        else:
+            self.batch_size = max(
+                int(self.batch_size * self.backoff_factor), self.min_batch_size
+            )
+
+    def get_performance_metrics(self) -> Dict[str, Any]:
+        r"""Get current performance metrics.
+
+        Returns:
+            Dict containing performance metrics including:
+            - total_processed: Total number of batches processed
+            - error_rate: Percentage of failed batches
+            - avg_processing_time: Average time per batch
+            - current_batch_size: Current batch size
+            - current_workers: Current number of workers
+            - current_cpu: Current CPU usage percentage
+            - current_memory: Current memory usage percentage
+        """
+        metrics = {
+            "total_processed": self.total_processed,
+            "error_rate": (self.total_errors / max(1, self.total_processed))
+            * 100,
+            "avg_processing_time": sum(self.processing_times)
+            / max(1, len(self.processing_times)),
+            "current_batch_size": self.batch_size,
+            "current_workers": self.max_workers,
+            "current_cpu": self.current_cpu,
+            "current_memory": self.current_memory,
+        }
+        return metrics
 
 
 def download_github_subdirectory(

camel/utils/deduplication.py

@@ -0,0 +1,199 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+
+
+from typing import Dict, List, Literal, Optional
+
+import numpy as np
+from pydantic import BaseModel
+from sklearn.metrics.pairwise import cosine_similarity
+
+from camel.embeddings.base import BaseEmbedding
+
+
+class DeduplicationResult(BaseModel):
+    """
+    The result of deduplication.
+
+    Attributes:
+        original_texts (List[str]): The original texts.
+        unique_ids (List[int]): A list of ids that are unique (not duplicates).
+        unique_embeddings_dict (Dict[int, List[float]]):
+            A mapping from the index of each unique text to its embedding.
+        duplicate_to_target_map (Dict[int, int]):
+            A mapping from the index of the duplicate text to the index
+            of the text it is considered a duplicate of.
+    """
+
+    original_texts: List[str]
+    unique_ids: List[int]
+    unique_embeddings_dict: Dict[int, List[float]]
+    duplicate_to_target_map: Dict[int, int]
+
+
+def deduplicate_internally(
+    texts: List[str],
+    threshold: float = 0.65,
+    embedding_instance: Optional[BaseEmbedding[str]] = None,
+    embeddings: Optional[List[List[float]]] = None,
+    strategy: Literal["top1", "llm-supervise"] = "top1",
+) -> DeduplicationResult:
+    """
+    Deduplicate a list of strings based on their cosine similarity.
+
+    You can either:
+    1) Provide a Camel `BaseEmbedding` instance via `embedding_instance` to let
+        this function handle the embedding internally, OR
+    2) Directly pass a list of pre-computed embeddings to `embeddings`.
+
+    If both `embedding_instance` and `embeddings` are provided, the function
+    will raise a ValueError to avoid ambiguous usage.
+
+    strategy is used to specify different strategies, where 'top1' selects the
+    one with highest similarity, and 'llm-supervise' uses LLM to determine if
+    texts are duplicates (not yet implemented).
+
+    Args:
+        texts (List[str]): The list of texts to be deduplicated.
+        threshold (float, optional): The similarity threshold for considering
+            two texts as duplicates. Default is 0.65.
+        embedding_instance (Optional[BaseEmbedding[str]], optional):
+            A Camel embedding instance for automatic embedding. Defaults to
+            None.
+        embeddings (Optional[List[List[float]]], optional):
+            Pre-computed embeddings of `texts`. Each element in the list
+            corresponds to the embedding of the text in the same index of
+            `texts`. Defaults to None.
+        strategy (Literal["top1", "llm-supervise"], optional):
+            The strategy to use for deduplication. Defaults to "top1".
+
+    Returns:
+        DeduplicationResult: An object that contains:
+            - `original_texts`: The original texts.
+            - `unique_ids`: The unique ids after deduplication.
+            - `unique_embeddings_dict`: A dict mapping from (unique) text id
+              to its embedding.
+            - `duplicate_to_target_map`: A dict mapping from the id of a
+              duplicate text to the id of the text it is considered a duplicate
+              of.
+
+    Raises:
+        NotImplementedError: If the strategy is not "top1".
+        ValueError: If neither embeddings nor embedding_instance is provided,
+                    or if both are provided at the same time.
+        ValueError: If the length of `embeddings` does not match the length of
+            `texts`.
+
+    Example:
+        >>> from camel.embeddings.openai_embedding import OpenAIEmbedding
+        >>> # Suppose we have 5 texts, some of which may be duplicates
+        >>> texts = [
+        ...     "What is AI?",
+        ...     "Artificial Intelligence is about machines",
+        ...     "What is AI?",
+        ...     "Deep Learning is a subset of AI",
+        ...     "What is artificial intelligence?"
+        ... ]
+        >>> # or any other BaseEmbedding instance
+        >>> embedding_model = OpenAIEmbedding()
+        >>> result = deduplicate_internally(
+        ...     texts=texts,
+        ...     threshold=0.7,
+        ...     embedding_instance=embedding_model
+        ... )
+        >>> print("Unique ids:")
+        >>> for uid in result.unique_ids:
+        ...     print(texts[uid])
+        Unique ids:
+        What is AI?
+        Artificial Intelligence is about machines
+        Deep Learning is a subset of AI
+        What is artificial intelligence?
+
+        >>> print("Duplicate map:")
+        >>> print(result.duplicate_to_target_map)
+        {2: 0}
+        # This indicates the text at index 2 is considered
+        # a duplicate of index 0.
+    """
+    if strategy == "llm-supervise":
+        # TODO: Implement LLM-supervise deduplication.
+        raise NotImplementedError(
+            "LLM-supervise deduplication is not yet implemented."
+        )
+
+    # Check if the parameters are valid.
+    if embedding_instance is None and embeddings is None:
+        raise ValueError(
+            "Either 'embedding_instance' or 'embeddings' must be provided."
+        )
+    if embedding_instance is not None and embeddings is not None:
+        raise ValueError(
+            "Cannot provide both 'embedding_instance' and 'embeddings'. "
+            "Please choose only one way to supply embeddings."
+        )
+
+    if embedding_instance is not None:
+        # Use Camel's embedding_instance to vectorize.
+        embeddings = embedding_instance.embed_list(texts)
+    else:
+        # Use pre-supplied embeddings.
+        if embeddings and len(embeddings) != len(texts):
+            raise ValueError(
+                "The length of 'embeddings' does not match the length "
+                "of 'texts'."
+            )
+
+    # Calculate cosine similarity.
+    similarity_matrix = cosine_similarity(embeddings)
+    n = len(texts)
+
+    # Use the lower triangle to avoid redundant comparisons
+    # (or self-comparisons).
+    tril_mask = np.tril(np.ones((n, n)), k=-1)
+    similarity_matrix = similarity_matrix * tril_mask
+
+    # For each row, find the column with the highest similarity
+    # that exceeds the threshold. If no similarity exceeds the threshold,
+    # set the column index to -1.
+    masked_similarities = np.where(
+        similarity_matrix > threshold, similarity_matrix, -1
+    )
+    max_indices = masked_similarities.argmax(axis=1)
+
+    duplicate_to_target_map: Dict[int, int] = {}
+    above_threshold = similarity_matrix[np.arange(n), max_indices] > threshold
+
+    # Construct the "duplicate->target" mapping.
+    for i in range(n):
+        if above_threshold[i]:
+            duplicate_to_target_map[i] = max_indices[i]
+
+    # Get the actual unique ids and embeddings.
+    unique_ids = []
+    unique_embeddings_dict = {}
+
+    assert embeddings, "embeddings must be valid"
+
+    for i, (_, emb) in enumerate(zip(texts, embeddings)):
+        if i not in duplicate_to_target_map:
+            unique_ids.append(i)
+            unique_embeddings_dict[i] = emb
+
+    return DeduplicationResult(
+        original_texts=texts,
+        unique_ids=unique_ids,
+        unique_embeddings_dict=unique_embeddings_dict,
+        duplicate_to_target_map=duplicate_to_target_map,
+    )

camel/utils/token_counting.py

@@ -112,7 +112,7 @@ class OpenAITokenCounter(BaseTokenCounter):
         elif ("gpt-3.5-turbo" in self.model) or ("gpt-4" in self.model):
             self.tokens_per_message = 3
             self.tokens_per_name = 1
-        elif "o1" in self.model:
+        elif ("o1" in self.model) or ("o3" in self.model):
             self.tokens_per_message = 2
             self.tokens_per_name = 1
         else:
@@ -267,44 +267,6 @@ class AnthropicTokenCounter(BaseTokenCounter):
         ).input_tokens
 
 
-class GeminiTokenCounter(BaseTokenCounter):
-    def __init__(self, model_type: UnifiedModelType):
-        r"""Constructor for the token counter for Gemini models.
-
-        Args:
-            model_type (UnifiedModelType): Model type for which tokens will be
-                counted.
-        """
-        import google.generativeai as genai
-
-        self._client = genai.GenerativeModel(model_type)
-
-    def count_tokens_from_messages(self, messages: List[OpenAIMessage]) -> int:
-        r"""Count number of tokens in the provided message list using
-        loaded tokenizer specific for this type of model.
-
-        Args:
-            messages (List[OpenAIMessage]): Message list with the chat history
-                in OpenAI API format.
-
-        Returns:
-            int: Number of tokens in the messages.
-        """
-        converted_messages = []
-        for message in messages:
-            role = message.get('role')
-            if role == 'assistant':
-                role_to_gemini = 'model'
-            else:
-                role_to_gemini = 'user'
-            converted_message = {
-                "role": role_to_gemini,
-                "parts": message.get("content"),
-            }
-            converted_messages.append(converted_message)
-        return self._client.count_tokens(converted_messages).total_tokens
-
-
 class LiteLLMTokenCounter(BaseTokenCounter):
     def __init__(self, model_type: UnifiedModelType):
         r"""Constructor for the token counter for LiteLLM models.

{camel_ai-0.2.18.dist-info → camel_ai-0.2.20.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: camel-ai
-Version: 0.2.18
+Version: 0.2.20
 Summary: Communicative Agents for AI Society Study
 License: Apache-2.0
 Keywords: communicative-ai,ai-societies,artificial-intelligence,deep-learning,multi-agent-systems,cooperative-ai,natural-language-processing,large-language-models
@@ -37,7 +37,7 @@ Requires-Dist: azure-storage-blob (>=12.21.0,<13.0.0) ; extra == "storage" or ex
 Requires-Dist: beautifulsoup4 (>=4,<5) ; extra == "document-tools" or extra == "all"
 Requires-Dist: botocore (>=1.35.3,<2.0.0) ; extra == "storage" or extra == "all"
 Requires-Dist: cohere (>=5.11.0,<6.0.0) ; extra == "rag" or extra == "model-platforms" or extra == "all"
-Requires-Dist: colorama (>=0,<1)
+Requires-Dist: colorama (>=0.4.6,<0.5.0)
 Requires-Dist: curl_cffi (==0.6.2)
 Requires-Dist: dappier (>=0.3.3,<0.4.0) ; extra == "web-tools" or extra == "all"
 Requires-Dist: datacommons (>=1.4.3,<2.0.0) ; extra == "data-tools" or extra == "all"
@@ -55,7 +55,6 @@ Requires-Dist: ffmpeg-python (>=0.2.0,<0.3.0) ; extra == "media-tools" or extra
 Requires-Dist: firecrawl-py (>=1.0.0,<2.0.0) ; extra == "web-tools" or extra == "all"
 Requires-Dist: fish-audio-sdk (>=2024.12.5,<2025.0.0) ; extra == "model-platforms" or extra == "all"
 Requires-Dist: google-cloud-storage (>=2.18.0,<3.0.0) ; extra == "storage" or extra == "all"
-Requires-Dist: google-generativeai (>=0.6.0,<0.7.0) ; extra == "model-platforms" or extra == "all"
 Requires-Dist: googlemaps (>=4.10.0,<5.0.0) ; extra == "web-tools" or extra == "all"
 Requires-Dist: httpx (>=0.23.0,<0.27.3)
 Requires-Dist: imageio[pyav] (>=2.34.2,<3.0.0) ; extra == "media-tools" or extra == "all"
@@ -70,21 +69,21 @@ Requires-Dist: nebula3-python (==3.8.2) ; extra == "rag" or extra == "storage" o
 Requires-Dist: neo4j (>=5.18.0,<6.0.0) ; extra == "rag" or extra == "storage" or extra == "all"
 Requires-Dist: newspaper3k (>=0.2.8,<0.3.0) ; extra == "web-tools" or extra == "all"
 Requires-Dist: notion-client (>=2.2.1,<3.0.0) ; extra == "communication-tools" or extra == "all"
-Requires-Dist: numpy (>=1,<2)
-Requires-Dist: openai (>=1.58.1,<2.0.0)
+Requires-Dist: numpy (>=1.26,<2.0)
+Requires-Dist: openai (>=1.59.7,<2.0.0)
 Requires-Dist: openapi-spec-validator (>=0.7.1,<0.8.0) ; extra == "document-tools" or extra == "all"
 Requires-Dist: openbb (>=4.3.5,<5.0.0) ; extra == "data-tools" or extra == "all"
 Requires-Dist: opencv-python (>=4,<5) ; extra == "huggingface" or extra == "all"
 Requires-Dist: outlines (>=0.1.7,<0.2.0) ; extra == "all"
 Requires-Dist: pandas (>=1.5.3,<2.0.0) ; extra == "data-tools" or extra == "all"
 Requires-Dist: pandasai (>=2.3.0,<3.0.0) ; extra == "rag" or extra == "document-tools" or extra == "all"
-Requires-Dist: pandoc
-Requires-Dist: pathlib (>=1.0.1,<2.0.0)
+Requires-Dist: pandoc (>=2.4,<3.0)
 Requires-Dist: pdfplumber (>=0.11.0,<0.12.0) ; extra == "document-tools" or extra == "all"
 Requires-Dist: pillow (>=10.1.0,<11.0.0) ; extra == "media-tools" or extra == "all"
 Requires-Dist: prance (>=23.6.21.0,<24.0.0.0) ; extra == "document-tools" or extra == "all"
 Requires-Dist: praw (>=7.7.1,<8.0.0) ; extra == "communication-tools" or extra == "all"
-Requires-Dist: protobuf (>=4,<5)
+Requires-Dist: protobuf (>=5,<6)
+Requires-Dist: psutil (>=5.9.8,<6.0.0)
 Requires-Dist: pyTelegramBotAPI (>=4.18.0,<5.0.0) ; extra == "communication-tools" or extra == "all"
 Requires-Dist: pydantic (>=1.9,<2.10)
 Requires-Dist: pydub (>=0.25.1,<0.26.0) ; extra == "media-tools" or extra == "all"
@@ -95,7 +94,6 @@ Requires-Dist: pytest (>=7,<8) ; extra == "test"
 Requires-Dist: pytest-asyncio (>=0.23.0,<0.24.0) ; extra == "test"
 Requires-Dist: pyyaml (>=6.0.2,<7.0.0)
 Requires-Dist: qdrant-client (>=1.9.0,<2.0.0) ; extra == "rag" or extra == "storage" or extra == "all"
-Requires-Dist: ragas (<=0.1.6) ; extra == "all"
 Requires-Dist: rank-bm25 (>=0.2.2,<0.3.0) ; extra == "rag" or extra == "all"
 Requires-Dist: redis (>=5.0.6,<6.0.0) ; extra == "storage" or extra == "all"
 Requires-Dist: reka-api (>=3.0.8,<4.0.0) ; extra == "model-platforms" or extra == "all"
@@ -103,12 +101,13 @@ Requires-Dist: requests_oauthlib (>=1.3.1,<2.0.0) ; extra == "web-tools" or extr
 Requires-Dist: rouge (>=1.0.1,<2.0.0) ; extra == "data-tools" or extra == "all"
 Requires-Dist: scholarly[tor] (==1.7.11) ; extra == "research-tools" or extra == "all"
 Requires-Dist: sentence-transformers (>=3.0.1,<4.0.0) ; extra == "rag" or extra == "all"
-Requires-Dist: sentencepiece (>=0,<1) ; extra == "huggingface" or extra == "all"
+Requires-Dist: sentencepiece (>=0.2,<0.3) ; extra == "huggingface" or extra == "all"
 Requires-Dist: sglang (>=0.4.0,<0.5.0) ; extra == "model-platforms" or extra == "all"
 Requires-Dist: slack-bolt (>=1.20.1,<2.0.0) ; extra == "communication-tools" or extra == "all"
 Requires-Dist: slack-sdk (>=3.27.2,<4.0.0) ; extra == "communication-tools" or extra == "all"
-Requires-Dist: soundfile (>=0,<1) ; extra == "huggingface" or extra == "all"
+Requires-Dist: soundfile (>=0.13,<0.14) ; extra == "huggingface" or extra == "all"
 Requires-Dist: stripe (>=11.3.0,<12.0.0) ; extra == "data-tools" or extra == "all"
+Requires-Dist: sympy (>=1.13.3,<2.0.0) ; extra == "web-tools" or extra == "all"
 Requires-Dist: tavily-python (>=0.5.0,<0.6.0) ; extra == "web-tools" or extra == "all"
 Requires-Dist: textblob (>=0.17.1,<0.18.0) ; extra == "data-tools" or extra == "all"
 Requires-Dist: tiktoken (>=0.7.0,<0.8.0)
@@ -408,7 +407,8 @@ Practical guides and tutorials for implementing specific functionalities in CAME
 | **[CoT Data Generation and SFT Qwen with Unsolth](https://docs.camel-ai.org/cookbooks/data_generation/cot_data_gen_sft_qwen_unsolth_upload_huggingface.html)** | Discover how to generate CoT data using CAMEL and SFT Qwen with Unsolth, and seamlessly upload your data and model to Huggingface. |
 | **[Agentic Data Generation, Evaluation & Filtering with Reward Models](https://docs.camel-ai.org/cookbooks/data_generation/synthetic_dataevaluation&filter_with_reward_model.html)** | Discover methods for generating, evaluating, and filtering agentic data using reward models to enhance the quality and efficiency of your synthetic data pipelines. |
 | **[Data Model Generation and Structured Output with Qwen Model](https://docs.camel-ai.org/cookbooks/data_generation/data_model_generation_and_structured_output_with_qwen.html)** |Learn how to generate data models and structured outputs using the Qwen Model for improved data representation.|
-
+| **[Distill Math Reasoning Data from DeepSeek R1](https://docs.camel-ai.org/cookbooks/data_generation/distill_math_reasoning_data_from_deepseek_r1.html)** |Learn  how to set up and leverage CAMEL's data distillation pipline for distilling high-quality maths reasoning data with thought process (Long CoT data)from deepseek R1, and uploading the results to Hugging Face.|
+| **[Self-Improving Math Reasoning Data Distillation from DeepSeek R1](https://docs.camel-ai.org/cookbooks/data_generation/self_improving_math_reasoning_data_distillation_from_deepSeek_r1.html)** |Learn how to set up and leverage CAMEL's data distillation pipline for self-improving math reasoning data distillation from deepseek R1, and uploading the results to Hugging Face.|
 
 ### Multi-Agent Systems & Applications
 | Cookbook | Description |
@@ -418,6 +418,7 @@ Practical guides and tutorials for implementing specific functionalities in CAME
 | **[Dynamic Travel Planner Role-Playing: Multi-Agent System with Real-Time Insights Powered by Dappier](https://docs.camel-ai.org/cookbooks/applications/dynamic_travel_planner.html)** | Explore an innovative approach to travel planning, blending AI-driven role-playing and real-time data for seamless experiences. |
 | **[Customer Service Discord Bot with Agentic RAG](https://docs.camel-ai.org/cookbooks/applications/customer_service_Discord_bot_using_SambaNova_with_agentic_RAG.html)** | Learn how to build a robust customer service bot for Discord using Agentic RAG. |
 | **[Customer Service Discord Bot with Local Model](https://docs.camel-ai.org/cookbooks/applications/customer_service_Discord_bot_using_local_model_with_agentic_RAG.html)** | Learn how to build a robust customer service bot for Discord using Agentic RAG which supports local deployment. |
+| **[Customer Service Discord Bot for Finance with OpenBB](https://docs.camel-ai.org/cookbooks/applications/finance_discord_bot.html)**| Learn how to build a sipmle yet powerful financial data assistant Discord bot using OpenBB tools. |
 
 ### Data Processing
 | Cookbook | Description |
@@ -456,6 +457,10 @@ We implemented amazing research ideas from other works for you to build, compare
 
 - `Self-Instruct` from *Yizhong Wang et al.*: [SELF-INSTRUCT: Aligning Language Models with Self-Generated Instructions](https://arxiv.org/pdf/2212.10560). [[Example](https://github.com/camel-ai/camel/blob/master/examples/datagen/self_instruct/self_instruct.py)]
 
+- `Source2Synth` from *Alisia Lupidi et al.*: [Source2Synth: Synthetic Data Generation and Curation Grounded in Real Data Sources](https://arxiv.org/abs/2409.08239). [[Example](https://github.com/camel-ai/camel/blob/master/examples/datagen/source2synth.py)]
+
+- `STaR` from *Eric Zelikman et al.*: [STaR: Bootstrapping Reasoning With Reasoning](https://arxiv.org/abs/2203.14465). [[Example](https://github.com/camel-ai/camel/blob/master/examples/datagen/star)]
+
 ## Other Research Works Based on Camel
 - [Agent Trust](http://agent-trust.camel-ai.org/): Can Large Language Model Agents Simulate Human Trust Behavior?