camel-ai 0.2.19__py3-none-any.whl → 0.2.20__py3-none-any.whl

camel/toolkits/whatsapp_toolkit.py

@@ -19,7 +19,7 @@ import requests
 
 from camel.toolkits import FunctionTool
 from camel.toolkits.base import BaseToolkit
-from camel.utils.commons import retry_request
+from camel.utils import retry_on_error
 
 
 class WhatsAppToolkit(BaseToolkit):
@@ -36,18 +36,8 @@ class WhatsAppToolkit(BaseToolkit):
         version (str): API version.
     """
 
-    def __init__(self, retries: int = 3, delay: int = 1):
-        r"""Initializes the WhatsAppToolkit with the specified number of
-        retries and delay.
-
-        Args:
-            retries (int): Number of times to retry the request in case of
-                failure. (default: :obj:`3`)
-            delay (int): Time in seconds to wait between retries.
-                (default: :obj:`1`)
-        """
-        self.retries = retries
-        self.delay = delay
+    def __init__(self):
+        r"""Initializes the WhatsAppToolkit."""
         self.base_url = "https://graph.facebook.com"
         self.version = "v17.0"
 
@@ -61,6 +51,7 @@ class WhatsAppToolkit(BaseToolkit):
                 "WHATSAPP_PHONE_NUMBER_ID environment variables."
             )
 
+    @retry_on_error()
     def send_message(
         self, to: str, message: str
     ) -> Union[Dict[str, Any], str]:
@@ -88,19 +79,15 @@ class WhatsAppToolkit(BaseToolkit):
         }
 
         try:
-            response = retry_request(
-                requests.post,
-                retries=self.retries,
-                delay=self.delay,
-                url=url,
-                headers=headers,
-                json=data,
-            )
+            response = requests.post(url=url, headers=headers, json=data)
             response.raise_for_status()
             return response.json()
+        except requests.exceptions.RequestException as e:
+            raise e
         except Exception as e:
             return f"Failed to send message: {e!s}"
 
+    @retry_on_error()
     def get_message_templates(self) -> Union[List[Dict[str, Any]], str]:
         r"""Retrieves all message templates for the WhatsApp Business account.
 
@@ -116,18 +103,13 @@ class WhatsAppToolkit(BaseToolkit):
         headers = {"Authorization": f"Bearer {self.access_token}"}
 
         try:
-            response = retry_request(
-                requests.get,
-                retries=self.retries,
-                delay=self.delay,
-                url=url,
-                headers=headers,
-            )
+            response = requests.get(url=url, headers=headers)
             response.raise_for_status()
             return response.json().get("data", [])
         except Exception as e:
             return f"Failed to retrieve message templates: {e!s}"
 
+    @retry_on_error()
     def get_business_profile(self) -> Union[Dict[str, Any], str]:
         r"""Retrieves the WhatsApp Business profile information.
 
@@ -149,10 +131,7 @@ class WhatsAppToolkit(BaseToolkit):
         }
 
         try:
-            response = retry_request(
-                requests.get,
-                retries=self.retries,
-                delay=self.delay,
+            response = requests.get(
                 url=url,
                 headers=headers,
                 params=params,

camel/types/enums.py

@@ -40,7 +40,17 @@ class ModelType(UnifiedModelType, Enum):
     O3_MINI = "o3-mini"
 
     GLM_4 = "glm-4"
-    GLM_4V = 'glm-4v'
+    GLM_4V = "glm-4v"
+    GLM_4V_FLASH = "glm-4v-flash"
+    GLM_4V_PLUS_0111 = "glm-4v-plus-0111"
+    GLM_4_PLUS = "glm-4-plus"
+    GLM_4_AIR = "glm-4-air"
+    GLM_4_AIR_0111 = "glm-4-air-0111"
+    GLM_4_AIRX = "glm-4-airx"
+    GLM_4_LONG = "glm-4-long"
+    GLM_4_FLASHX = "glm-4-flashx"
+    GLM_4_FLASH = "glm-4-flash"
+    GLM_ZERO_PREVIEW = "glm-zero-preview"
     GLM_3_TURBO = "glm-3-turbo"
 
     # Groq platform models
@@ -106,9 +116,12 @@ class ModelType(UnifiedModelType, Enum):
     NVIDIA_LLAMA3_3_70B_INSTRUCT = "meta/llama-3.3-70b-instruct"
 
     # Gemini models
+    GEMINI_2_0_FLASH = "gemini-2.0-flash-exp"
+    GEMINI_2_0_FLASH_THINKING = "gemini-2.0-flash-thinking-exp"
+    GEMINI_2_0_PRO_EXP = "gemini-2.0-pro-exp-02-05"
+    GEMINI_2_0_FLASH_LITE_PREVIEW = "gemini-2.0-flash-lite-preview-02-05"
     GEMINI_1_5_FLASH = "gemini-1.5-flash"
     GEMINI_1_5_PRO = "gemini-1.5-pro"
-    GEMINI_EXP_1114 = "gemini-exp-1114"
 
     # Mistral AI models
     MISTRAL_3B = "ministral-3b-latest"
@@ -145,10 +158,12 @@ class ModelType(UnifiedModelType, Enum):
     QWEN_MATH_TURBO = "qwen-math-turbo"
     QWEN_CODER_TURBO = "qwen-coder-turbo"
     QWEN_2_5_CODER_32B = "qwen2.5-coder-32b-instruct"
+    QWEN_2_5_VL_72B = "qwen2.5-vl-72b-instruct"
     QWEN_2_5_72B = "qwen2.5-72b-instruct"
     QWEN_2_5_32B = "qwen2.5-32b-instruct"
     QWEN_2_5_14B = "qwen2.5-14b-instruct"
     QWEN_QWQ_32B = "qwq-32b-preview"
+    QWEN_QVQ_72B = "qvq-72b-preview"
 
     # Yi models (01-ai)
     YI_LIGHTNING = "yi-lightning"
@@ -170,6 +185,29 @@ class ModelType(UnifiedModelType, Enum):
     INTERNLM2_5_LATEST = "internlm2.5-latest"
     INTERNLM2_PRO_CHAT = "internlm2-pro-chat"
 
+    # Moonshot models
+    MOONSHOT_V1_8K = "moonshot-v1-8k"
+    MOONSHOT_V1_32K = "moonshot-v1-32k"
+    MOONSHOT_V1_128K = "moonshot-v1-128k"
+
+    # SiliconFlow models support tool calling
+    SILICONFLOW_DEEPSEEK_V2_5 = "deepseek-ai/DeepSeek-V2.5"
+    SILICONFLOW_DEEPSEEK_V3 = "deepseek-ai/DeepSeek-V3"
+    SILICONFLOW_INTERN_LM2_5_20B_CHAT = "internlm/internlm2_5-20b-chat"
+    SILICONFLOW_INTERN_LM2_5_7B_CHAT = "internlm/internlm2_5-7b-chat"
+    SILICONFLOW_PRO_INTERN_LM2_5_7B_CHAT = "Pro/internlm/internlm2_5-7b-chat"
+    SILICONFLOW_QWEN2_5_72B_INSTRUCT = "Qwen/Qwen2.5-72B-Instruct"
+    SILICONFLOW_QWEN2_5_32B_INSTRUCT = "Qwen/Qwen2.5-32B-Instruct"
+    SILICONFLOW_QWEN2_5_14B_INSTRUCT = "Qwen/Qwen2.5-14B-Instruct"
+    SILICONFLOW_QWEN2_5_7B_INSTRUCT = "Qwen/Qwen2.5-7B-Instruct"
+    SILICONFLOW_PRO_QWEN2_5_7B_INSTRUCT = "Pro/Qwen/Qwen2.5-7B-Instruct"
+    SILICONFLOW_THUDM_GLM_4_9B_CHAT = "THUDM/glm-4-9b-chat"
+    SILICONFLOW_PRO_THUDM_GLM_4_9B_CHAT = "Pro/THUDM/glm-4-9b-chat"
+
+    # AIML models support tool calling
+    AIML_MIXTRAL_8X7B = "mistralai/Mixtral-8x7B-Instruct-v0.1"
+    AIML_MISTRAL_7B_INSTRUCT = "mistralai/Mistral-7B-Instruct-v0.1"
+
     def __str__(self):
         return self.value
 
@@ -184,7 +222,11 @@ class ModelType(UnifiedModelType, Enum):
 
     @property
     def support_native_structured_output(self) -> bool:
-        return self.is_openai
+        return any(
+            [
+                self.is_openai,
+            ]
+        )
 
     @property
     def support_native_tool_calling(self) -> bool:
@@ -201,6 +243,10 @@ class ModelType(UnifiedModelType, Enum):
                 self.is_sambanova,
                 self.is_groq,
                 self.is_sglang,
+                self.is_moonshot,
+                self.is_siliconflow,
+                self.is_zhipuai,
+                self.is_aiml,
             ]
         )
 
@@ -239,6 +285,16 @@ class ModelType(UnifiedModelType, Enum):
             ModelType.GLM_3_TURBO,
             ModelType.GLM_4,
             ModelType.GLM_4V,
+            ModelType.GLM_4V_FLASH,
+            ModelType.GLM_4V_PLUS_0111,
+            ModelType.GLM_4_PLUS,
+            ModelType.GLM_4_AIR,
+            ModelType.GLM_4_AIR_0111,
+            ModelType.GLM_4_AIRX,
+            ModelType.GLM_4_LONG,
+            ModelType.GLM_4_FLASHX,
+            ModelType.GLM_4_FLASH,
+            ModelType.GLM_ZERO_PREVIEW,
         }
 
     @property
@@ -286,7 +342,7 @@ class ModelType(UnifiedModelType, Enum):
 
     @property
     def is_sambanova(self) -> bool:
-        r"""Returns whether this type of models is served by SambaNova AI."""
+        r"""Returns whether this type of model is served by SambaNova AI."""
         return self in {
             ModelType.SAMBA_LLAMA_3_1_8B,
             ModelType.SAMBA_LLAMA_3_1_70B,
@@ -335,9 +391,12 @@ class ModelType(UnifiedModelType, Enum):
             bool: Whether this type of models is gemini.
         """
         return self in {
+            ModelType.GEMINI_2_0_FLASH,
             ModelType.GEMINI_1_5_FLASH,
             ModelType.GEMINI_1_5_PRO,
-            ModelType.GEMINI_EXP_1114,
+            ModelType.GEMINI_2_0_FLASH_THINKING,
+            ModelType.GEMINI_2_0_PRO_EXP,
+            ModelType.GEMINI_2_0_FLASH_LITE_PREVIEW,
         }
 
     @property
@@ -400,10 +459,12 @@ class ModelType(UnifiedModelType, Enum):
             ModelType.QWEN_MATH_TURBO,
             ModelType.QWEN_CODER_TURBO,
             ModelType.QWEN_2_5_CODER_32B,
+            ModelType.QWEN_2_5_VL_72B,
             ModelType.QWEN_2_5_72B,
             ModelType.QWEN_2_5_32B,
             ModelType.QWEN_2_5_14B,
             ModelType.QWEN_QWQ_32B,
+            ModelType.QWEN_QVQ_72B,
         }
 
     @property
@@ -422,6 +483,14 @@ class ModelType(UnifiedModelType, Enum):
             ModelType.INTERNLM2_PRO_CHAT,
         }
 
+    @property
+    def is_moonshot(self) -> bool:
+        return self in {
+            ModelType.MOONSHOT_V1_8K,
+            ModelType.MOONSHOT_V1_32K,
+            ModelType.MOONSHOT_V1_128K,
+        }
+
     @property
     def is_sglang(self) -> bool:
         return self in {
@@ -436,6 +505,30 @@ class ModelType(UnifiedModelType, Enum):
             ModelType.SGLANG_QWEN_2_5_72B,
         }
 
+    @property
+    def is_siliconflow(self) -> bool:
+        return self in {
+            ModelType.SILICONFLOW_DEEPSEEK_V2_5,
+            ModelType.SILICONFLOW_DEEPSEEK_V3,
+            ModelType.SILICONFLOW_INTERN_LM2_5_20B_CHAT,
+            ModelType.SILICONFLOW_INTERN_LM2_5_7B_CHAT,
+            ModelType.SILICONFLOW_PRO_INTERN_LM2_5_7B_CHAT,
+            ModelType.SILICONFLOW_QWEN2_5_72B_INSTRUCT,
+            ModelType.SILICONFLOW_QWEN2_5_32B_INSTRUCT,
+            ModelType.SILICONFLOW_QWEN2_5_14B_INSTRUCT,
+            ModelType.SILICONFLOW_QWEN2_5_7B_INSTRUCT,
+            ModelType.SILICONFLOW_PRO_QWEN2_5_7B_INSTRUCT,
+            ModelType.SILICONFLOW_THUDM_GLM_4_9B_CHAT,
+            ModelType.SILICONFLOW_PRO_THUDM_GLM_4_9B_CHAT,
+        }
+
+    @property
+    def is_aiml(self) -> bool:
+        return self in {
+            ModelType.AIML_MIXTRAL_8X7B,
+            ModelType.AIML_MISTRAL_7B_INSTRUCT,
+        }
+
     @property
     def token_limit(self) -> int:
         r"""Returns the maximum token limit for a given model.
@@ -469,6 +562,9 @@ class ModelType(UnifiedModelType, Enum):
             ModelType.QWEN_VL_PLUS,
             ModelType.NVIDIA_LLAMA3_70B,
             ModelType.TOGETHER_MISTRAL_7B,
+            ModelType.MOONSHOT_V1_8K,
+            ModelType.GLM_4V_FLASH,
+            ModelType.GLM_4_AIRX,
         }:
             return 8_192
         elif self in {
@@ -481,6 +577,8 @@ class ModelType(UnifiedModelType, Enum):
             ModelType.YI_LARGE_RAG,
             ModelType.SAMBA_LLAMA_3_1_8B,
             ModelType.SAMBA_LLAMA_3_1_405B,
+            ModelType.GLM_4V_PLUS_0111,
+            ModelType.GLM_ZERO_PREVIEW,
         }:
             return 16_384
         elif self in {
@@ -496,12 +594,16 @@ class ModelType(UnifiedModelType, Enum):
             ModelType.NVIDIA_MISTRAL_LARGE,
             ModelType.NVIDIA_MIXTRAL_8X7B,
             ModelType.QWEN_QWQ_32B,
+            ModelType.QWEN_QVQ_72B,
             ModelType.INTERNLM3_8B_INSTRUCT,
             ModelType.INTERNLM3_LATEST,
             ModelType.INTERNLM2_5_LATEST,
             ModelType.INTERNLM2_PRO_CHAT,
             ModelType.TOGETHER_MIXTRAL_8_7B,
             ModelType.SGLANG_MISTRAL_7B,
+            ModelType.MOONSHOT_V1_32K,
+            ModelType.AIML_MIXTRAL_8X7B,
+            ModelType.AIML_MISTRAL_7B_INSTRUCT,
         }:
             return 32_768
         elif self in {
@@ -527,6 +629,7 @@ class ModelType(UnifiedModelType, Enum):
             ModelType.MISTRAL_8B,
             ModelType.MISTRAL_3B,
             ModelType.QWEN_2_5_CODER_32B,
+            ModelType.QWEN_2_5_VL_72B,
             ModelType.QWEN_2_5_72B,
             ModelType.QWEN_2_5_32B,
             ModelType.QWEN_2_5_14B,
@@ -546,6 +649,12 @@ class ModelType(UnifiedModelType, Enum):
             ModelType.SGLANG_LLAMA_3_1_405B,
             ModelType.SGLANG_LLAMA_3_2_1B,
             ModelType.SGLANG_MIXTRAL_NEMO,
+            ModelType.MOONSHOT_V1_128K,
+            ModelType.GLM_4_PLUS,
+            ModelType.GLM_4_AIR,
+            ModelType.GLM_4_AIR_0111,
+            ModelType.GLM_4_FLASHX,
+            ModelType.GLM_4_FLASH,
         }:
             return 128_000
         elif self in {
@@ -579,9 +688,13 @@ class ModelType(UnifiedModelType, Enum):
         }:
             return 256_000
         elif self in {
+            ModelType.GEMINI_2_0_FLASH,
             ModelType.GEMINI_1_5_FLASH,
             ModelType.GEMINI_1_5_PRO,
-            ModelType.GEMINI_EXP_1114,  # Not given in docs, assuming the same
+            ModelType.GEMINI_2_0_FLASH_THINKING,
+            ModelType.GEMINI_2_0_FLASH_LITE_PREVIEW,
+            ModelType.GEMINI_2_0_PRO_EXP,  # Not given in doc, assume the same
+            ModelType.GLM_4_LONG,
         }:
             return 1_048_576
         elif self in {
@@ -767,6 +880,9 @@ class ModelPlatformType(Enum):
     DEEPSEEK = "deepseek"
     SGLANG = "sglang"
     INTERNLM = "internlm"
+    MOONSHOT = "moonshot"
+    SILICONFLOW = "siliconflow"
+    AIML = "aiml"
 
     @property
     def is_openai(self) -> bool:
@@ -874,6 +990,21 @@ class ModelPlatformType(Enum):
         r"""Returns whether this platform is InternLM."""
         return self is ModelPlatformType.INTERNLM
 
+    @property
+    def is_moonshot(self) -> bool:
+        r"""Returns whether this platform is Moonshot model."""
+        return self is ModelPlatformType.MOONSHOT
+
+    @property
+    def is_siliconflow(self) -> bool:
+        r"""Returns whether this platform is SiliconFlow."""
+        return self is ModelPlatformType.SILICONFLOW
+
+    @property
+    def is_aiml(self) -> bool:
+        r"""Returns whether this platform is AIML."""
+        return self is ModelPlatformType.AIML
+
 
 class AudioModelType(Enum):
     TTS_1 = "tts-1"

camel/types/unified_model_type.py

@@ -118,6 +118,11 @@ class UnifiedModelType(str):
         r"""Returns whether the model is a InternLM model."""
         return True
 
+    @property
+    def is_moonshot(self) -> bool:
+        r"""Returns whether this platform is Moonshot model."""
+        return True
+
     @property
     def support_native_structured_output(self) -> bool:
         r"""Returns whether the model supports native structured output."""

camel/utils/__init__.py

@@ -14,6 +14,7 @@
 
 from .commons import (
     AgentOpsMeta,
+    BatchProcessor,
     agentops_decorator,
     api_keys_required,
     check_server_running,
@@ -33,16 +34,17 @@ from .commons import (
     is_docker_running,
     json_to_function_code,
     print_text_animated,
+    retry_on_error,
     text_extract_from_web,
     to_pascal,
     track_agent,
 )
 from .constants import Constants
+from .deduplication import DeduplicationResult, deduplicate_internally
 from .response_format import get_pydantic_model
 from .token_counting import (
     AnthropicTokenCounter,
     BaseTokenCounter,
-    GeminiTokenCounter,
     LiteLLMTokenCounter,
     MistralTokenCounter,
     OpenAITokenCounter,
@@ -69,7 +71,6 @@ __all__ = [
     "dependencies_required",
     "api_keys_required",
     "is_docker_running",
-    "GeminiTokenCounter",
     "MistralTokenCounter",
     "get_pydantic_major_version",
     "get_pydantic_object_schema",
@@ -82,4 +83,8 @@ __all__ = [
     "get_pydantic_model",
     "download_github_subdirectory",
     "generate_prompt_for_structured_output",
+    "deduplicate_internally",
+    "DeduplicationResult",
+    "retry_on_error",
+    "BatchProcessor",
 ]

camel/utils/commons.py

@@ -11,7 +11,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+import functools
 import importlib
+import logging
 import os
 import platform
 import re
@@ -47,6 +49,8 @@ from .constants import Constants
 
 F = TypeVar('F', bound=Callable[..., Any])
 
+logger = logging.getLogger(__name__)
+
 
 def print_text_animated(text, delay: float = 0.02, end: str = ""):
     r"""Prints the given text with an animated effect.
@@ -620,33 +624,206 @@ def handle_http_error(response: requests.Response) -> str:
         return "HTTP Error"
 
 
-def retry_request(
-    func: Callable, retries: int = 3, delay: int = 1, *args: Any, **kwargs: Any
-) -> Any:
-    r"""Retries a function in case of any errors.
+def retry_on_error(
+    max_retries: int = 3, initial_delay: float = 1.0
+) -> Callable:
+    r"""Decorator to retry function calls on exception with exponential
+    backoff.
 
     Args:
-        func (Callable): The function to be retried.
-        retries (int): Number of retry attempts. (default: :obj:`3`)
-        delay (int): Delay between retries in seconds. (default: :obj:`1`)
-        *args: Arguments to pass to the function.
-        **kwargs: Keyword arguments to pass to the function.
+        max_retries (int): Maximum number of retry attempts
+        initial_delay (float): Initial delay between retries in seconds
 
     Returns:
-        Any: The result of the function call if successful.
+        Callable: Decorated function with retry logic
+    """
 
-    Raises:
-        Exception: If all retry attempts fail.
+    def decorator(func: Callable) -> Callable:
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs):
+            delay = initial_delay
+            last_exception = None
+
+            for attempt in range(max_retries + 1):
+                try:
+                    return func(*args, **kwargs)
+                except Exception as e:
+                    last_exception = e
+                    if attempt == max_retries:
+                        logger.error(
+                            f"Failed after {max_retries} retries: {e!s}"
+                        )
+                        raise
+
+                    logger.warning(
+                        f"Attempt {attempt + 1} failed: {e!s}. "
+                        f"Retrying in {delay:.1f}s..."
+                    )
+                    time.sleep(delay)
+                    delay *= 2  # Exponential backoff
+
+            raise last_exception
+
+        return wrapper
+
+    return decorator
+
+
+class BatchProcessor:
+    r"""Handles batch processing with dynamic sizing and error handling based
+    on system load.
     """
-    for attempt in range(retries):
-        try:
-            return func(*args, **kwargs)
-        except Exception as e:
-            print(f"Attempt {attempt + 1}/{retries} failed: {e}")
-            if attempt < retries - 1:
-                time.sleep(delay)
-            else:
-                raise
+
+    def __init__(
+        self,
+        max_workers: Optional[int] = None,
+        initial_batch_size: Optional[int] = None,
+        monitoring_interval: float = 5.0,
+        cpu_threshold: float = 80.0,
+        memory_threshold: float = 85.0,
+    ):
+        r"""Initialize the BatchProcessor with dynamic worker allocation.
+
+        Args:
+            max_workers: Maximum number of workers. If None, will be
+                determined dynamically based on system resources.
+                (default: :obj:`None`)
+            initial_batch_size: Initial size of each batch. If `None`,
+                defaults to `10`. (default: :obj:`None`)
+            monitoring_interval: Interval in seconds between resource checks.
+                (default: :obj:`5.0`)
+            cpu_threshold: CPU usage percentage threshold for scaling down.
+                (default: :obj:`80.0`)
+            memory_threshold: Memory usage percentage threshold for scaling
+                down. (default: :obj:`85.0`)
+        """
+        import psutil
+
+        self.monitoring_interval = monitoring_interval
+        self.cpu_threshold = cpu_threshold
+        self.memory_threshold = memory_threshold
+        self.last_check_time = time.time()
+        self.psutil = psutil
+
+        # Initialize performance metrics
+        self.total_processed = 0
+        self.total_errors = 0
+        self.processing_times: List = []
+
+        if max_workers is None:
+            self.max_workers = self._calculate_optimal_workers()
+        else:
+            self.max_workers = max_workers
+
+        self.batch_size = (
+            10 if initial_batch_size is None else initial_batch_size
+        )
+        self.min_batch_size = 1
+        self.max_batch_size = 20
+        self.backoff_factor = 0.8
+        self.success_factor = 1.2
+
+        # Initial resource check
+        self._update_resource_metrics()
+
+    def _calculate_optimal_workers(self) -> int:
+        r"""Calculate optimal number of workers based on system resources."""
+        cpu_count = self.psutil.cpu_count()
+        cpu_percent = self.psutil.cpu_percent(interval=1)
+        memory = self.psutil.virtual_memory()
+
+        # Base number of workers on CPU count and current load
+        if cpu_percent > self.cpu_threshold:
+            workers = max(1, cpu_count // 4)
+        elif cpu_percent > 60:
+            workers = max(1, cpu_count // 2)
+        else:
+            workers = max(1, cpu_count - 1)
+
+        # Further reduce if memory is constrained
+        if memory.percent > self.memory_threshold:
+            workers = max(1, workers // 2)
+
+        return workers
+
+    def _update_resource_metrics(self) -> None:
+        r"""Update current resource usage metrics."""
+        self.current_cpu = self.psutil.cpu_percent()
+        self.current_memory = self.psutil.virtual_memory().percent
+        self.last_check_time = time.time()
+
+    def _should_check_resources(self) -> bool:
+        r"""Determine if it's time to check resource usage again."""
+        return time.time() - self.last_check_time >= self.monitoring_interval
+
+    def adjust_batch_size(
+        self, success: bool, processing_time: Optional[float] = None
+    ) -> None:
+        r"""Adjust batch size based on success/failure and system resources.
+
+        Args:
+            success (bool): Whether the last batch completed successfully
+            processing_time (Optional[float]): Time taken to process the last
+                batch. (default: :obj:`None`)
+        """
+        # Update metrics
+        self.total_processed += 1
+        if not success:
+            self.total_errors += 1
+        if processing_time is not None:
+            self.processing_times.append(processing_time)
+
+        # Check system resources if interval has elapsed
+        if self._should_check_resources():
+            self._update_resource_metrics()
+
+            # Adjust based on resource usage
+            if (
+                self.current_cpu > self.cpu_threshold
+                or self.current_memory > self.memory_threshold
+            ):
+                self.batch_size = max(
+                    int(self.batch_size * self.backoff_factor),
+                    self.min_batch_size,
+                )
+                self.max_workers = max(1, self.max_workers - 1)
+                return
+
+        # Adjust based on success/failure
+        if success:
+            self.batch_size = min(
+                int(self.batch_size * self.success_factor), self.max_batch_size
+            )
+        else:
+            self.batch_size = max(
+                int(self.batch_size * self.backoff_factor), self.min_batch_size
+            )
+
+    def get_performance_metrics(self) -> Dict[str, Any]:
+        r"""Get current performance metrics.
+
+        Returns:
+            Dict containing performance metrics including:
+            - total_processed: Total number of batches processed
+            - error_rate: Percentage of failed batches
+            - avg_processing_time: Average time per batch
+            - current_batch_size: Current batch size
+            - current_workers: Current number of workers
+            - current_cpu: Current CPU usage percentage
+            - current_memory: Current memory usage percentage
+        """
+        metrics = {
+            "total_processed": self.total_processed,
+            "error_rate": (self.total_errors / max(1, self.total_processed))
+            * 100,
+            "avg_processing_time": sum(self.processing_times)
+            / max(1, len(self.processing_times)),
+            "current_batch_size": self.batch_size,
+            "current_workers": self.max_workers,
+            "current_cpu": self.current_cpu,
+            "current_memory": self.current_memory,
+        }
+        return metrics
 
 
 def download_github_subdirectory(