camel-ai 0.2.3a1__py3-none-any.whl → 0.2.3a2__py3-none-any.whl

camel/loaders/chunkr_reader.py

@@ -0,0 +1,163 @@
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+
+import json
+import logging
+import os
+import time
+from typing import IO, Any, Optional, Union
+
+import requests
+
+from camel.utils import api_keys_required
+
+logger = logging.getLogger(__name__)
+
+
+class ChunkrReader:
+    r"""Chunkr Reader for processing documents and returning content
+    in various formats.
+
+    Args:
+        api_key (Optional[str], optional): The API key for Chunkr API. If not
+            provided, it will be retrieved from the environment variable
+            `CHUNKR_API_KEY`. (default: :obj:`None`)
+        url (Optional[str], optional): The url to the Chunkr service.
+            (default: :obj:`https://api.chunkr.ai/api/v1/task`)
+        timeout (int, optional): The maximum time in seconds to wait for the
+            API responses. (default: :obj:`30`)
+        **kwargs (Any): Additional keyword arguments for request headers.
+    """
+
+    @api_keys_required("CHUNKR_API_KEY")
+    def __init__(
+        self,
+        api_key: Optional[str] = None,
+        url: Optional[str] = "https://api.chunkr.ai/api/v1/task",
+        timeout: int = 30,
+        **kwargs: Any,
+    ) -> None:
+        self._api_key = api_key or os.getenv('CHUNKR_API_KEY')
+        self._url = os.getenv('CHUNKR_API_URL') or url
+        self._headers = {
+            "Authorization": f"{self._api_key}",
+            **kwargs,
+        }
+        self.timeout = timeout
+
+    def submit_task(
+        self,
+        file_path: str,
+        model: str = "Fast",
+        ocr_strategy: str = "Auto",
+        target_chunk_length: str = "512",
+    ) -> str:
+        r"""Submits a file to the Chunkr API and returns the task ID.
+
+        Args:
+            file_path (str): The path to the file to be uploaded.
+            model (str, optional): The model to be used for the task.
+                (default: :obj:`Fast`)
+            ocr_strategy (str, optional): The OCR strategy. Defaults to 'Auto'.
+            target_chunk_length (str, optional): The target chunk length.
+                (default: :obj:`512`)
+
+        Returns:
+            str: The task ID.
+        """
+        with open(file_path, 'rb') as file:
+            files: dict[
+                str, Union[tuple[None, IO[bytes]], tuple[None, str]]
+            ] = {
+                'file': (
+                    None,
+                    file,
+                ),  # Properly pass the file as a binary stream
+                'model': (None, model),
+                'ocr_strategy': (None, ocr_strategy),
+                'target_chunk_length': (None, target_chunk_length),
+            }
+            try:
+                response = requests.post(
+                    self._url,  # type: ignore[arg-type]
+                    headers=self._headers,
+                    files=files,
+                    timeout=self.timeout,
+                )
+                response.raise_for_status()
+                task_id = response.json().get('task_id')
+                if not task_id:
+                    raise ValueError("Task ID not returned in the response.")
+                logger.info(f"Task submitted successfully. Task ID: {task_id}")
+                return task_id
+            except Exception as e:
+                logger.error(f"Failed to submit task: {e}")
+                raise ValueError(f"Failed to submit task: {e}") from e
+
+    def get_task_output(self, task_id: str, max_retries: int = 5) -> str:
+        r"""Polls the Chunkr API to check the task status and returns the task
+        result.
+
+        Args:
+            task_id (str): The task ID to check the status for.
+            max_retries (int, optional): Maximum number of retry attempts.
+                (default: :obj:`5`)
+
+        Returns:
+            str: The formatted task result in JSON format.
+
+        Raises:
+            ValueError: If the task status cannot be retrieved.
+            RuntimeError: If the maximum number of retries is reached without
+                a successful task completion.
+        """
+        url_get = f"{self._url}/{task_id}"
+        attempts = 0
+
+        while attempts < max_retries:
+            try:
+                response = requests.get(
+                    url_get, headers=self._headers, timeout=self.timeout
+                )
+                response.raise_for_status()
+                task_status = response.json().get('status')
+
+                if task_status == "Succeeded":
+                    logger.info(f"Task {task_id} completed successfully.")
+                    return self._pretty_print_response(response.json())
+                else:
+                    logger.info(
+                        f"Task {task_id} is still {task_status}. Retrying "
+                        "in 5 seconds..."
+                    )
+            except Exception as e:
+                logger.error(f"Failed to retrieve task status: {e}")
+                raise ValueError(f"Failed to retrieve task status: {e}") from e
+
+            attempts += 1
+            time.sleep(5)
+
+        logger.error(f"Max retries reached for task {task_id}.")
+        raise RuntimeError(f"Max retries reached for task {task_id}.")
+
+    def _pretty_print_response(self, response_json: dict) -> str:
+        r"""Pretty prints the JSON response.
+
+        Args:
+            response_json (dict): The response JSON to pretty print.
+
+        Returns:
+            str: Formatted JSON as a string.
+        """
+        return json.dumps(response_json, indent=4)

camel/loaders/firecrawl_reader.py

@@ -155,12 +155,12 @@ class Firecrawl:
         except Exception as e:
             raise RuntimeError(f"Failed to scrape the URL: {e}")
 
-    def structured_scrape(self, url: str, output_schema: BaseModel) -> Dict:
+    def structured_scrape(self, url: str, response_format: BaseModel) -> Dict:
         r"""Use LLM to extract structured data from given URL.
 
         Args:
             url (str): The URL to read.
-            output_schema (BaseModel): A pydantic model
+            response_format (BaseModel): A pydantic model
                 that includes value types and field descriptions used to
                 generate a structured response by LLM. This schema helps
                 in defining the expected output format.
@@ -176,7 +176,7 @@ class Firecrawl:
                 url,
                 {
                     'formats': ['extract'],
-                    'extract': {'schema': output_schema.model_json_schema()},
+                    'extract': {'schema': response_format.model_json_schema()},
                 },
             )
             return data.get("extract", {})

camel/loaders/unstructured_io.py

@@ -13,8 +13,9 @@
 # =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
 import uuid
 import warnings
-from io import IOBase
 from typing import (
+    IO,
+    TYPE_CHECKING,
     Any,
     Dict,
     List,
@@ -24,7 +25,8 @@ from typing import (
     Union,
 )
 
-from unstructured.documents.elements import Element
+if TYPE_CHECKING:
+    from unstructured.documents.elements import Element
 
 
 class UnstructuredIO:
@@ -40,33 +42,34 @@ class UnstructuredIO:
     @staticmethod
     def create_element_from_text(
         text: str,
-        element_id: Optional[Union[str, uuid.UUID]] = None,
+        element_id: Optional[str] = None,
         embeddings: Optional[List[float]] = None,
         filename: Optional[str] = None,
         file_directory: Optional[str] = None,
         last_modified: Optional[str] = None,
         filetype: Optional[str] = None,
-        parent_id: Optional[Union[str, uuid.UUID]] = None,
-    ) -> Element:
+        parent_id: Optional[str] = None,
+    ) -> "Element":
         r"""Creates a Text element from a given text input, with optional
         metadata and embeddings.
 
         Args:
             text (str): The text content for the element.
-            element_id (Optional[Union[str, uuid.UUID]], optional): Unique
-                identifier for the element. Defaults to `None`.
-            embeddings (Optional[List[float]], optional): A list of float
-                numbers representing the text embeddings. Defaults to `None`.
+            element_id (Optional[str], optional): Unique identifier for the
+                element. (default: :obj:`None`)
+            embeddings (List[float], optional): A list of float
+                numbers representing the text embeddings.
+                (default: :obj:`None`)
             filename (Optional[str], optional): The name of the file the
-                element is associated with. Defaults to `None`.
+                element is associated with. (default: :obj:`None`)
             file_directory (Optional[str], optional): The directory path where
-                the file is located. Defaults to `None`.
+                the file is located. (default: :obj:`None`)
             last_modified (Optional[str], optional): The last modified date of
-                the file. Defaults to `None`.
-            filetype (Optional[str], optional): The type of the file. Defaults
-                to `None`.
-            parent_id (Optional[Union[str, uuid.UUID]], optional): The
-                identifier of the parent element. Defaults to `None`.
+                the file. (default: :obj:`None`)
+            filetype (Optional[str], optional): The type of the file.
+                (default: :obj:`None`)
+            parent_id (Optional[str], optional): The identifier of the parent
+                element. (default: :obj:`None`)
 
         Returns:
             Element: An instance of Text with the provided content and
@@ -84,7 +87,7 @@ class UnstructuredIO:
 
         return Text(
             text=text,
-            element_id=element_id or uuid.uuid4(),
+            element_id=element_id or str(uuid.uuid4()),
             metadata=metadata,
             embeddings=embeddings,
         )
@@ -93,7 +96,7 @@ class UnstructuredIO:
     def parse_file_or_url(
         input_path: str,
         **kwargs: Any,
-    ) -> Union[List[Element], None]:
+    ) -> Union[List["Element"], None]:
         r"""Loads a file or a URL and parses its contents into elements.
 
         Args:
@@ -119,25 +122,23 @@ class UnstructuredIO:
         import os
         from urllib.parse import urlparse
 
+        from unstructured.partition.auto import partition
+
         # Check if the input is a URL
         parsed_url = urlparse(input_path)
         is_url = all([parsed_url.scheme, parsed_url.netloc])
 
+        # Handling URL
         if is_url:
-            # Handling URL
-            from unstructured.partition.html import partition_html
-
             try:
-                elements = partition_html(url=input_path, **kwargs)
+                elements = partition(url=input_path, **kwargs)
                 return elements
             except Exception:
                 warnings.warn(f"Failed to parse the URL: {input_path}")
                 return None
 
+        # Handling file
         else:
-            # Handling file
-            from unstructured.partition.auto import partition
-
             # Check if the file exists
             if not os.path.exists(input_path):
                 raise FileNotFoundError(
@@ -154,11 +155,13 @@ class UnstructuredIO:
                 return None
 
     @staticmethod
-    def parse_bytes(file: IOBase, **kwargs: Any) -> Union[List[Element], None]:
+    def parse_bytes(
+        file: IO[bytes], **kwargs: Any
+    ) -> Union[List["Element"], None]:
         r"""Parses a bytes stream and converts its contents into elements.
 
         Args:
-            file (IOBase): The file in bytes format to be parsed.
+            file (IO[bytes]): The file in bytes format to be parsed.
             **kwargs: Extra kwargs passed to the partition function.
 
         Returns:
@@ -181,8 +184,6 @@ class UnstructuredIO:
             elements = partition(file=file, **kwargs)
             return elements
         except Exception as e:
-            import warnings
-
             warnings.warn(f"Failed to partition the file stream: {e}")
             return None
 
@@ -196,7 +197,7 @@ class UnstructuredIO:
 
         This function applies multiple text cleaning utilities by calling the
         `unstructured` library's cleaning bricks for operations like
-        replacing unicode quotes, removing extra whitespace, dashes, non-ascii
+        replacing Unicode quotes, removing extra whitespace, dashes, non-ascii
         characters, and more.
 
         If no cleaning options are provided, a default set of cleaning
@@ -283,7 +284,8 @@ class UnstructuredIO:
                 )
             else:
                 raise ValueError(
-                    f"'{func_name}' is not a valid function in `unstructured`."
+                    f"'{func_name}' is not a valid function in "
+                    "`Unstructured IO`."
                 )
 
         return cleaned_text
@@ -440,8 +442,8 @@ class UnstructuredIO:
 
     @staticmethod
     def chunk_elements(
-        elements: List[Any], chunk_type: str, **kwargs
-    ) -> List[Element]:
+        elements: List["Element"], chunk_type: str, **kwargs
+    ) -> List["Element"]:
         r"""Chunks elements by titles.
 
         Args:

camel/messages/__init__.py

@@ -32,6 +32,7 @@ __all__ = [
     'OpenAISystemMessage',
     'OpenAIAssistantMessage',
     'OpenAIUserMessage',
+    'OpenAIFunctionMessage',
     'OpenAIMessage',
     'BaseMessage',
     'FunctionCallingMessage',

camel/models/__init__.py

@@ -21,9 +21,8 @@ from .mistral_model import MistralModel
 from .model_factory import ModelFactory
 from .nemotron_model import NemotronModel
 from .ollama_model import OllamaModel
-from .open_source_model import OpenSourceModel
 from .openai_audio_models import OpenAIAudioModels
-from .openai_compatibility_model import OpenAICompatibilityModel
+from .openai_compatible_model import OpenAICompatibleModel
 from .openai_model import OpenAIModel
 from .reka_model import RekaModel
 from .samba_model import SambaModel
@@ -41,7 +40,6 @@ __all__ = [
     'GroqModel',
     'StubModel',
     'ZhipuAIModel',
-    'OpenSourceModel',
     'ModelFactory',
     'LiteLLMModel',
     'OpenAIAudioModels',
@@ -49,7 +47,7 @@ __all__ = [
     'OllamaModel',
     'VLLMModel',
     'GeminiModel',
-    'OpenAICompatibilityModel',
+    'OpenAICompatibleModel',
     'RekaModel',
     'SambaModel',
     'TogetherAIModel',

camel/models/anthropic_model.py

@@ -12,11 +12,9 @@
 # limitations under the License.
 # =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
 import os
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List, Optional, Union
 
-from anthropic import NOT_GIVEN, Anthropic
-
-from camel.configs import ANTHROPIC_API_PARAMS
+from camel.configs import ANTHROPIC_API_PARAMS, AnthropicConfig
 from camel.messages import OpenAIMessage
 from camel.models.base_model import BaseModelBackend
 from camel.types import ChatCompletion, ModelType
@@ -24,40 +22,47 @@ from camel.utils import (
     AnthropicTokenCounter,
     BaseTokenCounter,
     api_keys_required,
+    dependencies_required,
 )
 
 
 class AnthropicModel(BaseModelBackend):
-    r"""Anthropic API in a unified BaseModelBackend interface."""
-
+    r"""Anthropic API in a unified BaseModelBackend interface.
+
+    Args:
+        model_type (Union[ModelType, str]): Model for which a backend is
+            created, one of CLAUDE_* series.
+        model_config_dict (Optional[Dict[str, Any]], optional): A dictionary
+            that will be fed into Anthropic.messages.create().  If
+            :obj:`None`, :obj:`AnthropicConfig().as_dict()` will be used.
+            (default::obj:`None`)
+        api_key (Optional[str], optional): The API key for authenticating with
+            the Anthropic service. (default: :obj:`None`)
+        url (Optional[str], optional): The url to the Anthropic service.
+            (default: :obj:`None`)
+        token_counter (Optional[BaseTokenCounter], optional): Token counter to
+            use for the model. If not provided, :obj:`AnthropicTokenCounter`
+            will be used. (default: :obj:`None`)
+    """
+
+    @dependencies_required('anthropic')
     def __init__(
         self,
-        model_type: ModelType,
-        model_config_dict: Dict[str, Any],
+        model_type: Union[ModelType, str],
+        model_config_dict: Optional[Dict[str, Any]] = None,
         api_key: Optional[str] = None,
         url: Optional[str] = None,
         token_counter: Optional[BaseTokenCounter] = None,
     ) -> None:
-        r"""Constructor for Anthropic backend.
+        from anthropic import Anthropic
 
-        Args:
-            model_type (ModelType): Model for which a backend is created,
-                one of CLAUDE_* series.
-            model_config_dict (Dict[str, Any]): A dictionary that will
-                be fed into Anthropic.messages.create().
-            api_key (Optional[str]): The API key for authenticating with the
-                Anthropic service. (default: :obj:`None`)
-            url (Optional[str]): The url to the Anthropic service. (default:
-                :obj:`None`)
-            token_counter (Optional[BaseTokenCounter]): Token counter to use
-                for the model. If not provided, `AnthropicTokenCounter` will
-                be used.
-        """
+        if model_config_dict is None:
+            model_config_dict = AnthropicConfig().as_dict()
+        api_key = api_key or os.environ.get("ANTHROPIC_API_KEY")
+        url = url or os.environ.get("ANTHROPIC_API_BASE_URL")
         super().__init__(
             model_type, model_config_dict, api_key, url, token_counter
         )
-        self._api_key = api_key or os.environ.get("ANTHROPIC_API_KEY")
-        self._url = url or os.environ.get("ANTHROPIC_API_BASE_URL")
         self.client = Anthropic(api_key=self._api_key, base_url=self._url)
 
     def _convert_response_from_anthropic_to_openai(self, response):
@@ -89,7 +94,7 @@ class AnthropicModel(BaseModelBackend):
                 tokenization style.
         """
         if not self._token_counter:
-            self._token_counter = AnthropicTokenCounter(self.model_type)
+            self._token_counter = AnthropicTokenCounter()
         return self._token_counter
 
     def count_tokens_from_prompt(self, prompt: str) -> int:
@@ -117,13 +122,14 @@ class AnthropicModel(BaseModelBackend):
         Returns:
             ChatCompletion: Response in the OpenAI API format.
         """
+        from anthropic import NOT_GIVEN
 
         if messages[0]["role"] == "system":
             sys_msg = str(messages.pop(0)["content"])
         else:
             sys_msg = NOT_GIVEN  # type: ignore[assignment]
         response = self.client.messages.create(
-            model=self.model_type.value,
+            model=self.model_type,
             system=sys_msg,
             messages=messages,  # type: ignore[arg-type]
             **self.model_config_dict,

camel/models/azure_openai_model.py

@@ -16,60 +16,65 @@ from typing import Any, Dict, List, Optional, Union
 
 from openai import AzureOpenAI, Stream
 
-from camel.configs import OPENAI_API_PARAMS
+from camel.configs import OPENAI_API_PARAMS, ChatGPTConfig
 from camel.messages import OpenAIMessage
 from camel.models.base_model import BaseModelBackend
-from camel.types import ChatCompletion, ChatCompletionChunk, ModelType
+from camel.types import (
+    ChatCompletion,
+    ChatCompletionChunk,
+    ModelType,
+)
 from camel.utils import BaseTokenCounter, OpenAITokenCounter, api_keys_required
 
 
 class AzureOpenAIModel(BaseModelBackend):
     r"""Azure OpenAI API in a unified BaseModelBackend interface.
-    Doc: https://learn.microsoft.com/en-us/azure/ai-services/openai/
+
+    Args:
+        model_type (Union[ModelType, str]): Model for which a backend is
+            created, one of GPT_* series.
+        model_config_dict (Optional[Dict[str, Any]], optional): A dictionary
+            that will be fed into:obj:`openai.ChatCompletion.create()`. If
+            :obj:`None`, :obj:`ChatGPTConfig().as_dict()` will be used.
+            (default: :obj:`None`)
+        api_key (Optional[str], optional): The API key for authenticating with
+            the OpenAI service. (default: :obj:`None`)
+        url (Optional[str], optional): The url to the OpenAI service.
+            (default: :obj:`None`)
+        api_version (Optional[str], optional): The api version for the model.
+            (default: :obj:`None`)
+        azure_deployment_name (Optional[str], optional): The deployment name
+            you chose when you deployed an azure model. (default: :obj:`None`)
+        token_counter (Optional[BaseTokenCounter], optional): Token counter to
+            use for the model. If not provided, :obj:`OpenAITokenCounter`
+            will be used. (default: :obj:`None`)
+
+    References:
+        https://learn.microsoft.com/en-us/azure/ai-services/openai/
     """
 
     def __init__(
         self,
-        model_type: ModelType,
-        model_config_dict: Dict[str, Any],
+        model_type: Union[ModelType, str],
+        model_config_dict: Optional[Dict[str, Any]] = None,
         api_key: Optional[str] = None,
         url: Optional[str] = None,
+        token_counter: Optional[BaseTokenCounter] = None,
         api_version: Optional[str] = None,
         azure_deployment_name: Optional[str] = None,
     ) -> None:
-        r"""Constructor for OpenAI backend.
+        if model_config_dict is None:
+            model_config_dict = ChatGPTConfig().as_dict()
+        api_key = api_key or os.environ.get("AZURE_OPENAI_API_KEY")
+        url = url or os.environ.get("AZURE_OPENAI_BASE_URL")
+        super().__init__(
+            model_type, model_config_dict, api_key, url, token_counter
+        )
 
-        Args:
-            model_type (ModelType): Model for which a backend is created,
-                one of GPT_* series.
-            model_config_dict (Dict[str, Any]): A dictionary that will
-                be fed into openai.ChatCompletion.create().
-            api_key (Optional[str]): The API key for authenticating with the
-                OpenAI service. (default: :obj:`None`)
-            url (Optional[str]): The url to the OpenAI service. (default:
-                :obj:`None`)
-            api_version (Optional[str]): The api version for the model.
-            azure_deployment_name (Optional[str]): The deployment name you
-                chose when you deployed an azure model. (default: :obj:`None`)
-        """
-        super().__init__(model_type, model_config_dict, api_key, url)
-        self._url = url or os.environ.get("AZURE_OPENAI_ENDPOINT")
-        self._api_key = api_key or os.environ.get("AZURE_OPENAI_API_KEY")
         self.api_version = api_version or os.environ.get("AZURE_API_VERSION")
         self.azure_deployment_name = azure_deployment_name or os.environ.get(
             "AZURE_DEPLOYMENT_NAME"
         )
-
-        if self._url is None:
-            raise ValueError(
-                "Must provide either the `url` argument "
-                "or `AZURE_OPENAI_ENDPOINT` environment variable."
-            )
-        if self._api_key is None:
-            raise ValueError(
-                "Must provide either the `api_key` argument "
-                "or `AZURE_OPENAI_API_KEY` environment variable."
-            )
         if self.api_version is None:
             raise ValueError(
                 "Must provide either the `api_version` argument "
@@ -80,7 +85,6 @@ class AzureOpenAIModel(BaseModelBackend):
                 "Must provide either the `azure_deployment_name` argument "
                 "or `AZURE_DEPLOYMENT_NAME` environment variable."
             )
-        self.model = str(self.azure_deployment_name)
 
         self._client = AzureOpenAI(
             azure_endpoint=str(self._url),
@@ -90,7 +94,6 @@ class AzureOpenAIModel(BaseModelBackend):
             timeout=60,
             max_retries=3,
         )
-        self._token_counter: Optional[BaseTokenCounter] = None
 
     @property
     def token_counter(self) -> BaseTokenCounter:
@@ -122,7 +125,7 @@ class AzureOpenAIModel(BaseModelBackend):
         """
         response = self._client.chat.completions.create(
             messages=messages,
-            model=self.model,
+            model=self.azure_deployment_name,  # type:ignore[arg-type]
             **self.model_config_dict,
         )
         return response