camel-ai 0.1.1__py3-none-any.whl → 0.1.4__py3-none-any.whl

camel/{functions/unstructured_io_fuctions.py → loaders/unstructured_io.py}

@@ -11,11 +11,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+import uuid
+from typing import Any, Dict, List, Literal, Optional, Tuple, Union
 
-from typing import Any, Dict, List, Optional, Tuple, Union
+from unstructured.documents.elements import Element
 
 
-class UnstructuredModules:
+class UnstructuredIO:
     r"""A class to handle various functionalities provided by the
     Unstructured library, including version checking, parsing, cleaning,
     extracting, staging, chunking data, and integrating with cloud
@@ -29,13 +31,13 @@ class UnstructuredModules:
     UNSTRUCTURED_MIN_VERSION = "0.10.30"  # Define the minimum version
 
     def __init__(self):
-        r"""Initializes the UnstructuredModules class and ensures the
+        r"""Initializes the UnstructuredIO class and ensures the
         installed version of Unstructured library meets the minimum
         requirements.
         """
-        self.ensure_unstructured_version(self.UNSTRUCTURED_MIN_VERSION)
+        self._ensure_unstructured_version(self.UNSTRUCTURED_MIN_VERSION)
 
-    def ensure_unstructured_version(self, min_version: str) -> None:
+    def _ensure_unstructured_version(self, min_version: str) -> None:
         r"""Validates that the installed 'Unstructured' library version
         satisfies the specified minimum version requirement. This function is
         essential for ensuring compatibility with features that depend on a
@@ -68,17 +70,74 @@ class UnstructuredModules:
         installed_ver = version.parse(__version__)
 
         if installed_ver < min_ver:
-            raise ValueError(f"Require `unstructured>={min_version}`, "
-                             f"you have {__version__}.")
+            raise ValueError(
+                f"Require `unstructured>={min_version}`, "
+                f"you have {__version__}."
+            )
+
+    def create_element_from_text(
+        self,
+        text: str,
+        element_id: Optional[Union[str, uuid.UUID]] = None,
+        embeddings: Optional[List[float]] = None,
+        filename: Optional[str] = None,
+        file_directory: Optional[str] = None,
+        last_modified: Optional[str] = None,
+        filetype: Optional[str] = None,
+        parent_id: Optional[Union[str, uuid.UUID]] = None,
+    ) -> Element:
+        r"""Creates a Text element from a given text input, with optional
+        metadata and embeddings.
+
+        Args:
+            text (str): The text content for the element.
+            element_id (Union[str, uuid.UUID], optional): Unique identifier
+                forthe element. Defaults to an empty string.
+            embeddings (Optional[List[float]], optional): A list of float
+                numbers representing the text embeddings. Defaults to `None`.
+            filename (Optional[str], optional): The name of the file the
+                element is associated with. Defaults to `None`.
+            file_directory (Optional[str], optional): The directory path where
+                the file is located. Defaults to `None`.
+            last_modified (Optional[str], optional): The last modified date of
+                the file. Defaults to `None`.
+            filetype (Optional[str], optional): The type of the file. Defaults
+                to `None`.
+            parent_id (Optional[Union[str, uuid.UUID]], optional): The
+                identifier of the parent element. Defaults to `None`.
+
+        Returns:
+            Element: An instance of Text with the provided content and
+                metadata.
+        """
+        from unstructured.documents.elements import ElementMetadata, Text
+
+        metadata = ElementMetadata(
+            filename=filename,
+            file_directory=file_directory,
+            last_modified=last_modified,
+            filetype=filetype,
+            parent_id=parent_id,
+        )
+
+        return Text(
+            text=text,
+            element_id=element_id if element_id else str(uuid.uuid4()),
+            metadata=metadata,
+            embeddings=embeddings,
+        )
 
     def parse_file_or_url(
         self,
         input_path: str,
+        **kwargs: Any,
     ) -> Union[Any, List[Any]]:
         r"""Loads a file or a URL and parses its contents as unstructured data.
 
         Args:
             input_path (str): Path to the file or URL to be parsed.
+            **kwargs: Extra kwargs passed to the partition function.
+
         Returns:
             List[Any]: The elements after parsing the file or URL, could be a
                 dict, list, etc., depending on the content. If return_str is
@@ -115,7 +174,7 @@ class UnstructuredModules:
             from unstructured.partition.html import partition_html
 
             try:
-                elements = partition_html(url=input_path)
+                elements = partition_html(url=input_path, **kwargs)
                 return elements
             except Exception as e:
                 raise Exception("Failed to parse the URL.") from e
@@ -126,17 +185,15 @@ class UnstructuredModules:
 
             # Check if the file exists
             if not os.path.exists(input_path):
-                raise FileNotFoundError(
-                    f"The file {input_path} was not found.")
+                raise FileNotFoundError(f"The file {input_path} was not found.")
 
             # Read the file
             try:
                 with open(input_path, "rb") as f:
-                    elements = partition(file=f)
+                    elements = partition(file=f, **kwargs)
                     return elements
             except Exception as e:
-                raise Exception(
-                    "Failed to parse the unstructured file.") from e
+                raise Exception("Failed to parse the unstructured file.") from e
 
     def clean_text_data(
         self,
@@ -144,10 +201,10 @@ class UnstructuredModules:
         clean_options: Optional[List[Tuple[str, Dict[str, Any]]]] = None,
     ) -> str:
         r"""Cleans text data using a variety of cleaning functions provided by
-        the `'unstructured'` library.
+        the `unstructured` library.
 
         This function applies multiple text cleaning utilities by calling the
-        `'unstructured'` library's cleaning bricks for operations like
+        `unstructured` library's cleaning bricks for operations like
         replacing unicode quotes, removing extra whitespace, dashes, non-ascii
         characters, and more.
 
@@ -158,36 +215,27 @@ class UnstructuredModules:
 
         Args:
             text (str): The text to be cleaned.
-            clean_options (dict): A dictionary specifying which
-                                cleaning options to apply. The keys should
-                                match the names of the cleaning functions,
-                                and the values should be dictionaries
-                                containing the parameters for each
-                                function. Supported types:
-                                'clean_extra_whitespace',
-                                'clean_bullets',
-                                'clean_ordered_bullets',
-                                'clean_postfix',
-                                'clean_prefix',
-                                'clean_dashes',
-                                'clean_trailing_punctuation',
-                                'clean_non_ascii_chars',
-                                'group_broken_paragraphs',
-                                'remove_punctuation',
-                                'replace_unicode_quotes',
-                                'bytes_string_to_string',
-                                'translate_text'.
+            clean_options (dict): A dictionary specifying which cleaning
+                options to apply. The keys should match the names of the
+                cleaning functions, and the values should be dictionaries
+                containing the parameters for each function. Supported types:
+                'clean_extra_whitespace', 'clean_bullets',
+                'clean_ordered_bullets', 'clean_postfix', 'clean_prefix',
+                'clean_dashes', 'clean_trailing_punctuation',
+                'clean_non_ascii_chars', 'group_broken_paragraphs',
+                'remove_punctuation', 'replace_unicode_quotes',
+                'bytes_string_to_string', 'translate_text'.
 
         Returns:
             str: The cleaned text.
 
         Raises:
             AttributeError: If a cleaning option does not correspond to a
-                valid cleaning function in 'unstructured'.
+                valid cleaning function in `unstructured`.
 
         Notes:
             The 'options' dictionary keys must correspond to valid cleaning
-            brick names from the 'unstructured' library.
+            brick names from the `unstructured` library.
             Each brick's parameters must be provided in a nested dictionary
             as the value for the key.
 
@@ -239,32 +287,43 @@ class UnstructuredModules:
         cleaned_text = text
         for func_name, params in clean_options:
             if func_name in cleaning_functions:
-                cleaned_text = cleaning_functions[func_name](cleaned_text,
-                                                             **params)
+                cleaned_text = cleaning_functions[func_name](
+                    cleaned_text, **params
+                )
             else:
                 raise ValueError(
-                    f"'{func_name}' is not a valid function in 'unstructured'."
+                    f"'{func_name}' is not a valid function in `unstructured`."
                 )
 
         return cleaned_text
 
-    def extract_data_from_text(self, text: str, extract_type: str,
-                               **kwargs) -> Any:
+    def extract_data_from_text(
+        self,
+        text: str,
+        extract_type: Literal[
+            'extract_datetimetz',
+            'extract_email_address',
+            'extract_ip_address',
+            'extract_ip_address_name',
+            'extract_mapi_id',
+            'extract_ordered_bullets',
+            'extract_text_after',
+            'extract_text_before',
+            'extract_us_phone_number',
+        ],
+        **kwargs,
+    ) -> Any:
         r"""Extracts various types of data from text using functions from
         unstructured.cleaners.extract.
 
         Args:
             text (str): Text to extract data from.
-            extract_type (str): Type of data to extract. Supported types:
-                                'extract_datetimetz',
-                                'extract_email_address',
-                                'extract_ip_address',
-                                'extract_ip_address_name',
-                                'extract_mapi_id',
-                                'extract_ordered_bullets',
-                                'extract_text_after',
-                                'extract_text_before',
-                                'extract_us_phone_number'.
+            extract_type (Literal['extract_datetimetz',
+                'extract_email_address', 'extract_ip_address',
+                'extract_ip_address_name', 'extract_mapi_id',
+                'extract_ordered_bullets', 'extract_text_after',
+                'extract_text_before', 'extract_us_phone_number']): Type of
+                data to extract.
             **kwargs: Additional keyword arguments for specific
                 extraction functions.
 
@@ -304,8 +363,24 @@ class UnstructuredModules:
 
         return extraction_functions[extract_type](text, **kwargs)
 
-    def stage_elements(self, elements: List[Any], stage_type: str,
-                       **kwargs) -> Union[str, List[Dict], Any]:
+    def stage_elements(
+        self,
+        elements: List[Any],
+        stage_type: Literal[
+            'convert_to_csv',
+            'convert_to_dataframe',
+            'convert_to_dict',
+            'dict_to_elements',
+            'stage_csv_for_prodigy',
+            'stage_for_prodigy',
+            'stage_for_baseplate',
+            'stage_for_datasaur',
+            'stage_for_label_box',
+            'stage_for_label_studio',
+            'stage_for_weaviate',
+        ],
+        **kwargs,
+    ) -> Union[str, List[Dict], Any]:
         r"""Stages elements for various platforms based on the
         specified staging type.
 
@@ -317,19 +392,12 @@ class UnstructuredModules:
 
         Args:
             elements (List[Any]): List of Element objects to be staged.
-            stage_type (str): Type of staging to perform. Supported types:
-                            'convert_to_csv',
-                            'convert_to_dataframe',
-                            'convert_to_dict',
-                            'dict_to_elements',
-                            'stage_csv_for_prodigy',
-                            'stage_for_prodigy',
-                            'stage_for_argilla',
-                            'stage_for_baseplate',
-                            'stage_for_datasaur',
-                            'stage_for_label_box',
-                            'stage_for_label_studio',
-                            'stage_for_weaviate'.
+            stage_type (Literal['convert_to_csv', 'convert_to_dataframe',
+                'convert_to_dict', 'dict_to_elements',
+                'stage_csv_for_prodigy', 'stage_for_prodigy',
+                'stage_for_baseplate', 'stage_for_datasaur',
+                'stage_for_label_box', 'stage_for_label_studio',
+                'stage_for_weaviate']): Type of staging to perform.
             **kwargs: Additional keyword arguments specific to
                 the staging type.
 
@@ -345,7 +413,6 @@ class UnstructuredModules:
         """
 
         from unstructured.staging import (
-            argilla,
             base,
             baseplate,
             datasaur,
@@ -356,33 +423,24 @@ class UnstructuredModules:
         )
 
         staging_functions = {
-            "convert_to_csv":
-            base.convert_to_csv,
-            "convert_to_dataframe":
-            base.convert_to_dataframe,
-            "convert_to_dict":
-            base.convert_to_dict,
-            "dict_to_elements":
-            base.dict_to_elements,
-            "stage_csv_for_prodigy":
-            lambda els, **kw: prodigy.stage_csv_for_prodigy(
-                els, kw.get('metadata', [])),
-            "stage_for_prodigy":
-            lambda els, **kw: prodigy.stage_for_prodigy(
-                els, kw.get('metadata', [])),
-            "stage_for_argilla":
-            lambda els, **kw: argilla.stage_for_argilla(els, **kw),
-            "stage_for_baseplate":
-            baseplate.stage_for_baseplate,
-            "stage_for_datasaur":
-            lambda els, **kw: datasaur.stage_for_datasaur(
-                els, kw.get('entities', [])),
-            "stage_for_label_box":
-            lambda els, **kw: label_box.stage_for_label_box(els, **kw),
-            "stage_for_label_studio":
-            lambda els, **kw: label_studio.stage_for_label_studio(els, **kw),
-            "stage_for_weaviate":
-            weaviate.stage_for_weaviate,
+            "convert_to_csv": base.convert_to_csv,
+            "convert_to_dataframe": base.convert_to_dataframe,
+            "convert_to_dict": base.convert_to_dict,
+            "dict_to_elements": base.dict_to_elements,
+            "stage_csv_for_prodigy": lambda els,
+            **kw: prodigy.stage_csv_for_prodigy(els, kw.get('metadata', [])),
+            "stage_for_prodigy": lambda els, **kw: prodigy.stage_for_prodigy(
+                els, kw.get('metadata', [])
+            ),
+            "stage_for_baseplate": baseplate.stage_for_baseplate,
+            "stage_for_datasaur": lambda els, **kw: datasaur.stage_for_datasaur(
+                els, kw.get('entities', [])
+            ),
+            "stage_for_label_box": lambda els,
+            **kw: label_box.stage_for_label_box(els, **kw),
+            "stage_for_label_studio": lambda els,
+            **kw: label_studio.stage_for_label_studio(els, **kw),
+            "stage_for_weaviate": weaviate.stage_for_weaviate,
         }
 
         if stage_type not in staging_functions:
@@ -390,14 +448,15 @@ class UnstructuredModules:
 
         return staging_functions[stage_type](elements, **kwargs)
 
-    def chunk_elements(self, elements: List[Any], chunk_type: str,
-                       **kwargs) -> List[Any]:
+    def chunk_elements(
+        self, elements: List[Any], chunk_type: str, **kwargs
+    ) -> List[Any]:
         r"""Chunks elements by titles.
 
         Args:
             elements (List[Any]): List of Element objects to be chunked.
             chunk_type (str): Type chunk going to apply. Supported types:
-                            'chunk_by_title'.
+                'chunk_by_title'.
             **kwargs: Additional keyword arguments for chunking.
 
         Returns:
@@ -419,8 +478,13 @@ class UnstructuredModules:
         # Format chunks into a list of dictionaries (or your preferred format)
         return chunking_functions[chunk_type](elements, **kwargs)
 
-    def run_s3_ingest(self, s3_url: str, output_dir: str,
-                      num_processes: int = 2, anonymous: bool = True) -> None:
+    def run_s3_ingest(
+        self,
+        s3_url: str,
+        output_dir: str,
+        num_processes: int = 2,
+        anonymous: bool = True,
+    ) -> None:
         r"""Processes documents from an S3 bucket and stores structured
         outputs locally.
 
@@ -460,8 +524,13 @@ class UnstructuredModules:
         )
         runner.run(anonymous=anonymous)
 
-    def run_azure_ingest(self, azure_url: str, output_dir: str,
-                         account_name: str, num_processes: int = 2) -> None:
+    def run_azure_ingest(
+        self,
+        azure_url: str,
+        output_dir: str,
+        account_name: str,
+        num_processes: int = 2,
+    ) -> None:
         """
         Processes documents from an Azure storage container and stores
         structured outputs locally.
@@ -500,8 +569,13 @@ class UnstructuredModules:
         )
         runner.run(account_name=account_name)
 
-    def run_github_ingest(self, repo_url: str, git_branch: str,
-                          output_dir: str, num_processes: int = 2) -> None:
+    def run_github_ingest(
+        self,
+        repo_url: str,
+        git_branch: str,
+        output_dir: str,
+        num_processes: int = 2,
+    ) -> None:
         r"""Processes documents from a GitHub repository and stores
         structured outputs locally.
 
@@ -537,9 +611,15 @@ class UnstructuredModules:
         )
         runner.run(url=repo_url, git_branch=git_branch)
 
-    def run_slack_ingest(self, channels: List[str], token: str,
-                         start_date: str, end_date: str, output_dir: str,
-                         num_processes: int = 2) -> None:
+    def run_slack_ingest(
+        self,
+        channels: List[str],
+        token: str,
+        start_date: str,
+        end_date: str,
+        output_dir: str,
+        num_processes: int = 2,
+    ) -> None:
         r"""Processes documents from specified Slack channels and stores
         structured outputs locally.
 
@@ -575,11 +655,20 @@ class UnstructuredModules:
             read_config=ReadConfig(),
             partition_config=PartitionConfig(),
         )
-        runner.run(channels=channels, token=token, start_date=start_date,
-                   end_date=end_date)
+        runner.run(
+            channels=channels,
+            token=token,
+            start_date=start_date,
+            end_date=end_date,
+        )
 
-    def run_discord_ingest(self, channels: List[str], token: str,
-                           output_dir: str, num_processes: int = 2) -> None:
+    def run_discord_ingest(
+        self,
+        channels: List[str],
+        token: str,
+        output_dir: str,
+        num_processes: int = 2,
+    ) -> None:
         r"""Processes messages from specified Discord channels and stores
         structured outputs locally.
 

camel/memories/__init__.py

@@ -12,17 +12,27 @@
 # limitations under the License.
 # =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
 
-from .records import MemoryRecord, ContextRecord
-from .base import BaseMemory
-from .context_creators.base import BaseContextCreator
+from .agent_memories import (
+    ChatHistoryMemory,
+    LongtermAgentMemory,
+    VectorDBMemory,
+)
+from .base import AgentMemory, BaseContextCreator, MemoryBlock
+from .blocks.chat_history_block import ChatHistoryBlock
+from .blocks.vectordb_block import VectorDBBlock
 from .context_creators.score_based import ScoreBasedContextCreator
-from .chat_history_memory import ChatHistoryMemory
+from .records import ContextRecord, MemoryRecord
 
 __all__ = [
     'MemoryRecord',
     'ContextRecord',
-    'BaseMemory',
+    'MemoryBlock',
+    "AgentMemory",
+    'BaseContextCreator',
+    'ScoreBasedContextCreator',
     'ChatHistoryMemory',
-    "BaseContextCreator",
-    "ScoreBasedContextCreator",
+    'VectorDBMemory',
+    'ChatHistoryBlock',
+    'VectorDBBlock',
+    'LongtermAgentMemory',
 ]

camel/memories/agent_memories.py

@@ -0,0 +1,156 @@
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+
+from typing import List, Optional
+
+from camel.memories.base import AgentMemory, BaseContextCreator
+from camel.memories.blocks import ChatHistoryBlock, VectorDBBlock
+from camel.memories.records import (
+    ContextRecord,
+    MemoryRecord,
+)
+from camel.storages import BaseKeyValueStorage, BaseVectorStorage
+from camel.types import OpenAIBackendRole
+
+
+class ChatHistoryMemory(AgentMemory):
+    r"""An agent memory wrapper of :obj:`ChatHistoryBlock`.
+
+    Args:
+        context_creator (BaseContextCreator): A model context creator.
+        storage (BaseKeyValueStorage, optional): A storage backend for storing
+            chat history. If `None`, an :obj:`InMemoryKeyValueStorage`
+            will be used. (default: :obj:`None`)
+        window_size (int, optional): The number of recent chat messages to
+            retrieve. If not provided, the entire chat history will be
+            retrieved.  (default: :obj:`None`)
+    """
+
+    def __init__(
+        self,
+        context_creator: BaseContextCreator,
+        storage: Optional[BaseKeyValueStorage] = None,
+        window_size: Optional[int] = None,
+    ) -> None:
+        if window_size is not None and not isinstance(window_size, int):
+            raise TypeError("`window_size` must be an integer or None.")
+        if window_size is not None and window_size < 0:
+            raise ValueError("`window_size` must be non-negative.")
+        self._context_creator = context_creator
+        self._window_size = window_size
+        self._chat_history_block = ChatHistoryBlock(storage=storage)
+
+    def retrieve(self) -> List[ContextRecord]:
+        return self._chat_history_block.retrieve(self._window_size)
+
+    def write_records(self, records: List[MemoryRecord]) -> None:
+        self._chat_history_block.write_records(records)
+
+    def get_context_creator(self) -> BaseContextCreator:
+        return self._context_creator
+
+    def clear(self) -> None:
+        self._chat_history_block.clear()
+
+
+class VectorDBMemory(AgentMemory):
+    r"""An agent memory wrapper of :obj:`VectorDBBlock`. This memory queries
+    messages stored in the vector database. Notice that the most recent
+    messages will not be added to the context.
+
+    Args:
+        context_creator (BaseContextCreator): A model context creator.
+        storage (BaseVectorStorage, optional): A vector storage storage. If
+            `None`, an :obj:`QdrantStorage` will be used.
+            (default: :obj:`None`)
+        retrieve_limit (int, optional): The maximum number of messages
+            to be added into the context.  (default: :obj:`3`)
+    """
+
+    def __init__(
+        self,
+        context_creator: BaseContextCreator,
+        storage: Optional[BaseVectorStorage] = None,
+        retrieve_limit: int = 3,
+    ) -> None:
+        self._context_creator = context_creator
+        self._retrieve_limit = retrieve_limit
+        self._vectordb_block = VectorDBBlock(storage=storage)
+
+        self._current_topic: str = ""
+
+    def retrieve(self) -> List[ContextRecord]:
+        return self._vectordb_block.retrieve(
+            self._current_topic,
+            limit=self._retrieve_limit,
+        )
+
+    def write_records(self, records: List[MemoryRecord]) -> None:
+        # Assume the last user input is the current topic.
+        for record in records:
+            if record.role_at_backend == OpenAIBackendRole.USER:
+                self._current_topic = record.message.content
+        self._vectordb_block.write_records(records)
+
+    def get_context_creator(self) -> BaseContextCreator:
+        return self._context_creator
+
+
+class LongtermAgentMemory(AgentMemory):
+    r"""An implementation of the :obj:`AgentMemory` abstract base class for
+    augumenting ChatHistoryMemory with VectorDBMemory.
+    """
+
+    def __init__(
+        self,
+        context_creator: BaseContextCreator,
+        chat_history_block: Optional[ChatHistoryBlock] = None,
+        vector_db_block: Optional[VectorDBBlock] = None,
+        retrieve_limit: int = 3,
+    ) -> None:
+        self.chat_history_block = chat_history_block or ChatHistoryBlock()
+        self.vector_db_block = vector_db_block or VectorDBBlock()
+        self.retrieve_limit = retrieve_limit
+        self._context_creator = context_creator
+        self._current_topic: str = ""
+
+    def get_context_creator(self) -> BaseContextCreator:
+        return self._context_creator
+
+    def retrieve(self) -> List[ContextRecord]:
+        chat_history = self.chat_history_block.retrieve()
+        vector_db_retrieve = self.vector_db_block.retrieve(
+            self._current_topic, self.retrieve_limit
+        )
+        return chat_history[:1] + vector_db_retrieve + chat_history[1:]
+
+    def write_records(self, records: List[MemoryRecord]) -> None:
+        r"""Converts the provided chat messages into vector representations and
+        writes them to the vector database.
+
+        Args:
+            records (List[MemoryRecord]): Messages to be added to the vector
+                database.
+        """
+        self.vector_db_block.write_records(records)
+        self.chat_history_block.write_records(records)
+
+        for record in records:
+            if record.role_at_backend == OpenAIBackendRole.USER:
+                self._current_topic = record.message.content
+
+    def clear(self) -> None:
+        r"""Removes all records from the memory."""
+        self.chat_history_block.clear()
+        self.vector_db_block.clear()