camel-ai 0.2.18__py3-none-any.whl → 0.2.20__py3-none-any.whl

camel/datagen/source2synth/__init__.py

@@ -0,0 +1,31 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+from .data_processor import (
+    DataCurator,
+    ExampleConstructor,
+    UserDataProcessor,
+)
+from .models import MultiHopQA, ReasoningStep
+from .user_data_processor_config import (
+    ProcessorConfig,
+)
+
+__all__ = [
+    "DataCurator",
+    "ExampleConstructor",
+    "ProcessorConfig",
+    "UserDataProcessor",
+    "ReasoningStep",
+    "MultiHopQA",
+]

camel/{synthetic_datagen → datagen}/source2synth/data_processor.py

@@ -15,33 +15,61 @@
 import random
 from typing import Any, Dict, List, Optional, Sequence
 
-import numpy as np
 from tqdm import tqdm
 
 from camel.agents.multi_hop_generator_agent import MultiHopGeneratorAgent
-from camel.logger import get_logger
-from camel.synthetic_datagen.source2synth.user_data_processor_config import (
+from camel.datagen.source2synth.user_data_processor_config import (
     ProcessorConfig,
 )
+from camel.logger import get_logger
 
 logger = get_logger(__name__)
 
 
 class UserDataProcessor:
-    r"""User Data Processor."""
+    r"""A processor for generating multi-hop question-answer pairs from user
+    data.
+
+    This class handles the processing of text data to generate multi-hop
+    question-answer pairs using either an AI model or rule-based approaches.
+    It manages the entire pipeline from text preprocessing to dataset curation.
+
+    Attributes:
+        config (ProcessorConfig): Configuration for data processing parameters.
+        rng (random.Random): Random number generator for reproducibility.
+        multi_hop_agent (Optional[MultiHopGeneratorAgent]): Agent for
+            generating QA pairs.
+    """
 
     def __init__(self, config: Optional[ProcessorConfig] = None):
+        r"""Initialize the UserDataProcessor.
+
+        Args:
+            config (Optional[ProcessorConfig], optional): Configuration for
+                data processing. (default: :obj:`None`)
+        """
         self.config = config or ProcessorConfig()
-        random.seed(self.config.seed)
-        np.random.seed(self.config.seed)
+        self.rng = random.Random(self.config.seed)
         self.multi_hop_agent = (
-            MultiHopGeneratorAgent() if self.config.use_ai_model else None
+            self.config.hop_generating_agent
+            if self.config.use_ai_model
+            else None
         )
 
     def process_text(
         self, text: str, source: str = "user_input"
     ) -> List[Dict[str, Any]]:
-        r"""Process a single text."""
+        r"""Process a single text to generate multi-hop QA pairs.
+
+        Args:
+            text (str): The input text to process.
+            source (str, optional): Source identifier for the text.
+                (default: :obj:`"user_input"`)
+
+        Returns:
+            List[Dict[str, Any]]: List of processed examples with QA pairs and
+                metadata.
+        """
         # Convert text to standard format
         raw_data = [
             {
@@ -55,7 +83,7 @@ class UserDataProcessor:
         examples = constructor.construct_examples(raw_data)
 
         # Manage data
-        curator = DataCurator(self.config)
+        curator = DataCurator(self.config, self.rng)
         final_dataset = curator.curate_dataset(examples)
 
         return final_dataset
@@ -63,7 +91,20 @@ class UserDataProcessor:
     def process_batch(
         self, texts: List[str], sources: Optional[List[str]] = None
     ) -> List[Dict[str, Any]]:
-        r"""Process multiple texts in batch."""
+        r"""Process multiple texts in batch to generate multi-hop QA pairs.
+
+        Args:
+            texts (List[str]): List of input texts to process.
+            sources (Optional[List[str]], optional): List of source
+                identifiers. (default: :obj:`None`)
+
+        Returns:
+            List[Dict[str, Any]]: List of processed examples with QA pairs and
+                metadata.
+
+        Raises:
+            ValueError: If length of sources doesn't match length of texts.
+        """
         if sources is None:
             sources = ["user_input"] * len(texts)
         elif len(sources) != len(texts):
@@ -82,27 +123,52 @@ class UserDataProcessor:
         examples = constructor.construct_examples(raw_data)
 
         # Manage data
-        curator = DataCurator(self.config)
+        curator = DataCurator(self.config, self.rng)
         final_dataset = curator.curate_dataset(examples)
 
         return final_dataset
 
 
 class ExampleConstructor:
-    r"""Example Constructor."""
+    r"""Constructs training examples from raw text data.
+
+    This class handles the construction of training examples by preprocessing
+    text, extracting information pairs, and generating question-answer pairs.
+
+    Attributes:
+        config (ProcessorConfig): Configuration for example construction.
+        multi_hop_agent (Optional[MultiHopGeneratorAgent]): Agent for QA
+            generation.
+    """
 
     def __init__(
         self,
         config: ProcessorConfig,
         multi_hop_agent: Optional[MultiHopGeneratorAgent] = None,
     ):
+        r"""Initialize the ExampleConstructor.
+
+        Args:
+            config (ProcessorConfig): Configuration for example construction.
+            multi_hop_agent (Optional[MultiHopGeneratorAgent], optional):
+                Agent for generating multi-hop QA pairs. (default: :obj:`None`)
+        """
         self.config = config
         self.multi_hop_agent = multi_hop_agent
 
     def construct_examples(
         self, raw_data: List[Dict[str, Any]]
     ) -> List[Dict[str, Any]]:
-        r"""Construct training examples."""
+        r"""Construct training examples from raw data.
+
+        Args:
+            raw_data (List[Dict[str, Any]]): List of raw data dictionaries
+                containing text and metadata.
+
+        Returns:
+            List[Dict[str, Any]]: List of constructed examples with QA pairs
+                and metadata.
+        """
         logger.info("Starting to construct training examples...")
         examples = []
 
@@ -135,7 +201,15 @@ class ExampleConstructor:
         return examples
 
     def _preprocess_text(self, text: str) -> str:
-        r"""Text preprocessing."""
+        r"""Preprocess input text for example construction.
+
+        Args:
+            text (str): Input text to preprocess.
+
+        Returns:
+            str: Preprocessed text, or empty string if text fails quality
+                checks.
+        """
         if not isinstance(text, str):
             return ''
 
@@ -156,7 +230,14 @@ class ExampleConstructor:
         return text
 
     def _check_text_quality(self, text: str) -> bool:
-        r"""Check text quality."""
+        r"""Check the quality of input text.
+
+        Args:
+            text (str): Text to check quality for.
+
+        Returns:
+            bool: True if text passes quality checks, False otherwise.
+        """
         # 1. Basic quality check
         if text.count('.') < 2:  # Must have at least 2 sentences
             return False
@@ -171,7 +252,15 @@ class ExampleConstructor:
         return True
 
     def _extract_info_pairs(self, text: str) -> List[Dict[str, Sequence[str]]]:
-        r"""Extract information pairs and relationships."""
+        r"""Extract information pairs and relationships from text.
+
+        Args:
+            text (str): Input text to extract information from.
+
+        Returns:
+            List[Dict[str, Sequence[str]]]: List of dictionaries containing
+                premise, intermediate, conclusion, and related contexts.
+        """
         # Split into sentences
         sentences = [s.strip() for s in text.split('.') if s.strip()]
         info_pairs = []
@@ -200,7 +289,15 @@ class ExampleConstructor:
     def _generate_qa_pairs(
         self, info_pairs: List[Dict[str, Sequence[str]]]
     ) -> List[Dict[str, str]]:
-        r"""Generate multi-hop question-answer pairs."""
+        r"""Generate multi-hop question-answer pairs from information pairs.
+
+        Args:
+            info_pairs (List[Dict[str, Sequence[str]]]): List of information
+                pairs extracted from text.
+
+        Returns:
+            List[Dict[str, str]]: List of generated QA pairs.
+        """
         qa_pairs = []
 
         for pair in info_pairs:
@@ -219,7 +316,15 @@ class ExampleConstructor:
         return qa_pairs
 
     def _calculate_complexity(self, qa_pairs: List[Dict[str, Any]]) -> float:
-        r"""Calculate complexity of QA pairs."""
+        r"""Calculate the complexity score for a set of QA pairs.
+
+        Args:
+            qa_pairs (List[Dict[str, Any]]): List of QA pairs to calculate
+                complexity for.
+
+        Returns:
+            float: Complexity score between 0.0 and 1.0.
+        """
         if not qa_pairs:
             return 0.0
 
@@ -233,10 +338,10 @@ class ExampleConstructor:
             supporting_facts_count = len(qa.get('supporting_facts', []))
 
             # 3. Question length
-            question_length = len(qa['question'].split())
+            question_length = len(qa.get('question', '').split())
 
             # 4. Answer length
-            answer_length = len(qa['answer'].split())
+            answer_length = len(qa.get('answer', '').split())
 
             # Calculate complexity of a single QA pair
             qa_complexity = (
@@ -256,15 +361,37 @@ class ExampleConstructor:
 
 
 class DataCurator:
-    r"""Data Manager."""
+    r"""Manages and curates datasets of multi-hop question-answer pairs.
+
+    This class handles dataset management tasks including quality filtering,
+    complexity filtering, deduplication, and dataset sampling.
 
-    def __init__(self, config: ProcessorConfig):
+    Attributes:
+        config (ProcessorConfig): Configuration for data curation parameters.
+        rng (random.Random): Random number generator for reproducible sampling.
+    """
+
+    def __init__(self, config: ProcessorConfig, rng: random.Random):
+        r"""Initialize the DataCurator.
+
+        Args:
+            config (ProcessorConfig): Configuration for data curation.
+            rng (random.Random): Random number generator for reproducibility.
+        """
         self.config = config
+        self.rng = rng
 
     def curate_dataset(
         self, examples: List[Dict[str, Any]]
     ) -> List[Dict[str, Any]]:
-        r"""Dataset management."""
+        r"""Manage and curate a dataset through multiple filtering stages.
+
+        Args:
+            examples (List[Dict[str, Any]]): List of examples to curate.
+
+        Returns:
+            List[Dict[str, Any]]: Curated dataset meeting quality criteria.
+        """
         logger.info("Starting dataset management...")
 
         # 1. Quality filtering
@@ -296,7 +423,14 @@ class DataCurator:
     def _quality_filter(
         self, examples: List[Dict[str, Any]]
     ) -> List[Dict[str, Any]]:
-        r"""Quality filtering."""
+        r"""Filter examples based on quality criteria.
+
+        Args:
+            examples (List[Dict[str, Any]]): List of examples to filter.
+
+        Returns:
+            List[Dict[str, Any]]: Examples that pass quality checks.
+        """
         filtered = []
 
         for example in examples:
@@ -314,7 +448,14 @@ class DataCurator:
         return filtered
 
     def _check_qa_quality(self, qa_pairs: List[Dict[str, str]]) -> bool:
-        r"""Check quality of QA pairs."""
+        r"""Check the quality of question-answer pairs.
+
+        Args:
+            qa_pairs (List[Dict[str, str]]): List of QA pairs to check.
+
+        Returns:
+            bool: True if QA pairs meet quality criteria, False otherwise.
+        """
         if not qa_pairs:
             return False
 
@@ -335,7 +476,17 @@ class DataCurator:
     def _complexity_filter(
         self, examples: List[Dict[str, Any]]
     ) -> List[Dict[str, Any]]:
-        r"""Complexity filtering."""
+        """
+        Filter examples based on complexity threshold.
+
+        Removes examples with complexity scores below the configured threshold.
+
+        Args:
+            examples (List[Dict[str, Any]]): List of examples to filter.
+
+        Returns:
+            List[Dict[str, Any]]: Examples meeting complexity threshold.
+        """
         return [
             example
             for example in examples
@@ -346,7 +497,14 @@ class DataCurator:
     def _remove_duplicates(
         self, examples: List[Dict[str, Any]]
     ) -> List[Dict[str, Any]]:
-        r"""Remove duplicates."""
+        r"""Remove duplicate examples from the dataset.
+
+        Args:
+            examples (List[Dict[str, Any]]): List of examples to deduplicate.
+
+        Returns:
+            List[Dict[str, Any]]: Deduplicated examples.
+        """
         seen = set()
         unique_examples = []
 
@@ -366,8 +524,15 @@ class DataCurator:
     def _sample_dataset(
         self, examples: List[Dict[str, Any]]
     ) -> List[Dict[str, Any]]:
-        r"""Sample to target dataset size."""
+        r"""Sample examples to match target dataset size.
+
+        Args:
+            examples (List[Dict[str, Any]]): List of examples to sample from.
+
+        Returns:
+            List[Dict[str, Any]]: Sampled dataset of target size or smaller.
+        """
         if len(examples) <= self.config.dataset_size:
             return examples
 
-        return random.sample(examples, self.config.dataset_size)
+        return self.rng.sample(examples, self.config.dataset_size)

camel/{synthetic_datagen → datagen}/source2synth/models.py

@@ -17,12 +17,30 @@ from pydantic import BaseModel, Field
 
 
 class ReasoningStep(BaseModel):
+    r"""A single step in a multi-hop reasoning process.
+
+    Attributes:
+        step (str): The textual description of the reasoning step.
+    """
+
     step: str = Field(
         ..., description="A single step in the reasoning process."
     )
 
 
 class MultiHopQA(BaseModel):
+    r"""A multi-hop question-answer pair with reasoning steps and supporting
+    facts.
+
+    Attributes:
+        question (str): The question requiring multi-hop reasoning.
+        reasoning_steps (List[ReasoningStep]): List of reasoning steps to
+            answer.
+        answer (str): The final answer to the question.
+        supporting_facts (List[str]): List of facts supporting the reasoning.
+        type (str): The type of question-answer pair.
+    """
+
     question: str = Field(
         ..., description="The question that requires multi-hop reasoning."
     )
@@ -57,6 +75,13 @@ class MultiHopQA(BaseModel):
 
 
 class ContextPrompt(BaseModel):
+    r"""A context prompt for generating multi-hop question-answer pairs.
+
+    Attributes:
+        main_context (str): The primary context for generating QA pairs.
+        related_contexts (Optional[List[str]]): Additional related contexts.
+    """
+
     main_context: str = Field(
         ...,
         description="The main context for generating"

camel/{synthetic_datagen → datagen}/source2synth/user_data_processor_config.py

@@ -23,7 +23,15 @@ class ProcessorConfig(BaseModel):
     r"""Data processing configuration class"""
 
     def __repr__(self):
-        return "MultiHopGeneratorAgent()"
+        return (
+            f"ProcessorConfig("
+            f"seed={self.seed}, min_length={self.min_length}, "
+            f"max_length={self.max_length}, "
+            f"complexity_threshold={self.complexity_threshold}, "
+            f"dataset_size={self.dataset_size}, "
+            f"use_ai_model={self.use_ai_model}"
+            f")"
+        )
 
     model_config = ConfigDict(
         validate_assignment=True,
@@ -45,13 +53,6 @@ class ProcessorConfig(BaseModel):
         default=512, description="Maximum text length", gt=0
     )
 
-    quality_threshold: float = Field(
-        default=0.7,
-        description="Quality threshold for processing",
-        ge=0.0,
-        le=1.0,
-    )
-
     complexity_threshold: float = Field(
         default=0.5,
         description="Complexity threshold for processing",

camel/datahubs/huggingface.py

@@ -32,19 +32,19 @@ class HuggingFaceDatasetManager(BaseDatasetManager):
 
     Args:
         token (str): The Hugging Face API token. If not provided, the token
-            will be read from the environment variable `HUGGING_FACE_TOKEN`.
+            will be read from the environment variable `HF_TOKEN`.
     """
 
     @api_keys_required(
         [
-            ("token", "HUGGING_FACE_TOKEN"),
+            ("token", "HF_TOKEN"),
         ]
     )
     @dependencies_required('huggingface_hub')
     def __init__(self, token: Optional[str] = None):
         from huggingface_hub import HfApi
 
-        self._api_key = token or os.getenv("HUGGING_FACE_TOKEN")
+        self._api_key = token or os.getenv("HF_TOKEN")
         self.api = HfApi(token=self._api_key)
 
     def create_dataset_card(

camel/embeddings/__init__.py

@@ -12,6 +12,7 @@
 # limitations under the License.
 # ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
 from .base import BaseEmbedding
+from .jina_embedding import JinaEmbedding
 from .mistral_embedding import MistralEmbedding
 from .openai_compatible_embedding import OpenAICompatibleEmbedding
 from .openai_embedding import OpenAIEmbedding
@@ -25,4 +26,5 @@ __all__ = [
     "VisionLanguageEmbedding",
     "MistralEmbedding",
     "OpenAICompatibleEmbedding",
+    "JinaEmbedding",
 ]

camel/embeddings/jina_embedding.py

@@ -0,0 +1,161 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+
+import base64
+import io
+import os
+from typing import Any, Optional, Union
+
+import requests
+from PIL import Image
+
+from camel.embeddings import BaseEmbedding
+from camel.types.enums import EmbeddingModelType
+from camel.utils import api_keys_required
+
+
+class JinaEmbedding(BaseEmbedding[Union[str, Image.Image]]):
+    r"""Provides text and image embedding functionalities using Jina AI's API.
+
+    Args:
+        model_type (EmbeddingModelType, optional): The model to use for
+            embeddings. (default: :obj:`JINA_EMBEDDINGS_V3`)
+        api_key (Optional[str], optional): The API key for authenticating with
+            Jina AI. (default: :obj:`None`)
+        dimensions (Optional[int], optional): The dimension of the output
+            embeddings. (default: :obj:`None`)
+        embedding_type (Optional[str], optional): The type of embedding format
+            to generate. Options: 'int8' (binary encoding with higher storage
+            and transfer efficiency), 'uint8' (unsigned binary encoding with
+            higher storage and transfer efficiency), 'base64' (base64 string
+            encoding with higher transfer efficiency). (default: :obj:`None`)
+        task (Optional[str], optional): The type of task for text embeddings.
+            Options: retrieval.query, retrieval.passage, text-matching,
+            classification, separation. (default: :obj:`None`)
+        late_chunking (bool, optional): If true, concatenates all sentences in
+            input and treats as a single input. (default: :obj:`False`)
+        normalized (bool, optional): If true, embeddings are normalized to unit
+            L2 norm. (default: :obj:`False`)
+    """
+
+    @api_keys_required([("api_key", 'JINA_API_KEY')])
+    def __init__(
+        self,
+        model_type: EmbeddingModelType = EmbeddingModelType.JINA_EMBEDDINGS_V3,
+        api_key: Optional[str] = None,
+        dimensions: Optional[int] = None,
+        embedding_type: Optional[str] = None,
+        task: Optional[str] = None,
+        late_chunking: bool = False,
+        normalized: bool = False,
+    ) -> None:
+        if not model_type.is_jina:
+            raise ValueError(
+                f"Model type {model_type} is not a Jina model. "
+                "Please use a valid Jina model type."
+            )
+        self.model_type = model_type
+        if dimensions is None:
+            self.output_dim = model_type.output_dim
+        else:
+            self.output_dim = dimensions
+        self._api_key = api_key or os.environ.get("JINA_API_KEY")
+
+        self.embedding_type = embedding_type
+        self.task = task
+        self.late_chunking = late_chunking
+        self.normalized = normalized
+        self.url = 'https://api.jina.ai/v1/embeddings'
+        self.headers = {
+            'Content-Type': 'application/json',
+            'Accept': 'application/json',
+            'Authorization': f'Bearer {self._api_key}',
+        }
+
+    def embed_list(
+        self,
+        objs: list[Union[str, Image.Image]],
+        **kwargs: Any,
+    ) -> list[list[float]]:
+        r"""Generates embeddings for the given texts or images.
+
+        Args:
+            objs (list[Union[str, Image.Image]]): The texts or images for which
+                to generate the embeddings.
+            **kwargs (Any): Extra kwargs passed to the embedding API. Not used
+                in this implementation.
+
+        Returns:
+            list[list[float]]: A list that represents the generated embedding
+                as a list of floating-point numbers.
+
+        Raises:
+            ValueError: If the input type is not supported.
+            RuntimeError: If the API request fails.
+        """
+        input_data = []
+        for obj in objs:
+            if isinstance(obj, str):
+                if self.model_type == EmbeddingModelType.JINA_CLIP_V2:
+                    input_data.append({"text": obj})
+                else:
+                    input_data.append(obj)  # type: ignore[arg-type]
+            elif isinstance(obj, Image.Image):
+                if self.model_type != EmbeddingModelType.JINA_CLIP_V2:
+                    raise ValueError(
+                        f"Model {self.model_type} does not support "
+                        "image input. Use JINA_CLIP_V2 for image embeddings."
+                    )
+                # Convert PIL Image to base64 string
+                buffered = io.BytesIO()
+                obj.save(buffered, format="PNG")
+                img_str = base64.b64encode(buffered.getvalue()).decode()
+                input_data.append({"image": img_str})
+            else:
+                raise ValueError(
+                    f"Input type {type(obj)} is not supported. "
+                    "Must be either str or PIL.Image."
+                )
+
+        data = {
+            "model": self.model_type.value,
+            "input": input_data,
+            "embedding_type": "float",
+        }
+
+        if self.embedding_type is not None:
+            data["embedding_type"] = self.embedding_type
+        if self.task is not None:
+            data["task"] = self.task
+        if self.late_chunking:
+            data["late_chunking"] = self.late_chunking  # type: ignore[assignment]
+        if self.normalized:
+            data["normalized"] = self.normalized  # type: ignore[assignment]
+        try:
+            response = requests.post(
+                self.url, headers=self.headers, json=data, timeout=180
+            )
+            response.raise_for_status()
+            result = response.json()
+            return [data["embedding"] for data in result["data"]]
+        except requests.exceptions.RequestException as e:
+            raise RuntimeError(f"Failed to get embeddings from Jina AI: {e}")
+
+    def get_output_dim(self) -> int:
+        r"""Returns the output dimension of the embeddings.
+
+        Returns:
+            int: The dimensionality of the embedding for the current model.
+        """
+        return self.output_dim

camel/messages/func_message.py

@@ -154,7 +154,7 @@ class FunctionCallingMessage(BaseMessage):
                 " due to missing function name."
             )
 
-        result_content = json.dumps(self.result)
+        result_content = str(self.result)
 
         return {
             "role": "tool",