camel-ai 0.2.16__py3-none-any.whl → 0.2.17__py3-none-any.whl

camel/synthetic_datagen/source2synth/data_processor.py

@@ -0,0 +1,373 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+
+import random
+from typing import Any, Dict, List, Optional, Sequence
+
+import numpy as np
+from tqdm import tqdm
+
+from camel.agents.multi_hop_generator_agent import MultiHopGeneratorAgent
+from camel.logger import get_logger
+from camel.synthetic_datagen.source2synth.user_data_processor_config import (
+    ProcessorConfig,
+)
+
+logger = get_logger(__name__)
+
+
+class UserDataProcessor:
+    r"""User Data Processor."""
+
+    def __init__(self, config: Optional[ProcessorConfig] = None):
+        self.config = config or ProcessorConfig()
+        random.seed(self.config.seed)
+        np.random.seed(self.config.seed)
+        self.multi_hop_agent = (
+            MultiHopGeneratorAgent() if self.config.use_ai_model else None
+        )
+
+    def process_text(
+        self, text: str, source: str = "user_input"
+    ) -> List[Dict[str, Any]]:
+        r"""Process a single text."""
+        # Convert text to standard format
+        raw_data = [
+            {
+                'text': text,
+                'source': source,
+            }
+        ]
+
+        # Construct examples
+        constructor = ExampleConstructor(self.config, self.multi_hop_agent)
+        examples = constructor.construct_examples(raw_data)
+
+        # Manage data
+        curator = DataCurator(self.config)
+        final_dataset = curator.curate_dataset(examples)
+
+        return final_dataset
+
+    def process_batch(
+        self, texts: List[str], sources: Optional[List[str]] = None
+    ) -> List[Dict[str, Any]]:
+        r"""Process multiple texts in batch."""
+        if sources is None:
+            sources = ["user_input"] * len(texts)
+        elif len(sources) != len(texts):
+            raise ValueError("Length of sources must match length of texts")
+
+        raw_data = [
+            {
+                'text': text,
+                'source': source,
+            }
+            for text, source in zip(texts, sources)
+        ]
+
+        # Construct examples
+        constructor = ExampleConstructor(self.config, self.multi_hop_agent)
+        examples = constructor.construct_examples(raw_data)
+
+        # Manage data
+        curator = DataCurator(self.config)
+        final_dataset = curator.curate_dataset(examples)
+
+        return final_dataset
+
+
+class ExampleConstructor:
+    r"""Example Constructor."""
+
+    def __init__(
+        self,
+        config: ProcessorConfig,
+        multi_hop_agent: Optional[MultiHopGeneratorAgent] = None,
+    ):
+        self.config = config
+        self.multi_hop_agent = multi_hop_agent
+
+    def construct_examples(
+        self, raw_data: List[Dict[str, Any]]
+    ) -> List[Dict[str, Any]]:
+        r"""Construct training examples."""
+        logger.info("Starting to construct training examples...")
+        examples = []
+
+        for data in tqdm(raw_data, desc="Constructing examples"):
+            # 1. Text preprocessing
+            processed_text = self._preprocess_text(data.get('text', ''))
+            if not processed_text:
+                continue
+
+            # 2. Generate key information pairs
+            info_pairs = self._extract_info_pairs(processed_text)
+
+            # 3. Construct question-answer pairs
+            qa_pairs = self._generate_qa_pairs(info_pairs)
+
+            # 4. Add metadata
+            example = {
+                'text': processed_text,
+                'qa_pairs': qa_pairs,
+                'metadata': {
+                    'source': data.get('source', 'unknown'),
+                    'timestamp': data.get('timestamp', ''),
+                    'complexity': self._calculate_complexity(qa_pairs),
+                },
+            }
+
+            examples.append(example)
+
+        logger.info(f"Successfully constructed {len(examples)} examples")
+        return examples
+
+    def _preprocess_text(self, text: str) -> str:
+        r"""Text preprocessing."""
+        if not isinstance(text, str):
+            return ''
+
+        # 1. Basic cleaning
+        text = text.strip()
+
+        # 2. Length check
+        if (
+            len(text) < self.config.min_length
+            or len(text) > self.config.max_length
+        ):
+            return ''
+
+        # 3. Quality check
+        if not self._check_text_quality(text):
+            return ''
+
+        return text
+
+    def _check_text_quality(self, text: str) -> bool:
+        r"""Check text quality."""
+        # 1. Basic quality check
+        if text.count('.') < 2:  # Must have at least 2 sentences
+            return False
+
+        # 2. Special character ratio check
+        special_char_ratio = len(
+            [c for c in text if not c.isalnum() and not c.isspace()]
+        ) / len(text)
+        if special_char_ratio > 0.3:  # No more than 30% special characters
+            return False
+
+        return True
+
+    def _extract_info_pairs(self, text: str) -> List[Dict[str, Sequence[str]]]:
+        r"""Extract information pairs and relationships."""
+        # Split into sentences
+        sentences = [s.strip() for s in text.split('.') if s.strip()]
+        info_pairs = []
+
+        # Extract combinations of multiple related sentences
+        for i in range(len(sentences) - 2):
+            if len(sentences[i]) > 10 and len(sentences[i + 1]) > 10:
+                info_pairs.append(
+                    {
+                        'premise': sentences[i],
+                        'intermediate': sentences[i + 1],
+                        'conclusion': sentences[i + 2]
+                        if i + 2 < len(sentences)
+                        else '',
+                        'related_contexts': [
+                            s
+                            for j, s in enumerate(sentences)
+                            if j != i and j != i + 1 and len(s) > 10
+                        ][:2],
+                        # Limit to 2 additional related contexts
+                    }
+                )
+
+        return info_pairs
+
+    def _generate_qa_pairs(
+        self, info_pairs: List[Dict[str, Sequence[str]]]
+    ) -> List[Dict[str, str]]:
+        r"""Generate multi-hop question-answer pairs."""
+        qa_pairs = []
+
+        for pair in info_pairs:
+            # 1. Generate multi-hop question-answer pair using AI
+            if self.multi_hop_agent:
+                # Construct full context
+                context = (
+                    f"{pair['premise']}. {pair['intermediate']}."
+                    f" {pair['conclusion']}"
+                )
+                response = self.multi_hop_agent.generate_multi_hop_qa(context)
+                if response:
+                    qa_pairs.append(response.value.dict())
+                    continue
+
+        return qa_pairs
+
+    def _calculate_complexity(self, qa_pairs: List[Dict[str, Any]]) -> float:
+        r"""Calculate complexity of QA pairs."""
+        if not qa_pairs:
+            return 0.0
+
+        # Calculate complexity based on multiple factors
+        complexities = []
+        for qa in qa_pairs:
+            # 1. Number of reasoning steps
+            reasoning_steps_count = len(qa.get('reasoning_steps', []))
+
+            # 2. Number of supporting facts
+            supporting_facts_count = len(qa.get('supporting_facts', []))
+
+            # 3. Question length
+            question_length = len(qa['question'].split())
+
+            # 4. Answer length
+            answer_length = len(qa['answer'].split())
+
+            # Calculate complexity of a single QA pair
+            qa_complexity = (
+                min(reasoning_steps_count / 3, 1.0)
+                * 0.4  # Weight for reasoning steps
+                + min(supporting_facts_count / 3, 1.0)
+                * 0.3  # Weight for supporting facts
+                + min(question_length / 20, 1.0)
+                * 0.15  # Weight for question length
+                + min(answer_length / 50, 1.0) * 0.15
+                # Weight for answer length
+            )
+
+            complexities.append(qa_complexity)
+
+        return sum(complexities) / len(complexities)
+
+
+class DataCurator:
+    r"""Data Manager."""
+
+    def __init__(self, config: ProcessorConfig):
+        self.config = config
+
+    def curate_dataset(
+        self, examples: List[Dict[str, Any]]
+    ) -> List[Dict[str, Any]]:
+        r"""Dataset management."""
+        logger.info("Starting dataset management...")
+
+        # 1. Quality filtering
+        quality_filtered = self._quality_filter(examples)
+        logger.info(
+            f"Remaining examples after quality filtering:"
+            f" {len(quality_filtered)}"
+        )
+
+        # 2. Complexity filtering
+        complexity_filtered = self._complexity_filter(quality_filtered)
+        logger.info(
+            f"Remaining examples after complexity filtering:"
+            f" {len(complexity_filtered)}"
+        )
+
+        # 3. Deduplication
+        deduplicated = self._remove_duplicates(complexity_filtered)
+        logger.info(
+            f"Remaining examples after deduplication: {len(deduplicated)}"
+        )
+
+        # 4. Sample to target size
+        final_dataset = self._sample_dataset(deduplicated)
+        logger.info(f"Final dataset size: {len(final_dataset)}")
+
+        return final_dataset
+
+    def _quality_filter(
+        self, examples: List[Dict[str, Any]]
+    ) -> List[Dict[str, Any]]:
+        r"""Quality filtering."""
+        filtered = []
+
+        for example in examples:
+            # 1. Check QA pair quality
+            qa_quality = self._check_qa_quality(example.get('qa_pairs', []))
+
+            # 2. Check text quality
+            text_quality = (
+                len(example.get('text', '').split()) >= 20
+            )  # At least 20 words
+
+            if qa_quality and text_quality:
+                filtered.append(example)
+
+        return filtered
+
+    def _check_qa_quality(self, qa_pairs: List[Dict[str, str]]) -> bool:
+        r"""Check quality of QA pairs."""
+        if not qa_pairs:
+            return False
+
+        for qa in qa_pairs:
+            # 1. Length check
+            if (
+                len(qa.get('question', '')) < 10
+                or len(qa.get('answer', '')) < 5
+            ):
+                return False
+
+            # 2. QA pair duplication check
+            if qa.get('question', '') == qa.get('answer', ''):
+                return False
+
+        return True
+
+    def _complexity_filter(
+        self, examples: List[Dict[str, Any]]
+    ) -> List[Dict[str, Any]]:
+        r"""Complexity filtering."""
+        return [
+            example
+            for example in examples
+            if example.get('metadata', {}).get('complexity', 0)
+            >= self.config.complexity_threshold
+        ]
+
+    def _remove_duplicates(
+        self, examples: List[Dict[str, Any]]
+    ) -> List[Dict[str, Any]]:
+        r"""Remove duplicates."""
+        seen = set()
+        unique_examples = []
+
+        for example in examples:
+            # Use text and QA pair combination as unique identifier
+            text = example.get('text', '')
+            qa_str = str(example.get('qa_pairs', []))
+
+            identifier = hash(text + qa_str)
+
+            if identifier not in seen:
+                seen.add(identifier)
+                unique_examples.append(example)
+
+        return unique_examples
+
+    def _sample_dataset(
+        self, examples: List[Dict[str, Any]]
+    ) -> List[Dict[str, Any]]:
+        r"""Sample to target dataset size."""
+        if len(examples) <= self.config.dataset_size:
+            return examples
+
+        return random.sample(examples, self.config.dataset_size)

camel/synthetic_datagen/source2synth/models.py

@@ -0,0 +1,68 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+from typing import Any, ClassVar, Dict, List, Optional
+
+from pydantic import BaseModel, Field
+
+
+class ReasoningStep(BaseModel):
+    step: str = Field(
+        ..., description="A single step in the reasoning process."
+    )
+
+
+class MultiHopQA(BaseModel):
+    question: str = Field(
+        ..., description="The question that requires multi-hop reasoning."
+    )
+    reasoning_steps: List[ReasoningStep] = Field(
+        ...,
+        description="The steps involved in reasoning to answer the question.",
+    )
+    answer: str = Field(
+        ..., description="The answer to the multi-hop question."
+    )
+    supporting_facts: List[str] = Field(
+        ..., description="Facts that support the reasoning and answer."
+    )
+    type: str = Field(description="The type of question-answer pair.")
+
+    class Config:
+        json_schema_extra: ClassVar[Dict[str, Any]] = {
+            "example": {
+                "question": "What is the capital of France?",
+                "reasoning_steps": [
+                    {"step": "Identify the country France."},
+                    {"step": "Find the capital city of France."},
+                ],
+                "answer": "Paris",
+                "supporting_facts": [
+                    "France is a country in Europe.",
+                    "Paris is the capital city of France.",
+                ],
+                "type": "multi_hop_qa",
+            }
+        }
+
+
+class ContextPrompt(BaseModel):
+    main_context: str = Field(
+        ...,
+        description="The main context for generating"
+        " the question-answer pair.",
+    )
+    related_contexts: Optional[List[str]] = Field(
+        default=None,
+        description="Additional contexts related to the main context.",
+    )

camel/synthetic_datagen/source2synth/user_data_processor_config.py

@@ -0,0 +1,73 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+
+import random
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from camel.agents.multi_hop_generator_agent import MultiHopGeneratorAgent
+
+
+class ProcessorConfig(BaseModel):
+    r"""Data processing configuration class"""
+
+    def __repr__(self):
+        return "MultiHopGeneratorAgent()"
+
+    model_config = ConfigDict(
+        validate_assignment=True,
+        frozen=False,
+        protected_namespaces=(),
+        arbitrary_types_allowed=True,
+    )
+
+    seed: int = Field(  # Generate a random seed for reproducibility
+        default_factory=lambda: random.randint(0, 1000),
+        description="Random seed for reproducibility",
+    )
+
+    min_length: int = Field(
+        default=50, description="Minimum text length", ge=0
+    )
+
+    max_length: int = Field(
+        default=512, description="Maximum text length", gt=0
+    )
+
+    quality_threshold: float = Field(
+        default=0.7,
+        description="Quality threshold for processing",
+        ge=0.0,
+        le=1.0,
+    )
+
+    complexity_threshold: float = Field(
+        default=0.5,
+        description="Complexity threshold for processing",
+        ge=0.0,
+        le=1.0,
+    )
+
+    dataset_size: int = Field(
+        default=1000, description="Target size of the dataset", gt=0
+    )
+
+    use_ai_model: bool = Field(
+        default=True, description="Whether to use AI model in processing"
+    )
+
+    hop_generating_agent: MultiHopGeneratorAgent = Field(
+        default_factory=lambda: MultiHopGeneratorAgent(),
+        description="Agent for generating multi-hop text",
+    )

camel/toolkits/google_scholar_toolkit.py

@@ -36,6 +36,7 @@ class GoogleScholarToolkit(BaseToolkit):
         self,
         author_identifier: str,
         is_author_name: bool = False,
+        use_free_proxies: bool = False,
         proxy_http: Optional[str] = None,
         proxy_https: Optional[str] = None,
     ) -> None:
@@ -46,6 +47,8 @@ class GoogleScholarToolkit(BaseToolkit):
                 of the author to search for.
             is_author_name (bool): Flag to indicate if the identifier is a
                 name. (default: :obj:`False`)
+            use_free_proxies (bool): Whether to use Free Proxies.
+                (default: :obj:`False`)
             proxy_http ( Optional[str]): Proxy http address pass to pg.
                 SingleProxy. (default: :obj:`None`)
             proxy_https ( Optional[str]): Proxy https address pass to pg.
@@ -53,6 +56,12 @@ class GoogleScholarToolkit(BaseToolkit):
         """
         from scholarly import ProxyGenerator, scholarly
 
+        # Set Free Proxies is needed
+        if use_free_proxies:
+            pg = ProxyGenerator()
+            pg.FreeProxies()
+            scholarly.use_proxy(pg)
+
         # Set Proxy is HTTP or HTTPS provided
         if proxy_http or proxy_https:
             pg = ProxyGenerator()

camel/types/__init__.py

@@ -33,10 +33,11 @@ from .openai_types import (
     ChatCompletion,
     ChatCompletionAssistantMessageParam,
     ChatCompletionChunk,
-    ChatCompletionFunctionMessageParam,
     ChatCompletionMessage,
     ChatCompletionMessageParam,
+    ChatCompletionMessageToolCall,
     ChatCompletionSystemMessageParam,
+    ChatCompletionToolMessageParam,
     ChatCompletionUserMessageParam,
     Choice,
     CompletionUsage,
@@ -62,7 +63,8 @@ __all__ = [
     'ChatCompletionSystemMessageParam',
     'ChatCompletionUserMessageParam',
     'ChatCompletionAssistantMessageParam',
-    'ChatCompletionFunctionMessageParam',
+    'ChatCompletionToolMessageParam',
+    'ChatCompletionMessageToolCall',
     'CompletionUsage',
     'OpenAIImageType',
     'OpenAIVisionDetailType',

camel/types/enums.py

@@ -142,6 +142,12 @@ class ModelType(UnifiedModelType, Enum):
     # DeepSeek models
     DEEPSEEK_CHAT = "deepseek-chat"
 
+    # InternLM models
+    INTERNLM3_LATEST = "internlm3-latest"
+    INTERNLM3_8B_INSTRUCT = "internlm3-8b-instruct"
+    INTERNLM2_5_LATEST = "internlm2.5-latest"
+    INTERNLM2_PRO_CHAT = "internlm2-pro-chat"
+
     def __str__(self):
         return self.value
 
@@ -161,7 +167,15 @@ class ModelType(UnifiedModelType, Enum):
     @property
     def support_native_tool_calling(self) -> bool:
         return any(
-            [self.is_openai, self.is_gemini, self.is_mistral, self.is_qwen]
+            [
+                self.is_openai,
+                self.is_gemini,
+                self.is_mistral,
+                self.is_qwen,
+                self.is_deepseek,
+                self.is_cohere,
+                self.is_internlm,
+            ]
         )
 
     @property
@@ -353,6 +367,15 @@ class ModelType(UnifiedModelType, Enum):
             ModelType.DEEPSEEK_CHAT,
         }
 
+    @property
+    def is_internlm(self) -> bool:
+        return self in {
+            ModelType.INTERNLM3_LATEST,
+            ModelType.INTERNLM3_8B_INSTRUCT,
+            ModelType.INTERNLM2_5_LATEST,
+            ModelType.INTERNLM2_PRO_CHAT,
+        }
+
     @property
     def token_limit(self) -> int:
         r"""Returns the maximum token limit for a given model.
@@ -411,6 +434,10 @@ class ModelType(UnifiedModelType, Enum):
             ModelType.NVIDIA_MISTRAL_LARGE,
             ModelType.NVIDIA_MIXTRAL_8X7B,
             ModelType.QWEN_QWQ_32B,
+            ModelType.INTERNLM3_8B_INSTRUCT,
+            ModelType.INTERNLM3_LATEST,
+            ModelType.INTERNLM2_5_LATEST,
+            ModelType.INTERNLM2_PRO_CHAT,
         }:
             return 32_768
         elif self in {
@@ -634,6 +661,7 @@ class ModelPlatformType(Enum):
     NVIDIA = "nvidia"
     DEEPSEEK = "deepseek"
     SGLANG = "sglang"
+    INTERNLM = "internlm"
 
     @property
     def is_openai(self) -> bool:
@@ -736,6 +764,11 @@ class ModelPlatformType(Enum):
         r"""Returns whether this platform is DeepSeek."""
         return self is ModelPlatformType.DEEPSEEK
 
+    @property
+    def is_internlm(self) -> bool:
+        r"""Returns whether this platform is InternLM."""
+        return self is ModelPlatformType.INTERNLM
+
 
 class AudioModelType(Enum):
     TTS_1 = "tts-1"

camel/types/openai_types.py

@@ -16,10 +16,10 @@ from openai.types.chat.chat_completion import ChatCompletion, Choice
 from openai.types.chat.chat_completion_assistant_message_param import (
     ChatCompletionAssistantMessageParam,
 )
-from openai.types.chat.chat_completion_chunk import ChatCompletionChunk
-from openai.types.chat.chat_completion_function_message_param import (
-    ChatCompletionFunctionMessageParam,
+from openai.types.chat.chat_completion_tool_message_param import (
+    ChatCompletionToolMessageParam,
 )
+from openai.types.chat.chat_completion_chunk import ChatCompletionChunk
 from openai.types.chat.chat_completion_message import ChatCompletionMessage
 from openai.types.chat.chat_completion_message_param import (
     ChatCompletionMessageParam,
@@ -33,6 +33,7 @@ from openai.types.chat.chat_completion_user_message_param import (
 from openai.types.completion_usage import CompletionUsage
 from openai.types.chat import ParsedChatCompletion
 from openai._types import NOT_GIVEN, NotGiven
+from openai.types.chat import ChatCompletionMessageToolCall
 
 Choice = Choice
 ChatCompletion = ChatCompletion
@@ -42,7 +43,8 @@ ChatCompletionMessageParam = ChatCompletionMessageParam
 ChatCompletionSystemMessageParam = ChatCompletionSystemMessageParam
 ChatCompletionUserMessageParam = ChatCompletionUserMessageParam
 ChatCompletionAssistantMessageParam = ChatCompletionAssistantMessageParam
-ChatCompletionFunctionMessageParam = ChatCompletionFunctionMessageParam
+ChatCompletionToolMessageParam = ChatCompletionToolMessageParam
+ChatCompletionMessageToolCall = ChatCompletionMessageToolCall
 CompletionUsage = CompletionUsage
 NOT_GIVEN = NOT_GIVEN
 NotGiven = NotGiven

camel/types/unified_model_type.py

@@ -113,6 +113,11 @@ class UnifiedModelType(str):
         r"""Returns whether the model is a Qwen model."""
         return True
 
+    @property
+    def is_internlm(self) -> bool:
+        r"""Returns whether the model is a InternLM model."""
+        return True
+
     @property
     def support_native_structured_output(self) -> bool:
         r"""Returns whether the model supports native structured output."""