camel-ai 0.2.15a0__py3-none-any.whl → 0.2.16__py3-none-any.whl

camel/datagen/self_instruct/filter/filter_function.py

@@ -0,0 +1,216 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+
+import re
+from abc import ABC, abstractmethod
+from typing import List
+
+from rouge import Rouge
+
+from camel.models.reward import BaseRewardModel
+
+
+class FilterFunction(ABC):
+    r"""A base abstract class for filter functions.
+
+    Subclasses must implement the `apply` method, which determines whether
+    a given instruction passes the filter criteria.
+    """
+
+    @abstractmethod
+    def apply(self, instruction: str) -> bool:
+        r"""Evaluate the given instruction based on the filter's criteria.
+
+        Args:
+            instruction (str): The instruction to evaluate.
+
+        Returns:
+            bool: True if the instruction passes the filter, False otherwise.
+        """
+        pass
+
+
+class LengthFilter(FilterFunction):
+    r"""Filters instructions based on their word count.
+
+    Args:
+        min_len (int): The minimum word count required for an instruction.
+            (default::obj:`5`)
+        max_len (int): The maximum word count allowed for an instruction.
+            (default::obj:`200`)
+    """
+
+    def __init__(self, min_len: int = 5, max_len: int = 200):
+        self.min_len = min_len
+        self.max_len = max_len
+
+    def apply(self, instruction: str) -> bool:
+        r"""Filter the instruction
+
+        Args:
+            instruction (str): the instruction to be filtered.
+
+        Returns:
+            bool: True if the length of the instruction is within the range
+                of [min_len, max_len]
+        """
+        word_count = len(instruction.split())
+        return self.min_len <= word_count <= self.max_len
+
+
+class KeywordFilter(FilterFunction):
+    r"""Filters instructions that contain specific undesirable keywords.
+
+    Args:
+        keywords (List[str]): A list of keywords to filter out.
+    """
+
+    def __init__(self, keywords: List[str]):
+        self.keywords = [keyword.lower() for keyword in keywords]
+
+    def apply(self, instruction: str) -> bool:
+        r"""Filter the instruction
+
+        Args:
+            instruction (str): the instruction to be filtered.
+
+        Returns:
+            bool: True Instruction must NOT contain any of the keywords.
+        """
+        lower_instr = instruction.lower()
+        return not any(keyword in lower_instr for keyword in self.keywords)
+
+
+class PunctuationFilter(FilterFunction):
+    r"""Filters instructions that begin with a non-alphanumeric character."""
+
+    def apply(self, instruction: str) -> bool:
+        r"""Filter the instruction
+
+        Args:
+            instruction (str): the instruction to be filtered.
+
+        Returns:
+            bool: True if the instruction does not start with punctuation.
+        """
+        return not re.match(r'^[^\w\s]', instruction)
+
+
+class NonEnglishFilter(FilterFunction):
+    r"""Filters instructions that do not begin with English letters."""
+
+    def apply(self, instruction: str) -> bool:
+        r"""Filter the instruction
+
+        Args:
+            instruction (str): the instruction to be filtered.
+
+        Returns:
+            bool: True if the instruction starts with an English letter.
+        """
+        return bool(re.match(r'^[A-Za-z]', instruction))
+
+
+class RougeSimilarityFilter(FilterFunction):
+    r"""Filters instructions that are too similar to existing instructions
+    based on ROUGE scores.
+
+    Args:
+        existing_instructions (List[str]): A list of existing instructions to
+            compare against.
+        threshold (float): The similarity threshold for filtering.
+            (default::obj:`0.7`)
+    """
+
+    def __init__(
+        self, existing_instructions: List[str], threshold: float = 0.7
+    ):
+        self.existing_instructions = existing_instructions
+        self.threshold = threshold
+        self.rouge = Rouge()
+
+    def apply(self, instruction: str) -> bool:
+        r"""Filter the instruction
+
+        Args:
+            instruction (str): the instruction to be filtered.
+
+        Returns:
+            bool: True if the instruction's similarity to any existing
+                instruction is below the threshold.
+        """
+        if not self.existing_instructions:
+            return True
+
+        for existing_instr in self.existing_instructions:
+            scores = self.rouge.get_scores(instruction, existing_instr)
+            score = scores[0]['rouge-l']['f']
+            if score > self.threshold:
+                return False
+
+        return True
+
+
+class RewardModelFilter(FilterFunction):
+    r"""Filters instructions based on scores provided by a reward model.
+
+    Args:
+        reward_model (BaseRewardModel): The reward model used to evaluate
+            the instructions.
+        threshold (float): The minimum score required for an instruction
+            to pass the filter.
+    """
+
+    def __init__(
+        self,
+        reward_model: BaseRewardModel,
+        threshold: float = 0.5,
+    ):
+        self.prompt = ""
+        self.reward_model = reward_model
+        self.threshold = threshold
+
+    def apply(self, instruction: str) -> bool:
+        r"""Filter the instruction
+
+        Args:
+            instruction (str): The instruction to be filtered.
+
+        Returns:
+            bool: True if the instruction's score is above the threshold.
+
+        Raises:
+            ValueError: ValueError: If `score_types` is empty or if the
+                required score is not found in `scores`.
+        """
+
+        data = [
+            {"role": "user", "content": self.prompt},
+            {"role": "assistant", "content": instruction},
+        ]
+        scores = self.reward_model.evaluate(data)
+        score_types = self.reward_model.get_scores_types()
+        if not score_types:
+            raise ValueError("No score types available from the reward model.")
+
+        score_type = score_types[0]
+        score = scores.get(score_type, None)
+
+        if score is None:
+            raise ValueError(
+                f"Score type '{score_type}' is not found in the "
+                "evaluation scores."
+            )
+
+        return score >= self.threshold

camel/datagen/self_instruct/filter/filter_registry.py

@@ -0,0 +1,56 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+from typing import Any, Callable, Dict
+
+from .filter_function import (
+    FilterFunction,
+    KeywordFilter,
+    LengthFilter,
+    NonEnglishFilter,
+    PunctuationFilter,
+    RewardModelFilter,
+    RougeSimilarityFilter,
+)
+
+FILTER_REGISTRY: Dict[str, Callable[[Dict[str, Any]], FilterFunction]] = {
+    "length": lambda kwargs: LengthFilter(
+        min_len=kwargs.get("min_len", 5), max_len=kwargs.get("max_len", 200)
+    ),
+    "keyword": lambda kwargs: KeywordFilter(
+        keywords=kwargs.get("keywords", ["image", "data"])
+    ),
+    "punctuation": lambda kwargs: PunctuationFilter(),
+    "non_english": lambda kwargs: NonEnglishFilter(),
+    "rouge_similarity": lambda kwargs: RougeSimilarityFilter(
+        existing_instructions=kwargs.get("existing_instructions", []),
+        threshold=kwargs.get("threshold", 0.7),
+    ),
+    "reward": lambda kwargs: RewardModelFilter(
+        reward_model=kwargs.get("reward_model"),  # type:ignore[arg-type]
+        threshold=kwargs.get("threshold", 0.7),
+    ),
+}
+
+
+def register_filter(
+    name: str, constructor: Callable[[Dict[str, Any]], FilterFunction]
+):
+    r"""Registers a new filter constructor in FILTER_REGISTRY.
+
+    Args:
+        name (str): Unique name of the filter.
+        constructor (Callable[[Dict[str, Any]], FilterFunction]): Function to
+            create the filter using a dictionary of parameters.
+    """
+    FILTER_REGISTRY[name] = constructor

camel/datagen/self_instruct/filter/instruction_filter.py

@@ -0,0 +1,81 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+from typing import Any, Dict, List
+
+from .filter_function import FilterFunction, RewardModelFilter
+from .filter_registry import FILTER_REGISTRY
+
+
+class InstructionFilter:
+    def __init__(self, filters_config: Dict[str, Dict[str, Any]]):
+        r"""Initialize the InstructionFilter with a dictionary of filter
+            configurations.
+
+        Args:
+            filters_config(Dict[str, Dict[str, Any]]):
+                Example filters_config:
+                {
+                    "length": {"min_len": 5, "max_len": 100},
+                    "keyword": {"keywords": ["image", "video"]},
+                    "non_english": {},
+                    "rouge_similarity": {
+                        "existing_instructions": ["Some existing text"],
+                        "threshold": 0.6
+                    }
+                }
+                Each key in filters_config corresponds to a filter name
+                    (registered in FILTER_REGISTRY).
+                Each value is a dict of parameters for that filter.
+        """
+        self.filters: List[FilterFunction] = []
+        for filter_name, params in filters_config.items():
+            if filter_name not in FILTER_REGISTRY:
+                raise ValueError(f"Unknown filter function: {filter_name}")
+            self.filters.append(FILTER_REGISTRY[filter_name](params))
+
+    def add_filter(self, filter_function: FilterFunction):
+        r"""Add a custom filter function to the InstructionFilter.
+        This allows adding filters that are not in the registry.
+
+        Args:
+            filter_function (FilterFunction): The filter function to be added
+        """
+        self.filters.append(filter_function)
+
+    def filter(
+        self, prompt: str, instruction: str, return_details: bool = False
+    ):
+        r"""Check if the given instruction passes all filter functions.
+
+        Args:
+            prompt (str): The prompt of generating the instruction.
+            instruction (str): The instruction to evaluate.
+            return_details (bool): If True, returns a tuple (bool, List[str])
+                where the list contains the names of filters that failed.
+                (default::obj:`False`)
+
+        Returns:
+            bool: True if the instruction passes all filters, False otherwise.
+                OR (bool, List[str]) if return_details is True.
+        """
+        failed_filters = []
+        for f in self.filters:
+            if isinstance(f, RewardModelFilter):
+                f.prompt = prompt
+            if not f.apply(instruction):
+                failed_filters.append(type(f).__name__)
+
+        if return_details:
+            return len(failed_filters) == 0, failed_filters
+        return len(failed_filters) == 0