crfm-helm 0.3.0__py3-none-any.whl → 0.5.0__py3-none-any.whl

helm/benchmark/annotation/image2structure/lilypond_compiler_annotator.py

@@ -0,0 +1,84 @@
+from typing import Tuple, Dict, Any
+import os
+import subprocess
+import tempfile
+
+from helm.benchmark.annotation.image2structure.image_compiler_annotator import ImageCompilerAnnotator, CompilationError
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.common.cache import CacheConfig
+from helm.common.optional_dependencies import handle_module_not_found_error, OptionalDependencyNotInstalled
+
+try:
+    from PIL import Image, ImageOps
+except ModuleNotFoundError as ex:
+    handle_module_not_found_error(ex, suggestions=["images"])
+
+
+class LilypondCompilerAnnotator(ImageCompilerAnnotator):
+    """Annotator that compiles the text completions into a music sheet with LilyPond."""
+
+    name: str = "lilypond_compiler"
+    base_path = "/home/josselin/installs/lilypond-2.24.3/bin"
+
+    def __init__(self, cache_config: CacheConfig, file_storage_path: str):
+        super().__init__(cache_config, file_storage_path)
+        try:
+            result = subprocess.run([f"{self.base_path}/lilypond", "--version"], capture_output=True, text=True)
+            if result.returncode != 0:
+                raise OptionalDependencyNotInstalled(
+                    "LilyPond is not installed. Download and install it from https://lilypond.org/download.html"
+                )
+        except FileNotFoundError as e:
+            raise OptionalDependencyNotInstalled(
+                "LilyPond is not installed. Download and install it from https://lilypond.org/download.html.\n"
+                f"Original error: {e}"
+            ) from e
+
+    def compile_completion_into_image(
+        self, request_state: RequestState, completion_text: str
+    ) -> Tuple[Image.Image, Dict[str, Any]]:
+        """Given a completion, compile it with LilyPond."""
+        # The LilyPond command requires a file on disk, so we write the completion to a temporary file
+        tmp = tempfile.NamedTemporaryFile()
+        ly_file_path: str = f"{tmp.name}.ly"
+        with open(ly_file_path, "w") as f:
+            f.write(completion_text)
+
+        # What we pass in as -o should be the same name as the .ly file, but without the extension
+        output_path: str = ly_file_path.replace(".ly", "")
+        # The image file of the music sheet should have the same path as the .ly file, but with .png extension
+        sheet_music_path: str = ly_file_path.replace(".ly", ".png")
+
+        try:
+            # Edits the LilyPond file to be compatible with the current version
+            result = subprocess.run(
+                [f"{self.base_path}/convert-ly", "-e", ly_file_path], capture_output=True, text=True
+            )
+            assert result.returncode == 0, f"convert-ly failed: {result.stderr}"
+
+            # Generate PNG image from the LilyPond file
+            # LilyPond supports partial compilation, which means it attempts to produce an image
+            # for the correct portions of the code, even if there are errors elsewhere
+            subprocess.run(
+                [f"{self.base_path}/lilypond", "--png", "-o", output_path, ly_file_path], capture_output=True, text=True
+            )
+            # If an image file is not generated, we consider it an absolute compilation failure
+            assert os.path.exists(sheet_music_path), "lilypond did not generate the image"
+
+            # Load the image as a PIL Image object
+            image = Image.open(sheet_music_path)
+
+            # Crop the image to remove the white space around the music sheet
+            (w, h) = image.size
+            image = image.crop((0, 0, w, h - int(h * 0.2)))  # Remove pagination
+            image = image.crop(ImageOps.invert(image).getbbox())  # Remove white border
+        except (AssertionError, RuntimeError) as e:
+            raise CompilationError(str(e)) from e
+        finally:
+            # Clean up the temporary files
+            if os.path.exists(ly_file_path):
+                os.remove(ly_file_path)
+            if os.path.exists(sheet_music_path):
+                os.remove(sheet_music_path)
+
+        return image, dict()

helm/benchmark/annotation/image2structure/webpage_compiler_annotator.py

@@ -0,0 +1,132 @@
+from typing import List, Tuple, Optional, Dict, Any
+import json
+import os
+import shutil
+import threading
+
+from helm.benchmark.annotation.image2structure.image_compiler_annotator import ImageCompilerAnnotator, CompilationError
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.common.optional_dependencies import handle_module_not_found_error
+from helm.benchmark.scenarios.vision_language.image2structure.webpage.driver import ScreenshotOptions
+from helm.benchmark.scenarios.vision_language.image2structure.webpage.utils import convert_html_to_text
+from helm.benchmark.scenarios.vision_language.image2structure.webpage_scenario import serve_and_take_screenshot
+from helm.benchmark.scenarios.scenario import ASSET_NAME_TAG, ASSET_PATH_TAG
+from helm.common.general import ensure_directory_exists
+from helm.common.cache import CacheConfig
+
+try:
+    from PIL import Image
+    from html2text import HTML2Text
+except ModuleNotFoundError as e:
+    handle_module_not_found_error(e, suggestions=["image2structure"])
+
+
+class WebpageCompilerAnnotator(ImageCompilerAnnotator):
+    """Annotator that compiles the text completions into a webpage
+    And takes a screenshot of the webpage."""
+
+    name: str = "webpage_compiler"
+
+    # Delimiters for the code block
+    DELIMITERS: List[Tuple[str, str]] = [
+        ("```json", "```"),
+        ("```", "```"),
+    ]
+
+    def __init__(self, cache_config: CacheConfig, file_storage_path: str):
+        super().__init__(cache_config, file_storage_path)
+        self._html2text = HTML2Text()
+        self._html2text.ignore_links = True
+
+    def postprocess_infos(self, infos: Dict[str, Any]) -> Dict[str, Any]:
+        """Postprocess the infos."""
+        annotations = super().postprocess_infos(infos)
+        assert "html" in annotations, "The html field should be present in the infos"
+        annotations["text"] = convert_html_to_text(self._html2text, infos["html"])
+        return annotations
+
+    def compile_completion_into_image(
+        self, request_state: RequestState, completion_text: str
+    ) -> Tuple[Image.Image, Dict[str, Any]]:
+        """Given a completion, parse the code and compile it into an image and return the image and the infos."""
+        # Create a temporary directory to store the files
+        cache_config: CacheConfig = self._cache.config
+        repo_path: str = "prod_env/tmp"
+        if hasattr(cache_config, "path"):
+            repo_path = os.path.join(os.path.dirname(cache_config.path), "tmp")
+        # Make the repo path thread safe by adding the thread id
+        repo_path = f"{repo_path}_{threading.get_ident()}"
+        ensure_directory_exists(repo_path)
+
+        # Check for code block delimiters
+        # After this completion should be a valid json object
+        for start, end in self.DELIMITERS:
+            if start in completion_text and end in completion_text[completion_text.index(start) + len(start) :]:
+                start_index = completion_text.index(start) + len(start)
+                end_index = completion_text.index(end, start_index)
+                completion_text = completion_text[start_index:end_index]
+                break
+
+        # Parse code into json object
+        structure: dict
+        try:
+            structure = json.loads(completion_text)
+        except json.JSONDecodeError as e:
+            raise CompilationError(f"Failed to parse the completion as a JSON object: {e}") from e
+
+        # Copy the assets
+        assets_paths: List[str] = []
+        assets_names: List[str] = []
+        for reference in request_state.instance.references:
+            if ASSET_PATH_TAG in reference.tags:
+                assert reference.output.multimedia_content is not None
+                for media_object in reference.output.multimedia_content.media_objects:
+                    assert media_object.is_local_file
+                    assert media_object.is_type("image")
+                    assert type(media_object.location) == str
+                    assets_paths.append(media_object.location)
+            if ASSET_NAME_TAG in reference.tags:
+                assert reference.output.multimedia_content is not None
+                for media_object in reference.output.multimedia_content.media_objects:
+                    assert media_object.is_type("text")
+                    assert type(media_object.text) == str
+                    assets_names.append(media_object.text)
+        assert len(assets_paths) == len(assets_names)
+        for asset_path, asset_name in zip(assets_paths, assets_names):
+            dest_path: str = os.path.join(repo_path, asset_name)
+            # Make sure the parent directory exists
+            os.makedirs(os.path.dirname(dest_path), exist_ok=True)
+            shutil.copyfile(asset_path, dest_path)
+            # os.symlink(asset_path, dest_path)
+
+        # Create each file in a temporary directory
+        if not isinstance(structure, list):
+            raise CompilationError("The completion should be a list of files")
+        for item in structure:
+            filename: Optional[str] = item.get("filename")
+            content: Optional[str] = item.get("content")
+            if filename is None or content is None:
+                raise CompilationError("Each file should have a valid filename and content")
+            # Create parent directories if they do not exist
+            if filename in assets_names:
+                # Some models will include assets in their response like this:
+                # {
+                #     "filename": "chmber.jpg",
+                #     "content": "The content of the chmber.jpg file is a binary image and cannot be displayed as text."
+                # }
+                # In this case, we skip the file creation
+                continue
+            parent_dir = os.path.join(repo_path, os.path.dirname(filename))
+            os.makedirs(parent_dir, exist_ok=True)
+            with open(os.path.join(repo_path, filename), "w") as f:
+                f.write(content)
+
+        # Save the screenshot, loads the image and remove the file
+        destination_path: str = os.path.join(repo_path, "output.png")
+        infos: Dict[str, Any] = serve_and_take_screenshot(repo_path, destination_path, ScreenshotOptions())
+        image: Image.Image = Image.open(destination_path)
+
+        # Delete the repository
+        shutil.rmtree(repo_path, ignore_errors=True)
+
+        return image, infos

helm/benchmark/annotation/test_annotator_factory.py

@@ -0,0 +1,26 @@
+from typing import Any, Dict
+import os
+import shutil
+
+from helm.benchmark.annotation.annotator_factory import AnnotatorFactory
+from helm.benchmark.annotation.annotator import Annotator, AnnotatorSpec
+from helm.common.cache_backend_config import BlackHoleCacheBackendConfig
+
+
+class TestAnnotatorFactory:
+    def setup_method(self):
+        credentials: Dict[str, Any] = {}
+        cache_config = BlackHoleCacheBackendConfig()
+        self.file_storage_path: str = "tmp"
+        self.annotator_factory = AnnotatorFactory(credentials, self.file_storage_path, cache_config)
+
+    def teardown_method(self):
+        if os.path.exists(self.file_storage_path):
+            shutil.rmtree(self.file_storage_path)
+
+    def test_get_annotator(self):
+        annotator = self.annotator_factory.get_annotator(
+            AnnotatorSpec(class_name="helm.benchmark.annotation.annotator.DummyAnnotator")
+        )
+        assert isinstance(annotator, Annotator)
+        assert annotator.name == "dummy"

helm/benchmark/annotation/test_dummy_annotator.py

@@ -0,0 +1,44 @@
+from typing import Any
+from dataclasses import replace
+import pytest
+
+from helm.benchmark.annotation.annotator import Annotator, DummyAnnotator
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.benchmark.scenarios.scenario import Instance, Input
+from helm.common.request import Request, RequestResult, GeneratedOutput
+
+
+class TestDummyAnnotator:
+    def setup_method(self):
+        self.annotator: Annotator = DummyAnnotator()
+        self.request_state = RequestState(
+            instance=Instance(input=Input(text="hello world"), references=[]),
+            request=Request(),
+            request_mode="original",
+            output_mapping=None,
+            result=None,
+            train_trial_index=0,
+            num_train_instances=0,
+            prompt_truncated=False,
+            reference_index=None,
+        )
+
+    def test_annotate(self):
+        request_state: RequestState = replace(
+            self.request_state,
+            result=RequestResult(
+                success=True,
+                embedding=[],
+                completions=[GeneratedOutput(text="How are you?", logprob=0, tokens=[])],
+                cached=True,
+            ),
+        )
+        annotations: Any = self.annotator.annotate(request_state)
+        assert annotations is not None
+        assert "all_caps" in annotations[0]
+        assert request_state.result is not None  # To make mypy happy
+        assert annotations[0]["all_caps"] == request_state.result.completions[0].text.upper()
+
+    def test_annotate_no_result(self):
+        with pytest.raises(ValueError):
+            self.annotator.annotate(self.request_state)

helm/benchmark/annotation_executor.py

@@ -0,0 +1,124 @@
+import os
+
+from typing import Optional, List, Dict, Any
+from dataclasses import dataclass, replace
+from helm.common.cache_backend_config import (
+    CacheBackendConfig,
+    BlackHoleCacheBackendConfig,
+    MongoCacheBackendConfig,
+    SqliteCacheBackendConfig,
+)
+
+from helm.common.general import ensure_directory_exists, parallel_map, get_credentials
+from helm.common.hierarchical_logger import htrack, hlog
+from helm.benchmark.adaptation.scenario_state import ScenarioState
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.benchmark.annotation.annotator import AnnotatorSpec, Annotator
+from helm.benchmark.annotation.annotator_factory import AnnotatorFactory
+from helm.proxy.services.service import CACHE_DIR
+
+
+class AnnotationExecutorError(Exception):
+    pass
+
+
+@dataclass(frozen=True)
+class AnnotationExecutionSpec:
+
+    local_path: str
+    """Path where API credentials and cache is stored.
+
+    This path is the same as `--base-path` when launching the proxy server (see server.py).
+    Required when url is not set."""
+
+    parallelism: int
+    """How many threads to have at once"""
+
+    dry_run: bool = False
+    """Whether to skip execution"""
+
+    sqlite_cache_backend_config: Optional[SqliteCacheBackendConfig] = None
+    """If set, SQLite will be used for the cache.
+
+    This specifies the directory in which the SQLite cache will store files.
+    At most one of sqlite_cache_backend_config and mongo_cache_backend_config can be set."""
+
+    mongo_cache_backend_config: Optional[MongoCacheBackendConfig] = None
+    """If set, MongoDB will be used for the cache.
+
+    This specifies the MongoDB database to be used by the MongoDB cache.
+    At most one of sqlite_cache_backend_config and mongo_cache_backend_config can be set."""
+
+
+class AnnotationExecutor:
+    """
+    An `Executor` takes a `ScenarioState` which has a bunch of requests.
+    Issue them to the API and return the results.
+    """
+
+    def __init__(self, execution_spec: AnnotationExecutionSpec):
+        self.execution_spec = execution_spec
+
+        cache_backend_config: CacheBackendConfig
+        if execution_spec.sqlite_cache_backend_config and execution_spec.mongo_cache_backend_config:
+            raise AnnotationExecutorError(
+                "At most one of sqlite_cache_backend_config and mongo_cache_backend_config can be set."
+            )
+        elif execution_spec.sqlite_cache_backend_config:
+            cache_backend_config = execution_spec.sqlite_cache_backend_config
+        elif execution_spec.mongo_cache_backend_config:
+            cache_backend_config = execution_spec.mongo_cache_backend_config
+        else:
+            cache_backend_config = BlackHoleCacheBackendConfig()
+
+        base_path: str = execution_spec.local_path
+        ensure_directory_exists(base_path)
+        client_file_storage_path = os.path.join(base_path, CACHE_DIR)
+        ensure_directory_exists(client_file_storage_path)
+        credentials: Dict[str, str] = get_credentials(base_path)
+        self.factory = AnnotatorFactory(
+            credentials=credentials,
+            file_storage_path=client_file_storage_path,
+            cache_backend_config=cache_backend_config,
+        )
+
+    @htrack(None)
+    def execute(self, scenario_state: ScenarioState) -> ScenarioState:
+        if self.execution_spec.dry_run:
+            hlog("Skipped annotation.")
+            return scenario_state
+
+        if scenario_state.annotator_specs is None or len(scenario_state.annotator_specs) == 0:
+            hlog("No annotators to run.")
+            return scenario_state
+
+        # Do it!
+        def do_it(request_state: RequestState) -> RequestState:
+            assert scenario_state.annotator_specs is not None
+            return self.process(scenario_state.annotator_specs, request_state)
+
+        self.annotator_specs = scenario_state.annotator_specs
+
+        request_states = parallel_map(
+            do_it,
+            scenario_state.request_states,
+            parallelism=self.execution_spec.parallelism,
+        )
+
+        hlog(f"Annotated {len(request_states)} requests")
+        return ScenarioState(
+            adapter_spec=scenario_state.adapter_spec,
+            request_states=request_states,
+            annotator_specs=scenario_state.annotator_specs,
+        )
+
+    def process(self, annotator_specs: List[AnnotatorSpec], state: RequestState) -> RequestState:
+        annotations: Dict[str, Any] = {}
+        try:
+            for annotator_spec in annotator_specs:
+                annotator: Annotator = self.factory.get_annotator(annotator_spec)
+                new_annotations = annotator.annotate(state)
+                annotations[annotator.name] = new_annotations
+        except Exception as e:
+            raise AnnotationExecutorError(f"{str(e)} Request: {state.request}") from e
+        return replace(state, annotations=annotations)

helm/benchmark/augmentations/cleva_perturbation.py

@@ -10,13 +10,13 @@ from helm.common.general import ensure_file_downloaded, ensure_directory_exists
 from helm.common.optional_dependencies import handle_module_not_found_error
 from helm.benchmark.scenarios.scenario import Input, Instance, Reference, Output
 from .perturbation_description import PerturbationDescription
-from .perturbation import Perturbation
+from .perturbation import Perturbation, TextPerturbation
 
 
 ############################################################
 
 
-class ChineseTyposPerturbation(Perturbation):
+class ChineseTyposPerturbation(TextPerturbation):
     """
     Chinese typos. For implementation details, see
     https://github.com/GEM-benchmark/NL-Augmenter/tree/main/nlaugmenter/transformations/chinese_butter_fingers_perturbation
@@ -271,7 +271,7 @@ class ChineseTyposPerturbation(Perturbation):
         return chars_with_similar_pinyin
 
 
-class ChineseSynonymPerturbation(Perturbation):
+class ChineseSynonymPerturbation(TextPerturbation):
     """
     Chinese synonyms. For implementation details, see
     https://github.com/GEM-benchmark/NL-Augmenter/blob/main/nlaugmenter/transformations/chinese_antonym_synonym_substitution
@@ -340,7 +340,7 @@ class ChineseSynonymPerturbation(Perturbation):
         return sample_list[index]
 
 
-class CLEVAMildMixPerturbation(Perturbation):
+class CLEVAMildMixPerturbation(TextPerturbation):
     """
     CLEVA robustness perturbation that composes several perturbations.
     """
@@ -370,7 +370,7 @@ class CLEVAMildMixPerturbation(Perturbation):
 ############################################################
 
 
-class ChineseGenderPerturbation(Perturbation):
+class ChineseGenderPerturbation(TextPerturbation):
     """Individual fairness perturbation for Chinese gender terms and pronouns."""
 
     name: str = "chinese_gender"
@@ -601,13 +601,6 @@ class ChinesePersonNamePerturbation(Perturbation):
         name = rng.choice(list(options))
         return name
 
-    def perturb(self, text: str, rng: Random) -> str:
-        """
-        Perturbing the text is handled in `perturb_with_persistency` to ensure that perturbed names
-        in `Instance`s and `Reference`s match.
-        """
-        raise NotImplementedError("Should never be called")
-
     def perturb_with_persistency(
         self, text: str, rng: Random, name_substitution_mapping: Dict[str, str], skipped_tokens: Set[str]
     ) -> str:
@@ -686,7 +679,7 @@ class ChinesePersonNamePerturbation(Perturbation):
         return tokens, tags
 
 
-class SimplifiedToTraditionalPerturbation(Perturbation):
+class SimplifiedToTraditionalPerturbation(TextPerturbation):
     """Individual fairness perturbation for Chinese simplified to Chinese traditional."""
 
     name: str = "simplified_to_traditional"
@@ -713,7 +706,7 @@ class SimplifiedToTraditionalPerturbation(Perturbation):
         return perturbed_text
 
 
-class MandarinToCantonesePerturbation(Perturbation):
+class MandarinToCantonesePerturbation(TextPerturbation):
     """
     Individual fairness perturbation for Mandarin to Cantonese translation.
     The implementation is inspired by https://justyy.com/tools/chinese-converter/

helm/benchmark/augmentations/contraction_expansion_perturbation.py

@@ -5,7 +5,7 @@ import re
 from random import Random
 
 from helm.common.general import match_case
-from .perturbation import Perturbation
+from .perturbation import TextPerturbation
 from .perturbation_description import PerturbationDescription
 
 
@@ -92,7 +92,7 @@ CONTRACTION_MAP: Dict[str, str] = {
 
 # The implementations below are based on
 # https://github.com/GEM-benchmark/NL-Augmenter/blob/main/transformations/contraction_expansions/transformation.py
-class ContractionPerturbation(Perturbation):
+class ContractionPerturbation(TextPerturbation):
     """
     Contractions.
     Replaces each expansion with its contracted version.
@@ -132,7 +132,7 @@ class ContractionPerturbation(Perturbation):
         return self.reverse_contraction_pattern.sub(cont, text)
 
 
-class ExpansionPerturbation(Perturbation):
+class ExpansionPerturbation(TextPerturbation):
     """
     Expansions.
     Replaces each contraction with its expanded version.

helm/benchmark/augmentations/contrast_sets_perturbation.py

@@ -81,6 +81,3 @@ class ContrastSetsPerturbation(Perturbation):
             references=perturbed_references,
             perturbation=description,
         )
-
-    def perturb(self, text: str, rng: Random) -> str:  # we need this since parent method is abstract
-        raise NotImplementedError("Should never be called since apply() was overridden")

helm/benchmark/augmentations/data_augmenter.py

@@ -35,7 +35,6 @@ class Processor:
 
 @dataclass(frozen=True)
 class DataAugmenter:
-
     # Perturbations to apply to generate new instances
     perturbations: List[Perturbation]
 
@@ -72,7 +71,6 @@ class DataAugmenter:
 
 @dataclass(frozen=True)
 class DataAugmenterSpec:
-
     # List of perturbation specs to use to augment the data
     perturbation_specs: List[PerturbationSpec] = field(default_factory=list)
 

helm/benchmark/augmentations/dialect_perturbation.py

@@ -8,10 +8,10 @@ from typing import Dict, Optional, List
 
 from helm.common.general import match_case, ensure_file_downloaded
 from .perturbation_description import PerturbationDescription
-from .perturbation import Perturbation
+from .perturbation import TextPerturbation
 
 
-class DialectPerturbation(Perturbation):
+class DialectPerturbation(TextPerturbation):
     """Individual fairness perturbation for dialect."""
 
     """ Short unique identifier of the perturbation (e.g., extra_space) """

helm/benchmark/augmentations/extra_space_perturbation.py

@@ -1,11 +1,11 @@
 from dataclasses import dataclass
 from random import Random
 
-from .perturbation import Perturbation
+from .perturbation import TextPerturbation
 from .perturbation_description import PerturbationDescription
 
 
-class ExtraSpacePerturbation(Perturbation):
+class ExtraSpacePerturbation(TextPerturbation):
     """
     A toy perturbation that replaces existing spaces in the text with
     `num_spaces` number of spaces.

helm/benchmark/augmentations/filler_words_perturbation.py

@@ -1,6 +1,6 @@
 from dataclasses import dataclass
 
-from .perturbation import Perturbation
+from .perturbation import TextPerturbation
 from .perturbation_description import PerturbationDescription
 
 from random import Random
@@ -31,7 +31,7 @@ UNCERTAIN_PHRASES = ["maybe", "perhaps", "probably", "possibly", "most likely"]
 FILL_PHRASE = ["uhm", "umm", "ahh", "err", "actually", "obviously", "naturally", "like", "you know"]
 
 
-class FillerWordsPerturbation(Perturbation):
+class FillerWordsPerturbation(TextPerturbation):
     """
     Randomly inserts filler words and phrases in the sentence.
     Perturbation example:

helm/benchmark/augmentations/gender_perturbation.py

@@ -6,7 +6,7 @@ from typing import Dict, List, Optional, Tuple
 
 from helm.common.general import match_case
 from .perturbation_description import PerturbationDescription
-from .perturbation import Perturbation
+from .perturbation import TextPerturbation
 
 
 """ Gender term mappings """
@@ -62,7 +62,7 @@ GENDER_PRONOUN_MAPPINGS: List[Tuple[str, ...]] = [
 ]
 
 
-class GenderPerturbation(Perturbation):
+class GenderPerturbation(TextPerturbation):
     """Individual fairness perturbation for gender terms and pronouns."""
 
     """ Short unique identifier of the perturbation (e.g., extra_space) """
@@ -214,7 +214,7 @@ class GenderPerturbation(Perturbation):
     def perturb(self, text: str, rng: Random) -> str:
         """Perform the perturbations on the provided text."""
         # Substitute the words
-        for (word, synonym) in self.word_synonym_pairs:
+        for word, synonym in self.word_synonym_pairs:
             text = self.substitute_word(text, word, synonym, rng)
 
         return text

helm/benchmark/augmentations/lowercase_perturbation.py

@@ -1,10 +1,10 @@
 from random import Random
 
-from .perturbation import Perturbation
+from .perturbation import TextPerturbation
 from .perturbation_description import PerturbationDescription
 
 
-class LowerCasePerturbation(Perturbation):
+class LowerCasePerturbation(TextPerturbation):
     """
     Simple perturbation turning input and references into lowercase.
     """

helm/benchmark/augmentations/mild_mix_perturbation.py

@@ -1,14 +1,14 @@
 from random import Random
 
 from .perturbation_description import PerturbationDescription
-from .perturbation import Perturbation
+from .perturbation import TextPerturbation
 from .lowercase_perturbation import LowerCasePerturbation
 from .contraction_expansion_perturbation import ContractionPerturbation
 from .space_perturbation import SpacePerturbation
 from .misspelling_perturbation import MisspellingPerturbation
 
 
-class MildMixPerturbation(Perturbation):
+class MildMixPerturbation(TextPerturbation):
     """
     Canonical robustness perturbation that composes several perturbations.
     These perturbations are chosen to be reasonable.

helm/benchmark/augmentations/misspelling_perturbation.py

@@ -6,13 +6,13 @@ from random import Random
 from typing import Dict, List
 
 from helm.common.general import match_case
-from .perturbation import Perturbation
+from .perturbation import TextPerturbation
 from .perturbation_description import PerturbationDescription
 
 
 # The implementation below is based on the following list of common misspellings:
 # https://en.wikipedia.org/wiki/Wikipedia:Lists_of_common_misspellings/For_machines
-class MisspellingPerturbation(Perturbation):
+class MisspellingPerturbation(TextPerturbation):
     """
     Replaces words randomly with common misspellings, from a list of common misspellings.