crfm-helm 0.4.0__py3-none-any.whl → 0.5.10__py3-none-any.whl

helm/common/moderations_api_request.py

@@ -0,0 +1,71 @@
+from dataclasses import dataclass
+from typing import Optional
+
+
+@dataclass(frozen=True)
+class ModerationAPIRequest:
+    # Text to check against OpenAI's content policy
+    text: str
+
+    # From https://beta.openai.com/docs/api-reference/moderations/create,
+    # "the default is text-moderation-latest which will be automatically upgraded over time.
+    # This ensures you are always using our most accurate model. If you use text-moderation-stable,
+    # we will provide advanced notice before updating the model. Accuracy of text-moderation-stable
+    # may be slightly lower than for text-moderation-latest."
+    use_latest_model: bool = False
+
+
+@dataclass(frozen=True)
+class ModerationCategoryFlaggedResults:
+    """
+    Contains per-category binary content violation flags.
+    For descriptions of the categories, see https://beta.openai.com/docs/guides/moderation/overview.
+    """
+
+    hate_flagged: bool
+    hate_threatening_flagged: bool
+    self_harm_flagged: bool
+    sexual_flagged: bool
+    sexual_minors_flagged: bool
+    violence_flagged: bool
+    violence_graphic_flagged: bool
+
+
+@dataclass(frozen=True)
+class ModerationCategoryScores:
+    """
+    Contains per-category scores. Values are between 0 and 1, where higher values denote higher
+    confidence. The scores should not be interpreted as probabilities.
+    For descriptions of the categories, see https://beta.openai.com/docs/guides/moderation/overview.
+    """
+
+    hate_score: float
+    hate_threatening_score: float
+    self_harm_score: float
+    sexual_score: float
+    sexual_minors_score: float
+    violence_score: float
+    violence_graphic_score: float
+
+
+@dataclass(frozen=True)
+class ModerationAPIRequestResult:
+    """Result after sending a `ModerationAPIRequest`."""
+
+    # Whether the request was successful
+    success: bool
+
+    # Whether the request was cached
+    cached: bool
+
+    # True if the model classifies the content as violating OpenAI's content policy, False otherwise
+    flagged: Optional[bool]
+
+    # Flagged results
+    flagged_results: Optional[ModerationCategoryFlaggedResults]
+
+    # Score results
+    scores: Optional[ModerationCategoryScores]
+
+    # If `success` is false, what was the error?
+    error: Optional[str] = None

helm/common/mongo_key_value_store.py

@@ -1,5 +1,5 @@
 import json
-from typing import Dict, Generator, Iterable, Optional, Tuple
+from typing import Dict, Generator, Iterable, Mapping, Optional, Tuple
 
 from helm.common.key_value_store import KeyValueStore
 from helm.common.optional_dependencies import handle_module_not_found_error
@@ -35,15 +35,15 @@ class MongoKeyValueStore(KeyValueStore):
     def __exit__(self, exc_type, exc_value, traceback) -> None:
         return
 
-    def _canonicalize_key(self, key: Dict) -> SON:
+    def _canonicalize_key(self, key: Mapping) -> SON:
         serialized = json.dumps(key, sort_keys=True)
         return json.loads(serialized, object_pairs_hook=SON)
 
-    def contains(self, key: Dict) -> bool:
+    def contains(self, key: Mapping) -> bool:
         query = {self._REQUEST_KEY: self._canonicalize_key(key)}
         return self._collection.find_one(query) is not None
 
-    def get(self, key: Dict) -> Optional[Dict]:
+    def get(self, key: Mapping) -> Optional[Dict]:
         query = {self._REQUEST_KEY: self._canonicalize_key(key)}
         document = self._collection.find_one(query)
         if document is not None:
@@ -63,7 +63,7 @@ class MongoKeyValueStore(KeyValueStore):
             else:
                 yield (request, response)
 
-    def put(self, key: Dict, value: Dict) -> None:
+    def put(self, key: Mapping, value: Dict) -> None:
         request = self._canonicalize_key(key)
         document = SON([(self._REQUEST_KEY, request), (self._RESPONSE_KEY, value)])
         # The MongoDB collection should have a unique indexed on "request"
@@ -84,5 +84,6 @@ class MongoKeyValueStore(KeyValueStore):
         # Note: unlike put, multi_put does not support documents with null bytes in keys.
         self._collection.bulk_write(operations)
 
-    def remove(self, key: Dict) -> None:
-        self._collection.delete_one(key)
+    def remove(self, key: Mapping) -> None:
+        query = {self._REQUEST_KEY: self._canonicalize_key(key)}
+        self._collection.delete_one(query)

helm/common/multimodal_request_utils.py

@@ -0,0 +1,57 @@
+import base64
+from typing import List, Optional
+
+import requests
+import urllib.parse
+
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.benchmark.scenarios.scenario import Reference
+from helm.common.request import RequestResult
+
+
+def get_contents_as_bytes(path: str) -> bytes:
+    """Get the contents at the location as bytes.
+
+    The location can be a local path or a URL."""
+    # Fetch the audio file and convert it to a base64 encoded string
+    is_remote = urllib.parse.urlparse(path).scheme in ["http", "https"]
+    if is_remote:
+        response = requests.get(path)
+        response.raise_for_status()
+        return response.content
+    else:
+        with open(path, "rb") as f:
+            return f.read()
+
+
+def get_contents_as_base64(path: str) -> str:
+    """Get the contents at the location as a base64-encoded string.
+
+    The location can be a local path or a URL."""
+    return base64.b64encode(get_contents_as_bytes(path)).decode("utf-8")
+
+
+def gather_generated_image_locations(request_result: RequestResult) -> List[str]:
+    """Gathers the locations (file paths or URLs) of the generated images."""
+    image_locations: List[str] = []
+    for image in request_result.completions:
+        # Models like DALL-E 2 can skip generating images for prompts that violate their content policy
+        if image.multimodal_content is None or image.multimodal_content.size == 0:
+            return []
+
+        location: Optional[str] = image.multimodal_content.media_objects[0].location
+        if location is not None:
+            image_locations.append(location)
+    return image_locations
+
+
+def get_gold_image_location(request_state: RequestState) -> str:
+    """Returns the first gold image location."""
+    references: List[Reference] = request_state.instance.references
+    assert (
+        len(references) > 0
+        and references[0].output.multimedia_content is not None
+        and references[0].output.multimedia_content.size > 0
+        and references[0].output.multimedia_content.media_objects[0].location is not None
+    ), "Expected at least one gold image"
+    return references[0].output.multimedia_content.media_objects[0].location

helm/common/nudity_check_request.py

@@ -0,0 +1,29 @@
+from dataclasses import dataclass, field
+from typing import List, Optional, Dict
+
+
+@dataclass(frozen=True)
+class NudityCheckRequest:
+    """
+    Checks for nudity for a given set of images.
+    """
+
+    # Batch of images
+    image_locations: List[str] = field(default_factory=list)
+
+
+@dataclass(frozen=True)
+class NudityCheckResult:
+    """Result after sending a `NudityCheckRequest`."""
+
+    # Whether the request was successful
+    success: bool
+
+    # Whether the request was cached
+    cached: bool
+
+    # Nudity results. True indicates the particular image contains nudity.
+    image_to_nudity: Dict[str, bool] = field(default_factory=dict)
+
+    # If `success` is false, what was the error?
+    error: Optional[str] = None

helm/common/object_spec.py

@@ -55,14 +55,23 @@ def inject_object_spec_args(
     This is loosely based on instance (constant) bindings and provider bindings in Guice dependency injection.
 
     Example:
-
-    class MyClass:
-        def __init__(a: int, b: int, c: int, d: int = 0):
-            pass
-
-    old_object_spec = ObjectSpec(class_name="MyClass", args={"a": 11})
-    new_object_spec = inject_object_spec_args(old_object_spec, {"b": 12}, {"c": lambda: 13})
-    # new_object_spec is now ObjectSpec(class_name="MyClass", args={"a": 11, "b": 12, "c": 13})
+        >>> from helm.common.object_spec import *  # NOQA
+        >>> import sys, types
+        >>> # Given a custom class with hashable arguments
+        >>> class MyClass:
+        ...     def __init__(a: int, b: int, c: int, d: int = 0):
+        ...         pass
+        >>> #
+        >>> # <boilerplate>: make a dummy module for MyClass to make this doctest exectuable
+        >>> sys.modules["my_module"] = type("MyModule", (types.ModuleType,), {"MyClass": MyClass})("my_module")
+        >>> # </boilerplate>
+        >>> #
+        >>> # Define new style and old style object specs
+        >>> old_object_spec = ObjectSpec(class_name="my_module.MyClass", args={"a": 11})
+        >>> new_object_spec = inject_object_spec_args(old_object_spec, {"b": 12}, {"c": lambda: 13})
+        >>> # new_object_spec is now
+        >>> print(new_object_spec)
+        ObjectSpec(class_name='my_module.MyClass', args={'a': 11, 'b': 12, 'c': 13})
     """
     cls = get_class_by_name(spec.class_name)
     init_signature = inspect.signature(cls.__init__)
@@ -93,6 +102,12 @@ def parse_object_spec(description: str) -> ObjectSpec:
         <class_name>:<key>=<value>,<key>=<value>
     Usually, the description is something that's succinct and can be typed on the command-line.
     Here, value defaults to string.
+
+    Example:
+        >>> from helm.common.object_spec import *  # NOQA
+        >>> description = 'mscoco:model=huggingface_stable-diffusion-v1-4'
+        >>> parse_object_spec(description)
+        ObjectSpec(class_name='mscoco', args={'model': 'huggingface_stable-diffusion-v1-4'})
     """
 
     def parse_arg(arg: str) -> Tuple[str, Any]:

helm/common/optional_dependencies.py

@@ -9,7 +9,7 @@ def handle_module_not_found_error(e: ModuleNotFoundError, suggestions: Optional[
     # TODO: Ask user to install more specific optional dependencies
     # e.g. crfm-helm[plots] or crfm-helm[server]
     suggested_commands = " or ".join(
-        [f"`pip install crfm-helm[{suggestion}]`" for suggestion in (suggestions or []) + ["all"]]
+        [f'`pip install "crfm-helm[{suggestion}]"`' for suggestion in (suggestions or []) + ["all"]]
     )
     raise OptionalDependencyNotInstalled(
         f"Optional dependency {e.name} is not installed. Please run {suggested_commands} to install it."

helm/common/reeval_parameters.py

@@ -0,0 +1,12 @@
+from dataclasses import dataclass
+from typing import Optional
+
+
+@dataclass(frozen=True)
+class REEvalParameters:
+    """
+    Parameters for reeval evaluation.
+    """
+
+    model_ability: Optional[float] = None
+    """The inital ability of the model to perform the task. Used for reeval evaluation."""

helm/common/remote_context.py

@@ -0,0 +1,61 @@
+from helm.common.context import Context
+from helm.common.cache import CacheConfig
+from helm.common.authentication import Authentication
+from helm.common.moderations_api_request import ModerationAPIRequest, ModerationAPIRequestResult
+from helm.common.critique_request import CritiqueRequest, CritiqueRequestResult
+from helm.common.nudity_check_request import NudityCheckRequest, NudityCheckResult
+from helm.common.file_upload_request import FileUploadRequest, FileUploadResult
+from helm.common.perspective_api_request import PerspectiveAPIRequest, PerspectiveAPIRequestResult
+from helm.common.clip_score_request import CLIPScoreRequest, CLIPScoreResult
+from helm.common.tokenization_request import (
+    TokenizationRequest,
+    TokenizationRequestResult,
+    DecodeRequestResult,
+    DecodeRequest,
+)
+from helm.common.request import Request, RequestResult
+from helm.proxy.query import Query, QueryResult
+from helm.proxy.services.remote_service import RemoteService
+from helm.proxy.services.service import GeneralInfo, Service
+
+
+class RemoteContext(Context):
+    def __init__(self, base_url: str, auth: Authentication):
+        self.service: Service = RemoteService(base_url)
+        self.auth = auth
+
+    def get_general_info(self) -> GeneralInfo:
+        return self.service.get_general_info()
+
+    def expand_query(self, query: Query) -> QueryResult:
+        return self.service.expand_query(query)
+
+    def make_request(self, request: Request) -> RequestResult:
+        return self.service.make_request(self.auth, request)
+
+    def tokenize(self, request: TokenizationRequest) -> TokenizationRequestResult:
+        return self.service.tokenize(self.auth, request)
+
+    def decode(self, request: DecodeRequest) -> DecodeRequestResult:
+        return self.service.decode(self.auth, request)
+
+    def upload(self, request: FileUploadRequest) -> FileUploadResult:
+        return self.service.upload(self.auth, request)
+
+    def check_nudity(self, request: NudityCheckRequest) -> NudityCheckResult:
+        return self.service.check_nudity(self.auth, request)
+
+    def compute_clip_score(self, request: CLIPScoreRequest) -> CLIPScoreResult:
+        return self.service.compute_clip_score(self.auth, request)
+
+    def get_toxicity_scores(self, request: PerspectiveAPIRequest) -> PerspectiveAPIRequestResult:
+        return self.service.get_toxicity_scores(self.auth, request)
+
+    def get_moderation_results(self, request: ModerationAPIRequest) -> ModerationAPIRequestResult:
+        return self.service.get_moderation_results(self.auth, request)
+
+    def make_critique_request(self, request: CritiqueRequest) -> CritiqueRequestResult:
+        return self.service.make_critique_request(self.auth, request)
+
+    def get_cache_config(self, shard_name: str) -> CacheConfig:
+        return self.service.get_cache_config(shard_name)

helm/common/request.py

@@ -3,7 +3,9 @@ from dataclasses import dataclass, field
 from typing import Any, Callable, Dict, List, Optional
 
 from helm.common.media_object import MultimediaObject
-from .general import indent_lines, format_text
+from helm.common.image_generation_parameters import ImageGenerationParameters
+from helm.common.general import indent_lines, format_text
+from helm.common.response_format import ResponseFormat
 
 
 @dataclass(frozen=True)
@@ -68,6 +70,28 @@ class Request:
     multimodal_prompt: Optional[MultimediaObject] = None
     """Multimodal prompt with media objects interleaved (e.g., text, video, image, text, ...)"""
 
+    image_generation_parameters: Optional[ImageGenerationParameters] = None
+    """Parameters for image generation."""
+
+    response_format: Optional[ResponseFormat] = None
+    """EXPERIMENTAL: Response format. Currently only supported by OpenAI and Together."""
+
+    def validate(self):
+        if (
+            (self.messages and self.prompt)
+            or (self.messages and self.multimodal_prompt)
+            or (self.prompt and self.multimodal_prompt)
+        ):
+            raise ValueError("Exactly one of the messages, prompt, multimodal_prompt fields should be set")
+
+        if self.multimodal_prompt:
+            for media_object in self.multimodal_prompt.media_objects:
+                if media_object.content_type == "text" and media_object.text is None:
+                    raise ValueError("Media object with text content type must have text set")
+
+                if media_object.content_type == "image" and media_object.location is None:
+                    raise ValueError("Media object with image content type must have location set")
+
     @property
     def model_host(self) -> str:
         """Returns the model host (referring to the deployment).
@@ -93,8 +117,6 @@ class Token:
     """
     A `Token` represents one token position in a `Sequence`, which has the
     chosen `text` as well as the top probabilities under the model.
-
-    Note: (text, logprob) could exist or not exist in `top_logprobs`.
     """
 
     # Text that was chosen
@@ -103,22 +125,20 @@ class Token:
     # Log probability of generating that
     logprob: float
 
-    # text -> log probability of generating that
-    top_logprobs: Dict[str, float]
-
     def render_lines(self) -> List[str]:
-        top_logprobs_entries = sorted(self.top_logprobs.items(), key=lambda entry: -entry[1])
-        top_logprobs_str = (
-            "{" + ", ".join(f"{format_text(text)}: {logprob}" for text, logprob in top_logprobs_entries) + "}"
-        )
         return [
-            f"{format_text(self.text)} logprob={self.logprob} top_logprobs={top_logprobs_str}",
+            f"{format_text(self.text)} logprob={self.logprob}",
         ]
 
 
 @dataclass(frozen=True)
-class Sequence:
-    """A `Sequence` is a sequence of tokens."""
+class Thinking:
+    text: Optional[str] = None
+
+
+@dataclass(frozen=True)
+class GeneratedOutput:
+    """A `GeneratedOutput` is a single generated output that may contain text or multimodal content."""
 
     # The concatenation of all the tokens
     text: str
@@ -130,10 +150,16 @@ class Sequence:
     tokens: List[Token]
 
     # Why did the sequence finish?
-    finish_reason: Optional[Dict] = None
+    finish_reason: Optional[Dict[str, Any]] = None
+
+    # Could be a sequence made up of multimedia content
+    multimodal_content: Optional[MultimediaObject] = None
+
+    # Could be reasoning
+    thinking: Optional[Thinking] = None
 
-    def __add__(self, other: "Sequence") -> "Sequence":
-        return Sequence(self.text + other.text, self.logprob + other.logprob, self.tokens + other.tokens)
+    def __add__(self, other: "GeneratedOutput") -> "GeneratedOutput":
+        return GeneratedOutput(self.text + other.text, self.logprob + other.logprob, self.tokens + other.tokens)
 
     def render_lines(self) -> List[str]:
         result = [
@@ -172,14 +198,14 @@ class RequestResult:
     embedding: List[float]
     """Fixed dimensional embedding corresponding to the entire prompt"""
 
-    completions: List[Sequence]
+    completions: List[GeneratedOutput]
     """List of completion"""
 
     cached: bool
     """Whether the request was actually cached"""
 
     request_time: Optional[float] = None
-    """How long did the request take?"""
+    """How long the request took in seconds"""
 
     request_datetime: Optional[int] = None
     """When was the request sent?
@@ -227,7 +253,7 @@ EMBEDDING_UNAVAILABLE_REQUEST_RESULT = RequestResult(
 )
 
 
-def wrap_request_time(compute: Callable[[], Dict[str, Any]]) -> Callable[[], Any]:
+def wrap_request_time(compute: Callable[[], Dict[str, Any]]) -> Callable[[], Dict[str, Any]]:
     """Return a version of `compute` that puts `request_time` into its output."""
 
     def wrapped_compute():

helm/common/response_format.py

@@ -0,0 +1,18 @@
+from dataclasses import dataclass
+from typing import Any, Dict, Optional
+
+
+@dataclass(frozen=True)
+class ResponseFormat:
+    """EXPERIMENTAL: Model response format.
+
+    Currently only supports JSON schema.
+
+    Currently only supported by OpenAI and Together.
+
+    See:
+    - https://platform.openai.com/docs/guides/structured-outputs
+    - https://docs.together.ai/docs/json-mode"""
+
+    json_schema: Optional[Dict[str, Any]] = None
+    """EXPERIMENTAL: The JSON schema that the model output should conform to."""

helm/common/test_cache.py

@@ -3,9 +3,7 @@ import tempfile
 import unittest
 import threading
 
-from helm.common.cache import Cache, SqliteCacheConfig, WithFollowerCacheConfig, cache_stats, get_all_from_sqlite
-
-from sqlitedict import SqliteDict
+from helm.common.cache import Cache, SqliteCacheConfig, cache_stats, get_all_from_sqlite
 
 
 class TestCache(unittest.TestCase):
@@ -99,51 +97,6 @@ class TestCache(unittest.TestCase):
         assert cache_stats.num_computes[self.cache_path] >= num_items
         assert cache_stats.num_computes[self.cache_path] <= num_items * num_threads
 
-    def test_follower(self):
-        cache = Cache(SqliteCacheConfig(self.cache_path))
-        request_1 = {"name": "request1"}
-        compute_1 = lambda: {"response": "response1"}
-
-        response, cached = cache.get(request_1, compute_1)
-        assert response == {"response": "response1"}
-        assert not cached
-        assert cache_stats.num_queries[self.cache_path] == 1
-        assert cache_stats.num_computes[self.cache_path] == 1
-
-        follower_cache_file = tempfile.NamedTemporaryFile(delete=False)
-        follower_cache_path = follower_cache_file.name
-        with follower_cache_file:
-            cache_with_follower_config = WithFollowerCacheConfig(
-                main=SqliteCacheConfig(self.cache_path),
-                follower=SqliteCacheConfig(follower_cache_path),
-            )
-            cache_with_follower = Cache(cache_with_follower_config)
-
-            response, cached = cache_with_follower.get(request_1, compute_1)
-            assert response == {"response": "response1"}
-            assert cached
-            assert cache_stats.num_queries[self.cache_path] == 2
-            assert cache_stats.num_computes[self.cache_path] == 1
-            assert cache_stats.num_queries[follower_cache_path] == 0
-            assert cache_stats.num_computes[follower_cache_path] == 0
-
-            request_2 = {"name": "request2"}
-            compute_2 = lambda: {"response": "response2"}
-
-            response, cached = cache_with_follower.get(request_2, compute_2)
-            assert response == {"response": "response2"}
-            assert not cached
-            assert cache_stats.num_queries[self.cache_path] == 3
-            assert cache_stats.num_computes[self.cache_path] == 2
-            assert cache_stats.num_queries[follower_cache_path] == 0
-            assert cache_stats.num_computes[follower_cache_path] == 0
-
-            expected_dict = {
-                '{"name": "request1"}': {"response": "response1"},
-                '{"name": "request2"}': {"response": "response2"},
-            }
-            self.assertCountEqual(SqliteDict(follower_cache_path).items(), expected_dict.items())
-
     def test_get_all_from_sqlite(self):
         cache = Cache(SqliteCacheConfig(self.cache_path))
         num_items = 10  # TODO: Inrcrease to 100

helm/common/test_general.py

@@ -1,5 +1,8 @@
 import shutil
 import os
+
+import pytest
+
 from helm.common.general import (
     ensure_file_downloaded,
     format_tags,
@@ -7,10 +10,12 @@ from helm.common.general import (
     format_split,
     get_file_name,
     unique_simplification,
+    is_url,
 )
 
 
 def test_ensure_file_downloaded():
+    pytest.skip("Skipping download tests because these downloads are not reliable and may be throttled")
     ensure_file_downloaded("https://ftp.gnu.org/gnu/tar/tar-1.34.tar.gz", "test-tar", unpack=True, unpack_type="untar")
     assert os.path.isdir("test-tar")
     shutil.rmtree("test-tar")
@@ -58,3 +63,8 @@ def test_unique_simplification():
 
 def test_get_file_name():
     assert get_file_name("/path/to/image.png") == "image.png"
+
+
+def test_is_url():
+    assert is_url("https://crfm.stanford.edu")
+    assert not is_url("/some/path")

helm/common/test_logging.py

@@ -0,0 +1,94 @@
+import sys
+import tempfile
+import textwrap
+import pathlib
+from helm.benchmark import run
+from typing import List, Optional
+
+
+class ArgvContext:
+    """
+    Helper to assign a temporary value to sys.argv and then restore it
+    """
+
+    def __init__(self, argv: Optional[List[str]]):
+        self.argv = argv
+        self._original_argv: Optional[List[str]] = None
+
+    def __enter__(self):
+        self._original_argv = sys.argv[:]
+        sys.argv = self.argv or []
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        assert self._original_argv is not None  # Satisfies mypy
+        sys.argv = self._original_argv
+
+
+def test_run_with_custom_logging_config():
+    # Setup temporary directory
+    with tempfile.TemporaryDirectory(prefix="helm_test_") as tmp_dir_str:
+        tmp_dir = pathlib.Path(tmp_dir_str)
+        log_path = tmp_dir / "test.log"
+        log_config_path = tmp_dir / "test_config.yaml"
+
+        # Write custom YAML log config to file
+        log_config_text = textwrap.dedent(
+            f"""
+            version: 1
+            disable_existing_loggers: false
+            formatters:
+              simple:
+                datefmt: '%Y-%m-%dT%H:%M:%S'
+                format: '%(asctime)s %(levelname)s %(name)s %(message)s'
+            handlers:
+              file:
+                class: logging.FileHandler
+                filename: {log_path}
+                formatter: simple
+                level: DEBUG
+                mode: w
+            loggers:
+              helm:
+                handlers:
+                - file
+                level: DEBUG
+                propagate: false
+            """
+        ).strip()
+
+        log_config_path.write_text(log_config_text)
+
+        # Simulate command-line arguments
+        argv = [
+            "run.py",  # Fake script name
+            "--run-entries",
+            "mmlu:subject=philosophy,model=openai/gpt2",
+            "-m",
+            "1",
+            "--suite",
+            "my-suite",
+            "--dry-run",
+            "--log-config",
+            str(log_config_path),
+        ]
+
+        # Call main
+        with ArgvContext(argv):
+            run.main()
+
+        # Check log file contents
+        assert log_path.exists(), "Log file was not created"
+        log_contents = log_path.read_text()
+
+        # Test that log file was written to disk as requested
+        print("Log Contents")
+        print("------------")
+        print(log_contents)
+
+        assert (
+            "mscoco" in log_contents or "huggingface" in log_contents or "dry-run" in log_contents
+        ), "Expected log content not found in log file:\n"
+
+
+if __name__ == "__main__":
+    test_run_with_custom_logging_config()

helm/common/test_media_object.py

@@ -1,6 +1,6 @@
 import unittest
 
-from .media_object import MediaObject, MultimediaObject
+from helm.common.media_object import MediaObject, MultimediaObject
 
 
 class TestMediaObject(unittest.TestCase):