crfm-helm 0.5.6__py3-none-any.whl → 0.5.10__py3-none-any.whl

helm/clients/audio_language/qwen_omni/qwen2_5_omni_utils/v2_5/vision_process.py

@@ -0,0 +1,380 @@
+from __future__ import annotations
+
+import base64
+import logging
+import math
+import os
+import sys
+import time
+import warnings
+from functools import lru_cache
+from io import BytesIO
+
+import requests
+import torch
+import torchvision
+from packaging import version
+from PIL import Image
+from torchvision import io, transforms
+from torchvision.transforms import InterpolationMode
+from typing import List, Optional, Union
+
+
+logger = logging.getLogger(__name__)
+
+IMAGE_FACTOR = 28
+MIN_PIXELS = 4 * 28 * 28
+MAX_PIXELS = 16384 * 28 * 28
+MAX_RATIO = 200
+
+VIDEO_MIN_PIXELS = 128 * 28 * 28
+VIDEO_MAX_PIXELS = 768 * 28 * 28
+FRAME_FACTOR = 2
+FPS = 2.0
+FPS_MIN_FRAMES = 4
+FPS_MAX_FRAMES = 768
+
+# Set the maximum number of video token inputs.
+# Here, 128K represents the maximum number of input tokens for the VLLM model.
+# Remember to adjust it according to your own configuration.
+VIDEO_TOTAL_PIXELS = int(float(os.environ.get("VIDEO_MAX_PIXELS", 128000 * 28 * 28 * 0.9)))
+logger.info(f"set VIDEO_TOTAL_PIXELS: {VIDEO_TOTAL_PIXELS}")
+
+
+def round_by_factor(number: int, factor: int) -> int:
+    """Returns the closest integer to 'number' that is divisible by 'factor'."""
+    return round(number / factor) * factor
+
+
+def ceil_by_factor(number: int, factor: int) -> int:
+    """Returns the smallest integer greater than or equal to 'number' that is divisible by 'factor'."""
+    return math.ceil(number / factor) * factor
+
+
+def floor_by_factor(number: int, factor: int) -> int:
+    """Returns the largest integer less than or equal to 'number' that is divisible by 'factor'."""
+    return math.floor(number / factor) * factor
+
+
+def smart_resize(
+    height: int, width: int, factor: int = IMAGE_FACTOR, min_pixels: int = MIN_PIXELS, max_pixels: int = MAX_PIXELS
+) -> tuple[int, int]:
+    """
+    Rescales the image so that the following conditions are met:
+
+    1. Both dimensions (height and width) are divisible by 'factor'.
+
+    2. The total number of pixels is within the range ['min_pixels', 'max_pixels'].
+
+    3. The aspect ratio of the image is maintained as closely as possible.
+    """
+    if max(height, width) / min(height, width) > MAX_RATIO:
+        raise ValueError(
+            f"absolute aspect ratio must be smaller than {MAX_RATIO}, got {max(height, width) / min(height, width)}"
+        )
+    h_bar = max(factor, round_by_factor(height, factor))
+    w_bar = max(factor, round_by_factor(width, factor))
+    if h_bar * w_bar > max_pixels:
+        beta = math.sqrt((height * width) / max_pixels)
+        h_bar = floor_by_factor(int(height / beta), factor)
+        w_bar = floor_by_factor(int(width / beta), factor)
+    elif h_bar * w_bar < min_pixels:
+        beta = math.sqrt(min_pixels / (height * width))
+        h_bar = ceil_by_factor(int(height * beta), factor)
+        w_bar = ceil_by_factor(int(width * beta), factor)
+    return h_bar, w_bar
+
+
+def to_rgb(pil_image: Image.Image) -> Image.Image:
+    if pil_image.mode == "RGBA":
+        white_background = Image.new("RGB", pil_image.size, (255, 255, 255))
+        white_background.paste(pil_image, mask=pil_image.split()[3])  # Use alpha channel as mask
+        return white_background
+    else:
+        return pil_image.convert("RGB")
+
+
+def fetch_image(ele, size_factor: int = IMAGE_FACTOR) -> Image.Image:
+    if "image" in ele:
+        image = ele["image"]
+    else:
+        image = ele["image_url"]
+    image_obj = None
+    if isinstance(image, Image.Image):
+        image_obj = image
+    elif image.startswith("http://") or image.startswith("https://"):
+        response = requests.get(image, stream=True)
+        image_obj = Image.open(BytesIO(response.content))
+    elif image.startswith("file://"):
+        image_obj = Image.open(image[7:])
+    elif image.startswith("data:image"):
+        if "base64," in image:
+            _, base64_data = image.split("base64,", 1)
+            data = base64.b64decode(base64_data)
+            image_obj = Image.open(BytesIO(data))
+    else:
+        image_obj = Image.open(image)
+    if image_obj is None:
+        raise ValueError(f"Unrecognized image input, support local path, http url, base64 and PIL.Image, got {image}")
+    image = to_rgb(image_obj)
+    # resize
+    if "resized_height" in ele and "resized_width" in ele:
+        resized_height, resized_width = smart_resize(
+            int(ele["resized_height"]),
+            int(ele["resized_width"]),
+            factor=size_factor,
+        )
+    else:
+        width, height = image.size
+        min_pixels = int(ele.get("min_pixels", MIN_PIXELS))
+        max_pixels = int(ele.get("max_pixels", MAX_PIXELS))
+        resized_height, resized_width = smart_resize(
+            height,
+            width,
+            factor=size_factor,
+            min_pixels=min_pixels,
+            max_pixels=max_pixels,
+        )
+    image = image.resize((resized_width, resized_height))
+
+    return image
+
+
+def smart_nframes(
+    ele: dict,
+    total_frames: int,
+    video_fps: Union[int, float],
+) -> int:
+    """calculate the number of frames for video used for model inputs.
+
+    Args:
+        ele (dict): a dict contains the configuration of video.
+            support either `fps` or `nframes`:
+                - nframes: the number of frames to extract for model inputs.
+                - fps: the fps to extract frames for model inputs.
+                    - min_frames: the minimum number of frames of the video, only used when fps is provided.
+                    - max_frames: the maximum number of frames of the video, only used when fps is provided.
+        total_frames (int): the original total number of frames of the video.
+        video_fps (int | float): the original fps of the video.
+
+    Raises:
+        ValueError: nframes should in interval [FRAME_FACTOR, total_frames].
+
+    Returns:
+        int: the number of frames for video used for model inputs.
+    """
+    assert not ("fps" in ele and "nframes" in ele), "Only accept either `fps` or `nframes`"
+    if "nframes" in ele:
+        nframes = round_by_factor(ele["nframes"], FRAME_FACTOR)
+    else:
+        fps = ele.get("fps", FPS)
+        min_frames = ceil_by_factor(ele.get("min_frames", FPS_MIN_FRAMES), FRAME_FACTOR)
+        max_frames = floor_by_factor(ele.get("max_frames", min(FPS_MAX_FRAMES, total_frames)), FRAME_FACTOR)
+        nframes = total_frames / video_fps * fps
+        if nframes > total_frames:
+            logger.warning(f"smart_nframes: nframes[{nframes}] > total_frames[{total_frames}]")
+        nframes = min(min(max(nframes, min_frames), max_frames), total_frames)
+        nframes = floor_by_factor(nframes, FRAME_FACTOR)
+    if not (FRAME_FACTOR <= nframes and nframes <= total_frames):
+        raise ValueError(f"nframes should in interval [{FRAME_FACTOR}, {total_frames}], but got {nframes}.")
+    return nframes
+
+
+def _read_video_torchvision(
+    ele: dict,
+):
+    """read video using torchvision.io.read_video
+
+    Args:
+        ele (dict): a dict contains the configuration of video.
+        support keys:
+            - video: the path of video. support "file://", "http://", "https://" and local path.
+            - video_start: the start time of video.
+            - video_end: the end time of video.
+    Returns:
+        torch.Tensor: the video tensor with shape (T, C, H, W).
+    """
+    video_path = ele["video"]
+    if version.parse(torchvision.__version__) < version.parse("0.19.0"):
+        if "http://" in video_path or "https://" in video_path:
+            warnings.warn("torchvision < 0.19.0 does not support http/https video path, please upgrade to 0.19.0.")
+        if "file://" in video_path:
+            video_path = video_path[7:]
+    st = time.time()
+    video, audio, info = io.read_video(
+        video_path,
+        start_pts=ele.get("video_start", 0.0),
+        end_pts=ele.get("video_end", None),
+        pts_unit="sec",
+        output_format="TCHW",
+    )
+    total_frames, video_fps = video.size(0), info["video_fps"]
+    logger.info(f"torchvision:  {video_path=}, {total_frames=}, {video_fps=}, time={time.time() - st:.3f}s")
+    nframes = smart_nframes(ele, total_frames=total_frames, video_fps=video_fps)
+    idx = torch.linspace(0, total_frames - 1, nframes).round().long()
+    sample_fps = nframes / max(total_frames, 1e-6) * video_fps
+    video = video[idx]
+    return video, sample_fps
+
+
+def is_decord_available() -> bool:
+    import importlib.util
+
+    return importlib.util.find_spec("decord") is not None
+
+
+def _read_video_decord(
+    ele: dict,
+):
+    """read video using decord.VideoReader
+
+    Args:
+        ele (dict): a dict contains the configuration of video.
+        support keys:
+            - video: the path of video. support "file://", "http://", "https://" and local path.
+            - video_start: the start time of video.
+            - video_end: the end time of video.
+    Returns:
+        torch.Tensor: the video tensor with shape (T, C, H, W).
+    """
+    import decord
+
+    video_path = ele["video"]
+    st = time.time()
+    vr = decord.VideoReader(video_path)
+    # TODO: support start_pts and end_pts
+    if "video_start" in ele or "video_end" in ele:
+        raise NotImplementedError("not support start_pts and end_pts in decord for now.")
+    total_frames, video_fps = len(vr), vr.get_avg_fps()
+    logger.info(f"decord:  {video_path=}, {total_frames=}, {video_fps=}, time={time.time() - st:.3f}s")
+    nframes = smart_nframes(ele, total_frames=total_frames, video_fps=video_fps)
+    idx = torch.linspace(0, total_frames - 1, nframes).round().long().tolist()
+    video = vr.get_batch(idx).asnumpy()
+    video = torch.tensor(video).permute(0, 3, 1, 2)  # Convert to TCHW format
+    sample_fps = nframes / max(total_frames, 1e-6) * video_fps
+    return video, sample_fps
+
+
+VIDEO_READER_BACKENDS = {
+    "decord": _read_video_decord,
+    "torchvision": _read_video_torchvision,
+}
+
+FORCE_QWENVL_VIDEO_READER = os.getenv("FORCE_QWENVL_VIDEO_READER", None)
+
+
+@lru_cache(maxsize=1)
+def get_video_reader_backend() -> str:
+    if FORCE_QWENVL_VIDEO_READER is not None:
+        video_reader_backend = FORCE_QWENVL_VIDEO_READER
+    elif is_decord_available():
+        video_reader_backend = "decord"
+    else:
+        video_reader_backend = "torchvision"
+    print(f"qwen-vl-utils using {video_reader_backend} to read video.", file=sys.stderr)
+    return video_reader_backend
+
+
+def fetch_video(ele: dict, image_factor: int = IMAGE_FACTOR, return_video_sample_fps: bool = False):
+    if isinstance(ele["video"], str):
+        video_reader_backend = get_video_reader_backend()
+        try:
+            video, sample_fps = VIDEO_READER_BACKENDS[video_reader_backend](ele)
+        except Exception as e:
+            logger.warning(f"video_reader_backend {video_reader_backend} error, use torchvision as default, msg: {e}")
+            video, sample_fps = VIDEO_READER_BACKENDS["torchvision"](ele)
+
+        nframes, _, height, width = video.shape
+        min_pixels = ele.get("min_pixels", VIDEO_MIN_PIXELS)
+        total_pixels = ele.get("total_pixels", VIDEO_TOTAL_PIXELS)
+        max_pixels = max(min(VIDEO_MAX_PIXELS, total_pixels / nframes * FRAME_FACTOR), int(min_pixels * 1.05))
+        max_pixels_supposed = ele.get("max_pixels", max_pixels)
+        if max_pixels_supposed > max_pixels:
+            logger.warning(f"The given max_pixels[{max_pixels_supposed}] exceeds limit[{max_pixels}].")
+        max_pixels = min(max_pixels_supposed, max_pixels)
+        if "resized_height" in ele and "resized_width" in ele:
+            resized_height, resized_width = smart_resize(
+                ele["resized_height"],
+                ele["resized_width"],
+                factor=image_factor,
+            )
+        else:
+            resized_height, resized_width = smart_resize(
+                height,
+                width,
+                factor=image_factor,
+                min_pixels=min_pixels,
+                max_pixels=max_pixels,
+            )
+        video = transforms.functional.resize(
+            video,
+            [resized_height, resized_width],
+            interpolation=InterpolationMode.BICUBIC,
+            antialias=True,
+        ).float()
+        if return_video_sample_fps:
+            return video, sample_fps
+        return video
+    else:
+        assert isinstance(ele["video"], (list, tuple))
+        process_info = ele.copy()
+        process_info.pop("type", None)
+        process_info.pop("video", None)
+        images = [
+            fetch_image({"image": video_element, **process_info}, size_factor=image_factor)
+            for video_element in ele["video"]
+        ]
+        nframes = ceil_by_factor(len(images), FRAME_FACTOR)
+        if len(images) < nframes:
+            images.extend([images[-1]] * (nframes - len(images)))
+        if return_video_sample_fps:
+            return images, process_info.pop("fps", 2.0)
+        return images
+
+
+def extract_vision_info(conversations) -> list[dict]:
+    vision_infos = []
+    if isinstance(conversations[0], dict):
+        conversations_p = [conversations]
+    for conversation in conversations_p:
+        for message in conversation:
+            if isinstance(message["content"], list):
+                for ele in message["content"]:
+                    if (
+                        "image" in ele
+                        or "image_url" in ele
+                        or "video" in ele
+                        or ele["type"] in ("image", "image_url", "video")
+                    ):
+                        vision_infos.append(ele)
+    return vision_infos
+
+
+def process_vision_info(
+    conversations: list[dict] | list[list[dict]],
+    return_video_kwargs: bool = False,
+):
+
+    vision_infos = extract_vision_info(conversations)
+    # Read images or videos
+    image_inputs: Optional[List] = []
+    video_inputs: Optional[List] = []
+    video_sample_fps_list = []
+    for vision_info in vision_infos:
+        if "image" in vision_info or "image_url" in vision_info:
+            assert image_inputs is not None
+            image_inputs.append(fetch_image(vision_info))
+        elif "video" in vision_info:
+            assert video_inputs is not None
+            video_input, video_sample_fps = fetch_video(vision_info, return_video_sample_fps=True)
+            video_sample_fps_list.append(video_sample_fps)
+            video_inputs.append(video_input)
+        else:
+            raise ValueError("image, image_url or video should in content.")
+    if image_inputs is not None and len(image_inputs) == 0:
+        image_inputs = None
+    if video_inputs is not None and len(video_inputs) == 0:
+        video_inputs = None
+    if return_video_kwargs:
+        return image_inputs, video_inputs, {"fps": video_sample_fps_list}
+    return image_inputs, video_inputs

helm/clients/bedrock_client.py

@@ -7,8 +7,10 @@ from datetime import datetime
 
 from helm.common.cache import CacheConfig
 from helm.clients.client import CachingClient, truncate_and_tokenize_response_text
+from helm.common.hierarchical_logger import hexception
 from helm.common.request import Request, RequestResult, GeneratedOutput, wrap_request_time
 from helm.clients.bedrock_utils import get_bedrock_client, get_bedrock_client_v1
+from helm.proxy.retry import NonRetriableException
 from helm.tokenizers.tokenizer import Tokenizer
 
 
@@ -41,6 +43,7 @@ class BedrockClient(CachingClient):
         tokenizer_name: str,
         assumed_role: Optional[str] = None,
         region: Optional[str] = None,
+        bedrock_model_id: Optional[str] = None,
     ):
         super().__init__(cache_config=cache_config)
         self.tokenizer = tokenizer
@@ -49,15 +52,19 @@ class BedrockClient(CachingClient):
             assumed_role=assumed_role or os.environ.get("BEDROCK_ASSUME_ROLE", None),
             region=region,
         )
+        self.bedrock_model_id = bedrock_model_id
 
     def make_request(self, request: Request) -> RequestResult:
-        # model_id should be something like "amazon.titan-tg1-large", replace amazon- prefix with model creator name
-        model_name = request.model.split("/")[-1]
-        # check if model_name starts with "amazon-"
-        if self.model_provider == "amazon":
-            model_id = f"{self.model_provider}.{model_name}"
+        if self.bedrock_model_id:
+            model_id = self.bedrock_model_id
         else:
-            model_id = model_name.replace("amazon-", f"{self.model_provider}.")
+            # model_id should be something like "amazon.titan-tg1-large", replace amazon- prefix with model creator name
+            model_name = request.model.split("/")[-1]
+            # check if model_name starts with "amazon-"
+            if self.model_provider == "amazon":
+                model_id = f"{self.model_provider}.{model_name}"
+            else:
+                model_id = model_name.replace("amazon-", f"{self.model_provider}.")
 
         raw_request = self.convert_request_to_raw_request(request)
 
@@ -75,6 +82,7 @@ class BedrockClient(CachingClient):
             response, cached = self.cache.get(cache_key, wrap_request_time(do_it))
 
         except Exception as error:
+            hexception(error)
             return RequestResult(
                 success=False,
                 cached=False,
@@ -322,3 +330,52 @@ class BedrockLlamaClient(BedrockClient):
         completions.append(completion)
 
         return completions
+
+
+class BedrockPalmyraClient(BedrockClient):
+    """Amazon Bedrock model for Palmyra models"""
+
+    _COMPLETION_REASON_TO_FINISH_REASON = {
+        "length": "length",
+        "stop": "endoftext",
+    }
+
+    model_provider = "writer"
+
+    def _get_messages_from_request(self, request: Request) -> List[Dict]:
+        if request.prompt and request.messages:
+            raise NonRetriableException(f"Only one of `prompt` and `messages` may be set in request: {request}")
+        if request.multimodal_prompt:
+            raise NonRetriableException("`multimodal_prompt` is not supported by WriterClient")
+        if request.messages:
+            return [{"role": message["role"], "content": message["content"]} for message in request.messages]
+        else:
+            return [{"role": "user", "content": request.prompt}]
+
+    def convert_request_to_raw_request(self, request: Request) -> Dict:
+        return {
+            "messages": self._get_messages_from_request(request),
+            "temperature": request.temperature,
+            "top_p": request.top_p,
+            "max_tokens": request.max_tokens,
+            "stop": request.stop_sequences,
+            "seed": request.random,
+            "presence_penalty": request.presence_penalty,
+            "frequency_penalty": request.frequency_penalty,
+        }
+
+    def convert_raw_response_to_completions(self, response: Dict, request: Request) -> List[GeneratedOutput]:
+        completions: List[GeneratedOutput] = []
+        if response.get("object") == "error":
+            raise Exception(f"BedrockPalmyraClient request failed: {response['message']}")
+        for choice in response["choices"]:
+            assert choice["message"]["role"] == "assistant"
+            output_text = choice["message"]["content"]
+            finish_reason = BedrockPalmyraClient._COMPLETION_REASON_TO_FINISH_REASON.get(
+                choice["finish_reason"], choice["finish_reason"].lower()
+            )
+            completion = truncate_and_tokenize_response_text(
+                output_text, request, self.tokenizer, self.tokenizer_name, finish_reason
+            )
+            completions.append(completion)
+        return completions

helm/clients/cohere_client.py

@@ -3,6 +3,7 @@ import requests
 from typing import List, Optional, Sequence, TypedDict
 
 from helm.common.cache import CacheConfig
+from helm.common.hierarchical_logger import hexception
 from helm.common.optional_dependencies import handle_module_not_found_error
 from helm.common.request import (
     wrap_request_time,
@@ -123,6 +124,7 @@ class CohereClient(CachingClient):
 
             response, cached = self.cache.get(raw_request, wrap_request_time(do_it))
         except (requests.exceptions.RequestException, AssertionError) as e:
+            hexception(e)
             error: str = f"CohereClient error: {e}"
             return RequestResult(success=False, cached=False, error=error, completions=[], embedding=[])
 
@@ -232,6 +234,7 @@ class CohereChatClient(CachingClient):
 
             response, cached = self.cache.get(raw_request, wrap_request_time(do_it))
         except (requests.exceptions.RequestException, AssertionError) as e:
+            hexception(e)
             error: str = f"CohereClient error: {e}"
             return RequestResult(success=False, cached=False, error=error, completions=[], embedding=[])
 

helm/clients/dspy_client.py

@@ -0,0 +1,135 @@
+import hashlib
+import tempfile
+from pathlib import Path
+from typing import Optional
+
+from helm.common.optional_dependencies import handle_module_not_found_error
+
+try:
+    import dspy
+except ModuleNotFoundError as e:
+    handle_module_not_found_error(e, ["dspy"])
+
+from helm.benchmark.runner import _get_current_run_spec_name
+from helm.clients.client import Client
+from helm.common.cache import Cache, CacheConfig
+from helm.common.general import ensure_file_downloaded
+from helm.common.hierarchical_logger import hlog
+from helm.common.request import Request, RequestResult, GeneratedOutput
+from helm.proxy.retry import NonRetriableException
+from helm.tokenizers.tokenizer import Tokenizer
+
+
+class DSPyClient(Client):
+    """
+    A HELM client that uses DSPy for inference instead of directly calling the model.
+    """
+
+    def __init__(
+        self,
+        cache_config: CacheConfig,
+        tokenizer: Tokenizer,
+        tokenizer_name: str,
+        model_name: Optional[str] = None,
+        dspy_agent_url: Optional[str] = None,
+        dspy_module: Optional[str] = None,
+        dspy_api_model: Optional[str] = None,
+        dspy_api_base: Optional[str] = None,
+        api_key: Optional[str] = None,
+    ):
+        """
+        Initializes the DSPyClient.
+
+        Args:
+            cache_config (CacheConfig): Configuration for caching.
+            tokenizer (Tokenizer): Tokenizer instance (unused but required by HELM interface).
+            tokenizer_name (str): Name of the tokenizer (unused but required by HELM interface).
+            model_name (str): The official model name used within HELM.
+            dspy_agent_url (str): URL for the DSPy agent JSON configuration.
+            dspy_module (str): The module to use with DSPy (Predict, ChainOfThought).
+            dspy_api_model (str): The actual model name (API) to use with DSPy.
+            dspy_api_base (str): Base URL for the model API.
+            api_key (str): API key for the DSPy model provider.
+        """
+
+        if not model_name:
+            raise NonRetriableException("Please specify the model name in model_deployments.yaml")
+        if not dspy_api_model:
+            raise NonRetriableException("Please specify dspy_api_model in model_deployments.yaml")
+
+        if ("o3-mini" in model_name) or ("deepseek-r1" in model_name):
+            self.lm = dspy.LM(
+                model=dspy_api_model, api_base=dspy_api_base, api_key=api_key, temperature=1.0, max_tokens=100000
+            )
+        else:
+            self.lm = dspy.LM(model=dspy_api_model, api_base=dspy_api_base, api_key=api_key, temperature=0.0)
+
+        run_spec_name = _get_current_run_spec_name()
+        self.scenario_name = run_spec_name.split(":")[0] if run_spec_name else "unknown"
+        self.model_name = model_name
+        self.dspy_agent_url_template = dspy_agent_url
+        self.dspy_module = dspy_module
+        self.dspy_api_model = dspy_api_model
+        self.dspy_api_base = dspy_api_base
+        self.api_key = api_key
+        self.cache_dir = Path(tempfile.gettempdir()) / "dspy_agent_cache"
+        self.cache_dir.mkdir(exist_ok=True)
+        self._current_cache_file = None
+        self._load_agent_for_scenario()
+        self.cache = Cache(cache_config) if cache_config else None
+
+    def _get_cache_file(self, url):
+        url_hash = hashlib.md5(url.encode()).hexdigest()
+        return self.cache_dir / f"agent_{url_hash}.json"
+
+    def _load_agent_for_scenario(self):
+        if self.dspy_module == "ChainOfThought":
+            self.agent = dspy.ChainOfThought("inputs -> output")
+        elif self.dspy_module == "Predict":
+            self.agent = dspy.Predict("inputs -> output")
+        else:
+            raise ValueError(f"Unknown dspy_module: {self.dspy_module}")
+
+        dspy_agent_url = (
+            self.dspy_agent_url_template.format(scenario=self.scenario_name) if self.dspy_agent_url_template else None
+        )
+        if dspy_agent_url:
+            cache_file = self._get_cache_file(dspy_agent_url)
+            if self._current_cache_file == cache_file:
+                return
+            try:
+                ensure_file_downloaded(source_url=dspy_agent_url, target_path=str(cache_file))
+                self.agent.load(str(cache_file))
+                self._current_cache_file = cache_file
+
+            except Exception as e:
+                raise NonRetriableException(f"Failed to load DSPy agent from URL {dspy_agent_url}: {str(e)}")
+        hlog(
+            f"DSPy client initialized - HELM Model: {self.model_name}, DSPy Model: {self.dspy_api_model}, API Base: {self.dspy_api_base}, API Key: {'***' if self.api_key else None}, DSPy Agent: {dspy_agent_url}, DSPy Module: {self.dspy_module}"
+        )
+
+    def make_request(self, request: Request) -> RequestResult:
+        run_spec_name = _get_current_run_spec_name()
+        current_scenario = run_spec_name.split(":")[0] if run_spec_name else "unknown"
+        if current_scenario != self.scenario_name:
+            self.scenario_name = current_scenario
+            if self.dspy_agent_url_template:
+                self._load_agent_for_scenario()
+            else:
+                hlog(
+                    f"DSPy client initialized - HELM Model: {self.model_name}, DSPy Model: {self.dspy_api_model}, API Base: {self.dspy_api_base}, API Key: {'***' if self.api_key else None}, DSPy Agent: None, DSPy Module: {self.dspy_module}"
+                )
+
+        prompt_text = request.prompt
+        if request.messages:
+            prompt_text = " ".join(msg["content"] for msg in request.messages if msg.get("role") != "system")
+
+        try:
+            with dspy.context(lm=self.lm):
+                prediction = self.agent(inputs=prompt_text)
+            output_text = prediction.output if hasattr(prediction, "output") else str(prediction)
+        except Exception as e:
+            return RequestResult(success=False, cached=False, completions=[], embedding=[], error=str(e))
+
+        output = GeneratedOutput(text=output_text, logprob=0.0, tokens=[])
+        return RequestResult(success=True, cached=False, completions=[output], embedding=[])

helm/clients/google_client.py

@@ -1,6 +1,7 @@
 from typing import List, Dict
 
 from helm.common.cache import CacheConfig
+from helm.common.hierarchical_logger import hexception
 from helm.common.request import Request, RequestResult, GeneratedOutput, Token
 from helm.clients.client import CachingClient, truncate_sequence
 
@@ -44,6 +45,7 @@ class GoogleClient(CachingClient):
             # If results are not cached for a given query, fail fast
             response, cached = self.cache.get(cache_key, fail)
         except RuntimeError as e:
+            hexception(e)
             error: str = f"GoogleClient error: {e}"
             return RequestResult(success=False, cached=False, error=error, completions=[], embedding=[])
 

helm/clients/http_model_client.py

@@ -3,6 +3,7 @@ from dataclasses import asdict
 from typing import Any, Dict
 
 from helm.common.cache import CacheConfig
+from helm.common.hierarchical_logger import hexception
 from helm.common.request import (
     wrap_request_time,
     Request,
@@ -76,5 +77,6 @@ class HTTPModelClient(CachingClient):
                 request_time=response["request_time"],
             )
         except requests.exceptions.RequestException as e:
+            hexception(e)
             error: str = f"Request error: {e}"
             return RequestResult(success=False, cached=False, error=error, completions=[], embedding=[])