crfm-helm 0.5.6__py3-none-any.whl → 0.5.8__py3-none-any.whl

helm/clients/test_openrouter_client.py

@@ -0,0 +1,69 @@
+import os
+import pytest
+import tempfile
+
+from helm.common.cache import BlackHoleCacheConfig, SqliteCacheConfig
+from helm.common.request import Request
+from helm.clients.openrouter_client import OpenRouterClient
+
+from helm.tokenizers.huggingface_tokenizer import HuggingFaceTokenizer
+
+
+class TestOpenRouterClient:
+    def setup_method(self, method):
+        cache_file = tempfile.NamedTemporaryFile(delete=False)
+        self.cache_path: str = cache_file.name
+        self.tokenizer_name = "mistralai/Mistral-7B-v0.1"
+        self.tokenizer = HuggingFaceTokenizer(
+            cache_config=BlackHoleCacheConfig(),
+            tokenizer_name=self.tokenizer_name,
+        )
+
+    def teardown_method(self, method):
+        os.remove(self.cache_path)
+
+    @pytest.mark.parametrize(
+        "model_name,test_input,expected_model",
+        [
+            (
+                "mistralai/mistral-medium-3.1",
+                Request(
+                    model="mistralai/mistral-medium-3.1",
+                    model_deployment="openrouter/mistral-medium-3.1",
+                ),
+                "mistralai/mistral-medium-3.1",
+            ),
+            (
+                None,
+                Request(model="openai/gpt-oss-20b:free", model_deployment="openrouter/gpt-oss-20b:free"),
+                "openai/gpt-oss-20b:free",
+            ),
+        ],
+    )
+    def test_get_model_for_request(self, model_name, test_input, expected_model):
+        client = OpenRouterClient(
+            tokenizer_name=self.tokenizer_name,
+            tokenizer=self.tokenizer,
+            cache_config=SqliteCacheConfig(self.cache_path),
+            model_name=model_name,
+            api_key="test_key",
+        )
+        assert client._get_model_for_request(test_input) == expected_model
+
+    def test_api_key_env_var(self, monkeypatch):
+        monkeypatch.setenv("OPENROUTER_API_KEY", "test_key")
+        client = OpenRouterClient(
+            tokenizer_name=self.tokenizer_name,
+            tokenizer=self.tokenizer,
+            cache_config=SqliteCacheConfig(self.cache_path),
+        )
+        assert client.api_key == "test_key"
+
+    def test_api_key_argument(self):
+        client = OpenRouterClient(
+            tokenizer_name=self.tokenizer_name,
+            tokenizer=self.tokenizer,
+            cache_config=BlackHoleCacheConfig(),
+            api_key="explicit_key",
+        )
+        assert client.api_key == "explicit_key"

helm/clients/together_client.py

@@ -25,8 +25,6 @@ except ModuleNotFoundError as e:
 class _RewriteRequestTags:
     """Tags that indicate that the request for the model must be rewritten before sending to Together."""
 
-    # TODO: Convert to StrEnum after upgrading to Python 3.11
-
     ADD_EOS_TOKEN_AS_STOP_SEQUENCE = "ADD_EOS_TOKEN_AS_STOP_SEQUENCE"
     """Indicates that the EOS token should be added as an extra stop sequence.
 
@@ -101,7 +99,20 @@ class JobNotFinishedError(TogetherClientError):
     pass
 
 
-def _parse_thinking(input: str) -> Tuple[str, str]:
+def _parse_thinking_deepseek_r1(input: str) -> Tuple[str, str]:
+    """Return a tuple of thinking text and output text."""
+    match = re.match(r"<think>\n(.*)\n</think>\n{0,2}(.*)", input, re.DOTALL)
+    if match:
+        return (match.group(1), match.group(2))
+
+    match = re.match(r"<think>\n?(.*)", input, re.DOTALL)
+    if match:
+        return (match.group(1), "")
+
+    return (input, "")
+
+
+def _parse_thinking_qwen3(input: str) -> Tuple[str, str]:
     """Return a tuple of thinking text and output text."""
     match = re.match(r"<think>\n(.*)\n</think>\n{0,2}(.*)", input, re.DOTALL)
     if match:
@@ -114,6 +125,31 @@ def _parse_thinking(input: str) -> Tuple[str, str]:
     return (input, "")
 
 
+def _parse_thinking_glm_4_5(input: str) -> Tuple[str, str]:
+    """Return a tuple of thinking text and output text."""
+    match = re.match(r"\n<think>(.*)</think>(.*)", input, re.DOTALL)
+    if match:
+        return (match.group(1), match.group(2))
+
+    match = re.match(r"\n<think>(.*)", input, re.DOTALL)
+    if match:
+        return (match.group(1), "")
+
+    return (input, "")
+
+
+def _parse_thinking(input: str, model_name: str) -> Tuple[str, str]:
+    # TODO: Come up with a more sustainable extensible way of doing this.
+    if "deepseek-r1" in model_name:
+        return _parse_thinking_deepseek_r1(input)
+    elif "qwen3" in model_name:
+        return _parse_thinking_qwen3(input)
+    elif "glm-4.5" in model_name:
+        return _parse_thinking_glm_4_5(input)
+    else:
+        raise Exception(f"No thinking parser available for model {model_name}")
+
+
 class TogetherClient(CachingClient):
     """
     Client for the models where we evaluate offline. Since the queries are handled offline, the `TogetherClient` just
@@ -348,9 +384,8 @@ class TogetherChatClient(CachingClient):
         self._client = Together(api_key=api_key)
         self._together_model = together_model
         self._disable_logprobs = bool(disable_logprobs)
-        # self.output_processor is actually a function, not a class
         self._parse_thinking = bool(parse_thinking)
-
+        # self.output_processor is actually a function, not a class
         self.output_processor: Optional[Callable[[str], str]] = (
             get_class_by_name(output_processor) if output_processor else None
         )
@@ -446,15 +481,15 @@ class TogetherChatClient(CachingClient):
             if self.output_processor:
                 output_text = self.output_processor(output_text)
 
+            thinking: Optional[Thinking] = None
             if self._parse_thinking:
-                thinking_text, output_text = _parse_thinking(output_text)
-                generated_outputs.append(
-                    GeneratedOutput(
-                        text=output_text, logprob=logprob, tokens=tokens, thinking=Thinking(text=thinking_text)
-                    )
-                )
-            else:
-                generated_outputs.append(GeneratedOutput(text=output_text, logprob=logprob, tokens=tokens))
+                thinking_text, output_text = _parse_thinking(output_text, request.model)
+                thinking = Thinking(text=thinking_text)
+            elif hasattr(choice.message, "reasoning_content"):
+                thinking = Thinking(text=choice.message.reasoning_content)
+            generated_outputs.append(
+                GeneratedOutput(text=output_text, logprob=logprob, tokens=tokens, thinking=thinking)
+            )
         return RequestResult(
             success=True,
             cached=cached,

helm/clients/vertexai_client.py

@@ -1,7 +1,7 @@
 import requests
 from abc import ABC, abstractmethod
 from threading import Lock
-from typing import Any, Dict, Mapping, Optional, List, Union
+from typing import Any, Dict, Mapping, Optional, List, Union, cast
 
 from helm.common.cache import CacheConfig
 from helm.common.multimodal_request_utils import get_contents_as_bytes
@@ -107,7 +107,7 @@ class VertexAITextClient(VertexAIClient):
 
     def make_request(self, request: Request) -> RequestResult:
         """Make a request"""
-        parameters = {
+        parameters: Dict[str, Any] = {
             "temperature": request.temperature,
             "max_output_tokens": request.max_tokens,
             "top_k": request.top_k_per_token,
@@ -207,21 +207,23 @@ class VertexAIChatClient(VertexAIClient):
 
     def make_request(self, request: Request) -> RequestResult:
         """Make a request"""
-        contents = [request.prompt]
+        # mypy is unhappy without this cast
+        contents: Union[List[Union[str, Image, Part]], List[Content]] = cast(
+            List[Union[str, Image, Part]], [request.prompt]
+        )
 
         # For the multimodal case, build up the content with the media objects of `request.multimodal_prompt`
         if request.multimodal_prompt is not None:
             return self._make_multimodal_request(request)
 
         if request.messages is not None:
-            contents = []
             role_mapping = {"user": "user", "assistant": "model"}
-            for msg in request.messages:
-                contents.append(
-                    Content(role=role_mapping.get(msg["role"], "user"), parts=[Part.from_text(msg["content"])])
-                )
+            contents = [
+                Content(role=role_mapping.get(msg["role"], "user"), parts=[Part.from_text(msg["content"])])
+                for msg in request.messages
+            ]
 
-        parameters = {
+        parameters: Dict[str, Any] = {
             "temperature": request.temperature,
             "max_output_tokens": request.max_tokens,
             "top_k": request.top_k_per_token,
@@ -274,8 +276,14 @@ class VertexAIChatClient(VertexAIClient):
                     if not candidate.content:
                         raise VertexAIContentBlockedError(f"No content in candidate: {candidate}")
                     if not candidate.content.parts:
-                        raise VertexAIContentBlockedError(f"No content parts in candidate: {candidate}")
-                    predictions.append({"text": candidate.content.text})
+                        if candidate.finish_reason == 2:  # MAX_TOKENS
+                            # This means that there is no text output because the maximum number of tokens were
+                            # reached during thinking.
+                            predictions.append({"text": ""})
+                        else:
+                            raise VertexAIContentBlockedError(f"No content parts in candidate: {candidate}")
+                    else:
+                        predictions.append({"text": candidate.content.text})
                     # TODO: Extract more information from the response
                 return {"predictions": predictions}
 

helm/clients/vllm_client.py

@@ -2,7 +2,7 @@ from typing import Any, Dict, Optional
 
 from helm.common.cache import CacheConfig
 from helm.common.request import Request
-from helm.clients.openai_client import OpenAILegacyCompletionsClient
+from helm.clients.openai_client import OpenAIClient, OpenAILegacyCompletionsClient
 from helm.tokenizers.tokenizer import Tokenizer
 
 
@@ -19,6 +19,8 @@ class VLLMClient(OpenAILegacyCompletionsClient):
         tokenizer_name: str,
         cache_config: CacheConfig,
         base_url: Optional[str] = None,
+        vllm_model_name: Optional[str] = None,
+        **kwargs,
     ):
         super().__init__(
             tokenizer=tokenizer,
@@ -27,18 +29,52 @@ class VLLMClient(OpenAILegacyCompletionsClient):
             api_key="EMPTY",
             org_id=None,
             base_url=base_url,
+            openai_model_name=vllm_model_name,
+            **kwargs,
         )
         self.tokenizer = tokenizer
         self.tokenizer_name = tokenizer_name
-
-    def _get_model_for_request(self, request: Request) -> str:
-        # The `model` parameter for vLLM should be the whole model name including the creator organization,
-        # unlike OpenAI which only uses the model engine.
-        return request.model
+        self.vllm_model_name = vllm_model_name
 
     def _to_raw_completion_request(self, request: Request) -> Dict[str, Any]:
         raw_request = super()._to_raw_completion_request(request)
         # This avoids the error: best_of must be 1 when using greedy sampling
-        if "best_of" in raw_request and raw_request["best_of"] > 1:
+        if (
+            "temperature" in raw_request
+            and raw_request["temperature"] == 0.0
+            and "best_of" in raw_request
+            and raw_request["best_of"] > 1
+        ):
             raw_request["best_of"] = 1
         return raw_request
+
+
+class VLLMChatClient(OpenAIClient):
+    """Sends request to a vLLM server using the OpenAI-compatible API.
+
+    Only uses the Chat Completions API.
+
+    See: https://docs.vllm.ai/en/latest/getting_started/quickstart.html#openai-compatible-server"""
+
+    def __init__(
+        self,
+        tokenizer: Tokenizer,
+        tokenizer_name: str,
+        cache_config: CacheConfig,
+        base_url: Optional[str] = None,
+        vllm_model_name: Optional[str] = None,
+        **kwargs,
+    ):
+        super().__init__(
+            tokenizer=tokenizer,
+            tokenizer_name=tokenizer_name,
+            cache_config=cache_config,
+            api_key="EMPTY",
+            org_id=None,
+            base_url=base_url,
+            openai_model_name=vllm_model_name,
+            **kwargs,
+        )
+        self.tokenizer = tokenizer
+        self.tokenizer_name = tokenizer_name
+        self.vllm_model_name = vllm_model_name

helm/clients/vllm_granite_thinking_client.py

@@ -0,0 +1,56 @@
+from dataclasses import replace
+import re
+from typing import Any, Dict, List, Tuple
+
+from helm.clients.vllm_client import VLLMChatClient
+from helm.common.request import GeneratedOutput, Request, RequestResult, Thinking
+
+
+class VLLMGraniteThinkingClient(VLLMChatClient):
+    """Sends request to a Granite model on vLLM server with thinking enabled.
+
+    From vLLM documentation at
+    https://docs.vllm.ai/en/v0.9.1/features/reasoning_outputs.html
+
+    IBM Granite 3.2 reasoning is disabled by default;
+    to enable it, you must also pass thinking=True in your chat_template_kwargs.
+    """
+
+    def _make_chat_raw_request(self, request: Request) -> Dict[str, Any]:
+        raw_request = super()._make_chat_raw_request(request)
+        raw_request["extra_body"] = {"chat_template_kwargs": {"thinking": True}}
+        return raw_request
+
+    def _parse_thinking(self, input: str) -> Tuple[str, str]:
+        """Return a tuple of thinking text and output text."""
+        match = re.match(r"<think>(.*)</think>\s*<response>(.*)</response>", input, re.DOTALL)
+        if match:
+            return (match.group(1), match.group(2))
+
+        match = re.match(r"<think>(.*)</think>\s*<response>(.*)", input, re.DOTALL)
+        if match:
+            return (match.group(1), match.group(2))
+
+        match = re.match(r"<think>(.*)</think>\s*", input, re.DOTALL)
+        if match:
+            return (match.group(1), "")
+
+        match = re.match(r"<think>(.*)", input, re.DOTALL)
+        if match:
+            return (match.group(1), "")
+
+        return (input, "")
+
+    def _make_chat_request(self, request: Request) -> RequestResult:
+        request_result = super()._make_chat_request(request)
+        modified_completions: List[GeneratedOutput] = []
+        for completion in request_result.completions:
+            thinking, modified_text = self._parse_thinking(completion.text)
+            modified_completions.append(
+                replace(
+                    completion,
+                    text=modified_text,
+                    thinking=Thinking(text=thinking),
+                )
+            )
+        return replace(request_result, completions=modified_completions)

helm/common/critique_request.py

@@ -6,7 +6,6 @@ from helm.common.media_object import MediaObject
 class QuestionType:
     """String enum of question types."""
 
-    # TODO: Make this a StrEnum after upgrading to Python 3.11
     MULTIPLE_CHOICE: str = "multiple_choice"
     CHECKBOX: str = "checkbox"
     FREE_RESPONSE: str = "free_response"

helm/common/hierarchical_logger.py

@@ -1,4 +1,7 @@
 import logging
+import logging.config
+import yaml
+import os
 import sys
 import time
 from typing import Any, Callable, List, Optional
@@ -34,22 +37,31 @@ class HierarchicalLogger(object):
     def indent(self) -> str:
         return "  " * len(self.start_times)
 
-    def track_begin(self, x: Any) -> None:
-        self.logger.info(self.indent() + str(x) + " {")
+    def track_begin(self, x: Any, **kwargs) -> None:
+        kwargs["stacklevel"] = kwargs.get("stacklevel", 1) + 1
+        self.logger.info(self.indent() + str(x) + " {", **kwargs)
         sys.stdout.flush()
         self.start_times.append(time.time())
 
-    def track_end(self) -> None:
+    def track_end(self, **kwargs) -> None:
+        kwargs["stacklevel"] = kwargs.get("stacklevel", 1) + 1
         t = time.time() - self.start_times.pop()
-        self.logger.info(self.indent() + "} [%s]" % (format_time(t)))
+        self.logger.info(self.indent() + "} [%s]" % (format_time(t)), **kwargs)
         sys.stdout.flush()
 
-    def log(self, x: Any) -> None:
-        self.logger.info(self.indent() + str(x))
+    def log(self, x: Any, **kwargs) -> None:
+        kwargs["stacklevel"] = kwargs.get("stacklevel", 1) + 1
+        self.logger.info(self.indent() + str(x), **kwargs)
         sys.stdout.flush()
 
-    def warn(self, x: Any) -> None:
-        self.logger.warning(self.indent() + str(x))
+    def debug(self, x: Any, **kwargs) -> None:
+        kwargs["stacklevel"] = kwargs.get("stacklevel", 1) + 1
+        self.logger.debug(self.indent() + str(x), **kwargs)
+        sys.stdout.flush()
+
+    def warn(self, x: Any, **kwargs) -> None:
+        kwargs["stacklevel"] = kwargs.get("stacklevel", 1) + 1
+        self.logger.warning(self.indent() + str(x), **kwargs)
         sys.stdout.flush()
 
 
@@ -69,23 +81,31 @@ singleton = HierarchicalLogger()
 # Exposed public methods
 
 
-def hlog(x: Any) -> None:
-    singleton.log(x)
+def hdebug(x: Any, **kwargs) -> None:
+    kwargs["stacklevel"] = kwargs.get("stacklevel", 1) + 1
+    singleton.debug(x, **kwargs)
+
+
+def hlog(x: Any, **kwargs) -> None:
+    kwargs["stacklevel"] = kwargs.get("stacklevel", 1) + 1
+    singleton.log(x, **kwargs)
 
 
-def hwarn(x: Any) -> None:
-    singleton.warn(x)
+def hwarn(x: Any, **kwargs) -> None:
+    kwargs["stacklevel"] = kwargs.get("stacklevel", 1) + 1
+    singleton.warn(x, **kwargs)
 
 
 class htrack_block:
-    def __init__(self, x: Any) -> None:
+    def __init__(self, x: Any, stacklevel=1) -> None:
+        self._stacklevel = stacklevel + 1
         self.x = x
 
     def __enter__(self) -> None:
-        singleton.track_begin(self.x)
+        singleton.track_begin(self.x, stacklevel=self._stacklevel)
 
     def __exit__(self, tpe: Any, value: Any, callback: Any) -> None:
-        singleton.track_end()
+        singleton.track_end(stacklevel=self._stacklevel)
 
 
 class htrack:
@@ -116,34 +136,63 @@ class htrack:
                     description = description.replace("$" + k, str(v))
             else:
                 description = ""
-            with htrack_block(parent + fn.__name__ + description):
+            with htrack_block(parent + fn.__name__ + description, stacklevel=2):
                 return fn(*args, **kwargs)
 
         return wrapper
 
 
-def setup_default_logging():
+def setup_default_logging(config_path: Optional[str] = None):
     """
-    Setup a default logger to STDOUT for HELM via Python logging
-    """
-    formatter = ColoredFormatter(
-        "%(bold_black)s%(asctime)s%(reset)s %(log_color)s%(levelname)-8s%(reset)s %(message)s",
-        datefmt="%Y-%m-%dT%H:%M:%S",
-        reset=True,
-        log_colors={
-            "DEBUG": "cyan",
-            "INFO": "green",
-            "WARNING": "yellow",
-            "ERROR": "red",
-            "CRITICAL": "red,bg_white",
-        },
-        secondary_log_colors={},
-        style="%",
-    )
+    Setup Python logging for HELM
 
+    Priority:
+    1. External config file (YAML or JSON).
+    2. ENV var LOG_LEVEL.
+    3. a default logger to STDOUT
+    """
     logger = logging.getLogger("helm")
-    logger.setLevel(logging.INFO)
     logger.propagate = False
+
+    if config_path and os.path.exists(config_path):
+        with open(config_path, "r") as f:
+            config = yaml.safe_load(f)
+        logging.config.dictConfig(config)
+        hdebug("setup custom HELM logging")
+        return
+
+    log_level = (os.getenv("HELM_LOG_LEVEL") or os.getenv("LOG_LEVEL") or "INFO").upper()
+    try:
+        logger.setLevel(getattr(logging, log_level))
+    except AttributeError:
+        logger.setLevel(logging.INFO)
+
+    # Set formatter
+    formatter: Optional[logging.Formatter] = None
+    if sys.stdout.isatty():
+        try:
+            formatter = ColoredFormatter(
+                "%(bold_black)s%(asctime)s%(reset)s %(log_color)s%(levelname)-8s%(reset)s %(message)s",
+                datefmt="%Y-%m-%dT%H:%M:%S",
+                reset=True,
+                log_colors={
+                    "DEBUG": "cyan",
+                    "INFO": "green",
+                    "WARNING": "yellow",
+                    "ERROR": "red",
+                    "CRITICAL": "red,bg_white",
+                },
+                style="%",
+            )
+        except ImportError:
+            pass
+
+    if formatter is None:
+        # fallback
+        formatter = logging.Formatter("%(asctime)s %(levelname)-8s %(message)s")
+
+    # Add default stdout handler
     handler = logging.StreamHandler(sys.stdout)
     handler.setFormatter(formatter)
     logger.addHandler(handler)
+    hdebug("setup default HELM logging")

helm/common/object_spec.py

@@ -55,14 +55,23 @@ def inject_object_spec_args(
     This is loosely based on instance (constant) bindings and provider bindings in Guice dependency injection.
 
     Example:
-
-    class MyClass:
-        def __init__(a: int, b: int, c: int, d: int = 0):
-            pass
-
-    old_object_spec = ObjectSpec(class_name="MyClass", args={"a": 11})
-    new_object_spec = inject_object_spec_args(old_object_spec, {"b": 12}, {"c": lambda: 13})
-    # new_object_spec is now ObjectSpec(class_name="MyClass", args={"a": 11, "b": 12, "c": 13})
+        >>> from helm.common.object_spec import *  # NOQA
+        >>> import sys, types
+        >>> # Given a custom class with hashable arguments
+        >>> class MyClass:
+        ...     def __init__(a: int, b: int, c: int, d: int = 0):
+        ...         pass
+        >>> #
+        >>> # <boilerplate>: make a dummy module for MyClass to make this doctest exectuable
+        >>> sys.modules["my_module"] = type("MyModule", (types.ModuleType,), {"MyClass": MyClass})("my_module")
+        >>> # </boilerplate>
+        >>> #
+        >>> # Define new style and old style object specs
+        >>> old_object_spec = ObjectSpec(class_name="my_module.MyClass", args={"a": 11})
+        >>> new_object_spec = inject_object_spec_args(old_object_spec, {"b": 12}, {"c": lambda: 13})
+        >>> # new_object_spec is now
+        >>> print(new_object_spec)
+        ObjectSpec(class_name='my_module.MyClass', args={'a': 11, 'b': 12, 'c': 13})
     """
     cls = get_class_by_name(spec.class_name)
     init_signature = inspect.signature(cls.__init__)
@@ -93,6 +102,12 @@ def parse_object_spec(description: str) -> ObjectSpec:
         <class_name>:<key>=<value>,<key>=<value>
     Usually, the description is something that's succinct and can be typed on the command-line.
     Here, value defaults to string.
+
+    Example:
+        >>> from helm.common.object_spec import *  # NOQA
+        >>> description = 'mscoco:model=huggingface_stable-diffusion-v1-4'
+        >>> parse_object_spec(description)
+        ObjectSpec(class_name='mscoco', args={'model': 'huggingface_stable-diffusion-v1-4'})
     """
 
     def parse_arg(arg: str) -> Tuple[str, Any]: