crfm-helm 0.5.6__py3-none-any.whl → 0.5.7__py3-none-any.whl

helm/clients/vllm_client.py

@@ -2,7 +2,7 @@ from typing import Any, Dict, Optional
 
 from helm.common.cache import CacheConfig
 from helm.common.request import Request
-from helm.clients.openai_client import OpenAILegacyCompletionsClient
+from helm.clients.openai_client import OpenAIClient, OpenAILegacyCompletionsClient
 from helm.tokenizers.tokenizer import Tokenizer
 
 
@@ -19,6 +19,8 @@ class VLLMClient(OpenAILegacyCompletionsClient):
         tokenizer_name: str,
         cache_config: CacheConfig,
         base_url: Optional[str] = None,
+        vllm_model_name: Optional[str] = None,
+        **kwargs,
     ):
         super().__init__(
             tokenizer=tokenizer,
@@ -27,18 +29,52 @@ class VLLMClient(OpenAILegacyCompletionsClient):
             api_key="EMPTY",
             org_id=None,
             base_url=base_url,
+            openai_model_name=vllm_model_name,
+            **kwargs,
         )
         self.tokenizer = tokenizer
         self.tokenizer_name = tokenizer_name
-
-    def _get_model_for_request(self, request: Request) -> str:
-        # The `model` parameter for vLLM should be the whole model name including the creator organization,
-        # unlike OpenAI which only uses the model engine.
-        return request.model
+        self.vllm_model_name = vllm_model_name
 
     def _to_raw_completion_request(self, request: Request) -> Dict[str, Any]:
         raw_request = super()._to_raw_completion_request(request)
         # This avoids the error: best_of must be 1 when using greedy sampling
-        if "best_of" in raw_request and raw_request["best_of"] > 1:
+        if (
+            "temperature" in raw_request
+            and raw_request["temperature"] == 0.0
+            and "best_of" in raw_request
+            and raw_request["best_of"] > 1
+        ):
             raw_request["best_of"] = 1
         return raw_request
+
+
+class VLLMChatClient(OpenAIClient):
+    """Sends request to a vLLM server using the OpenAI-compatible API.
+
+    Only uses the Chat Completions API.
+
+    See: https://docs.vllm.ai/en/latest/getting_started/quickstart.html#openai-compatible-server"""
+
+    def __init__(
+        self,
+        tokenizer: Tokenizer,
+        tokenizer_name: str,
+        cache_config: CacheConfig,
+        base_url: Optional[str] = None,
+        vllm_model_name: Optional[str] = None,
+        **kwargs,
+    ):
+        super().__init__(
+            tokenizer=tokenizer,
+            tokenizer_name=tokenizer_name,
+            cache_config=cache_config,
+            api_key="EMPTY",
+            org_id=None,
+            base_url=base_url,
+            openai_model_name=vllm_model_name,
+            **kwargs,
+        )
+        self.tokenizer = tokenizer
+        self.tokenizer_name = tokenizer_name
+        self.vllm_model_name = vllm_model_name

helm/clients/vllm_granite_thinking_client.py

@@ -0,0 +1,56 @@
+from dataclasses import replace
+import re
+from typing import Any, Dict, List, Tuple
+
+from helm.clients.vllm_client import VLLMChatClient
+from helm.common.request import GeneratedOutput, Request, RequestResult, Thinking
+
+
+class VLLMGraniteThinkingClient(VLLMChatClient):
+    """Sends request to a Granite model on vLLM server with thinking enabled.
+
+    From vLLM documentation at
+    https://docs.vllm.ai/en/v0.9.1/features/reasoning_outputs.html
+
+    IBM Granite 3.2 reasoning is disabled by default;
+    to enable it, you must also pass thinking=True in your chat_template_kwargs.
+    """
+
+    def _make_chat_raw_request(self, request: Request) -> Dict[str, Any]:
+        raw_request = super()._make_chat_raw_request(request)
+        raw_request["extra_body"] = {"chat_template_kwargs": {"thinking": True}}
+        return raw_request
+
+    def _parse_thinking(self, input: str) -> Tuple[str, str]:
+        """Return a tuple of thinking text and output text."""
+        match = re.match(r"<think>(.*)</think>\s*<response>(.*)</response>", input, re.DOTALL)
+        if match:
+            return (match.group(1), match.group(2))
+
+        match = re.match(r"<think>(.*)</think>\s*<response>(.*)", input, re.DOTALL)
+        if match:
+            return (match.group(1), match.group(2))
+
+        match = re.match(r"<think>(.*)</think>\s*", input, re.DOTALL)
+        if match:
+            return (match.group(1), "")
+
+        match = re.match(r"<think>(.*)", input, re.DOTALL)
+        if match:
+            return (match.group(1), "")
+
+        return (input, "")
+
+    def _make_chat_request(self, request: Request) -> RequestResult:
+        request_result = super()._make_chat_request(request)
+        modified_completions: List[GeneratedOutput] = []
+        for completion in request_result.completions:
+            thinking, modified_text = self._parse_thinking(completion.text)
+            modified_completions.append(
+                replace(
+                    completion,
+                    text=modified_text,
+                    thinking=Thinking(text=thinking),
+                )
+            )
+        return replace(request_result, completions=modified_completions)

helm/common/critique_request.py

@@ -6,7 +6,6 @@ from helm.common.media_object import MediaObject
 class QuestionType:
     """String enum of question types."""
 
-    # TODO: Make this a StrEnum after upgrading to Python 3.11
     MULTIPLE_CHOICE: str = "multiple_choice"
     CHECKBOX: str = "checkbox"
     FREE_RESPONSE: str = "free_response"

helm/common/hierarchical_logger.py

@@ -1,4 +1,7 @@
 import logging
+import logging.config
+import yaml
+import os
 import sys
 import time
 from typing import Any, Callable, List, Optional
@@ -34,22 +37,31 @@ class HierarchicalLogger(object):
     def indent(self) -> str:
         return "  " * len(self.start_times)
 
-    def track_begin(self, x: Any) -> None:
-        self.logger.info(self.indent() + str(x) + " {")
+    def track_begin(self, x: Any, **kwargs) -> None:
+        kwargs["stacklevel"] = kwargs.get("stacklevel", 1) + 1
+        self.logger.info(self.indent() + str(x) + " {", **kwargs)
         sys.stdout.flush()
         self.start_times.append(time.time())
 
-    def track_end(self) -> None:
+    def track_end(self, **kwargs) -> None:
+        kwargs["stacklevel"] = kwargs.get("stacklevel", 1) + 1
         t = time.time() - self.start_times.pop()
-        self.logger.info(self.indent() + "} [%s]" % (format_time(t)))
+        self.logger.info(self.indent() + "} [%s]" % (format_time(t)), **kwargs)
         sys.stdout.flush()
 
-    def log(self, x: Any) -> None:
-        self.logger.info(self.indent() + str(x))
+    def log(self, x: Any, **kwargs) -> None:
+        kwargs["stacklevel"] = kwargs.get("stacklevel", 1) + 1
+        self.logger.info(self.indent() + str(x), **kwargs)
         sys.stdout.flush()
 
-    def warn(self, x: Any) -> None:
-        self.logger.warning(self.indent() + str(x))
+    def debug(self, x: Any, **kwargs) -> None:
+        kwargs["stacklevel"] = kwargs.get("stacklevel", 1) + 1
+        self.logger.debug(self.indent() + str(x), **kwargs)
+        sys.stdout.flush()
+
+    def warn(self, x: Any, **kwargs) -> None:
+        kwargs["stacklevel"] = kwargs.get("stacklevel", 1) + 1
+        self.logger.warning(self.indent() + str(x), **kwargs)
         sys.stdout.flush()
 
 
@@ -69,23 +81,31 @@ singleton = HierarchicalLogger()
 # Exposed public methods
 
 
-def hlog(x: Any) -> None:
-    singleton.log(x)
+def hdebug(x: Any, **kwargs) -> None:
+    kwargs["stacklevel"] = kwargs.get("stacklevel", 1) + 1
+    singleton.debug(x, **kwargs)
+
+
+def hlog(x: Any, **kwargs) -> None:
+    kwargs["stacklevel"] = kwargs.get("stacklevel", 1) + 1
+    singleton.log(x, **kwargs)
 
 
-def hwarn(x: Any) -> None:
-    singleton.warn(x)
+def hwarn(x: Any, **kwargs) -> None:
+    kwargs["stacklevel"] = kwargs.get("stacklevel", 1) + 1
+    singleton.warn(x, **kwargs)
 
 
 class htrack_block:
-    def __init__(self, x: Any) -> None:
+    def __init__(self, x: Any, stacklevel=1) -> None:
+        self._stacklevel = stacklevel + 1
         self.x = x
 
     def __enter__(self) -> None:
-        singleton.track_begin(self.x)
+        singleton.track_begin(self.x, stacklevel=self._stacklevel)
 
     def __exit__(self, tpe: Any, value: Any, callback: Any) -> None:
-        singleton.track_end()
+        singleton.track_end(stacklevel=self._stacklevel)
 
 
 class htrack:
@@ -116,34 +136,63 @@ class htrack:
                     description = description.replace("$" + k, str(v))
             else:
                 description = ""
-            with htrack_block(parent + fn.__name__ + description):
+            with htrack_block(parent + fn.__name__ + description, stacklevel=2):
                 return fn(*args, **kwargs)
 
         return wrapper
 
 
-def setup_default_logging():
+def setup_default_logging(config_path: Optional[str] = None):
     """
-    Setup a default logger to STDOUT for HELM via Python logging
-    """
-    formatter = ColoredFormatter(
-        "%(bold_black)s%(asctime)s%(reset)s %(log_color)s%(levelname)-8s%(reset)s %(message)s",
-        datefmt="%Y-%m-%dT%H:%M:%S",
-        reset=True,
-        log_colors={
-            "DEBUG": "cyan",
-            "INFO": "green",
-            "WARNING": "yellow",
-            "ERROR": "red",
-            "CRITICAL": "red,bg_white",
-        },
-        secondary_log_colors={},
-        style="%",
-    )
+    Setup Python logging for HELM
 
+    Priority:
+    1. External config file (YAML or JSON).
+    2. ENV var LOG_LEVEL.
+    3. a default logger to STDOUT
+    """
     logger = logging.getLogger("helm")
-    logger.setLevel(logging.INFO)
     logger.propagate = False
+
+    if config_path and os.path.exists(config_path):
+        with open(config_path, "r") as f:
+            config = yaml.safe_load(f)
+        logging.config.dictConfig(config)
+        hdebug("setup custom HELM logging")
+        return
+
+    log_level = (os.getenv("HELM_LOG_LEVEL") or os.getenv("LOG_LEVEL") or "INFO").upper()
+    try:
+        logger.setLevel(getattr(logging, log_level))
+    except AttributeError:
+        logger.setLevel(logging.INFO)
+
+    # Set formatter
+    formatter: Optional[logging.Formatter] = None
+    if sys.stdout.isatty():
+        try:
+            formatter = ColoredFormatter(
+                "%(bold_black)s%(asctime)s%(reset)s %(log_color)s%(levelname)-8s%(reset)s %(message)s",
+                datefmt="%Y-%m-%dT%H:%M:%S",
+                reset=True,
+                log_colors={
+                    "DEBUG": "cyan",
+                    "INFO": "green",
+                    "WARNING": "yellow",
+                    "ERROR": "red",
+                    "CRITICAL": "red,bg_white",
+                },
+                style="%",
+            )
+        except ImportError:
+            pass
+
+    if formatter is None:
+        # fallback
+        formatter = logging.Formatter("%(asctime)s %(levelname)-8s %(message)s")
+
+    # Add default stdout handler
     handler = logging.StreamHandler(sys.stdout)
     handler.setFormatter(formatter)
     logger.addHandler(handler)
+    hdebug("setup default HELM logging")

helm/common/object_spec.py

@@ -55,14 +55,23 @@ def inject_object_spec_args(
     This is loosely based on instance (constant) bindings and provider bindings in Guice dependency injection.
 
     Example:
-
-    class MyClass:
-        def __init__(a: int, b: int, c: int, d: int = 0):
-            pass
-
-    old_object_spec = ObjectSpec(class_name="MyClass", args={"a": 11})
-    new_object_spec = inject_object_spec_args(old_object_spec, {"b": 12}, {"c": lambda: 13})
-    # new_object_spec is now ObjectSpec(class_name="MyClass", args={"a": 11, "b": 12, "c": 13})
+        >>> from helm.common.object_spec import *  # NOQA
+        >>> import sys, types
+        >>> # Given a custom class with hashable arguments
+        >>> class MyClass:
+        ...     def __init__(a: int, b: int, c: int, d: int = 0):
+        ...         pass
+        >>> #
+        >>> # <boilerplate>: make a dummy module for MyClass to make this doctest exectuable
+        >>> sys.modules["my_module"] = type("MyModule", (types.ModuleType,), {"MyClass": MyClass})("my_module")
+        >>> # </boilerplate>
+        >>> #
+        >>> # Define new style and old style object specs
+        >>> old_object_spec = ObjectSpec(class_name="my_module.MyClass", args={"a": 11})
+        >>> new_object_spec = inject_object_spec_args(old_object_spec, {"b": 12}, {"c": lambda: 13})
+        >>> # new_object_spec is now
+        >>> print(new_object_spec)
+        ObjectSpec(class_name='my_module.MyClass', args={'a': 11, 'b': 12, 'c': 13})
     """
     cls = get_class_by_name(spec.class_name)
     init_signature = inspect.signature(cls.__init__)
@@ -93,6 +102,12 @@ def parse_object_spec(description: str) -> ObjectSpec:
         <class_name>:<key>=<value>,<key>=<value>
     Usually, the description is something that's succinct and can be typed on the command-line.
     Here, value defaults to string.
+
+    Example:
+        >>> from helm.common.object_spec import *  # NOQA
+        >>> description = 'mscoco:model=huggingface_stable-diffusion-v1-4'
+        >>> parse_object_spec(description)
+        ObjectSpec(class_name='mscoco', args={'model': 'huggingface_stable-diffusion-v1-4'})
     """
 
     def parse_arg(arg: str) -> Tuple[str, Any]:

helm/common/test_logging.py

@@ -0,0 +1,94 @@
+import sys
+import tempfile
+import textwrap
+import pathlib
+from helm.benchmark import run
+from typing import List, Optional
+
+
+class ArgvContext:
+    """
+    Helper to assign a temporary value to sys.argv and then restore it
+    """
+
+    def __init__(self, argv: Optional[List[str]]):
+        self.argv = argv
+        self._original_argv: Optional[List[str]] = None
+
+    def __enter__(self):
+        self._original_argv = sys.argv[:]
+        sys.argv = self.argv or []
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        assert self._original_argv is not None  # Satisfies mypy
+        sys.argv = self._original_argv
+
+
+def test_run_with_custom_logging_config():
+    # Setup temporary directory
+    with tempfile.TemporaryDirectory(prefix="helm_test_") as tmp_dir_str:
+        tmp_dir = pathlib.Path(tmp_dir_str)
+        log_path = tmp_dir / "test.log"
+        log_config_path = tmp_dir / "test_config.yaml"
+
+        # Write custom YAML log config to file
+        log_config_text = textwrap.dedent(
+            f"""
+            version: 1
+            disable_existing_loggers: false
+            formatters:
+              simple:
+                datefmt: '%Y-%m-%dT%H:%M:%S'
+                format: '%(asctime)s %(levelname)s %(name)s %(message)s'
+            handlers:
+              file:
+                class: logging.FileHandler
+                filename: {log_path}
+                formatter: simple
+                level: DEBUG
+                mode: w
+            loggers:
+              helm:
+                handlers:
+                - file
+                level: DEBUG
+                propagate: false
+            """
+        ).strip()
+
+        log_config_path.write_text(log_config_text)
+
+        # Simulate command-line arguments
+        argv = [
+            "run.py",  # Fake script name
+            "--run-entries",
+            "mmlu:subject=philosophy,model=openai/gpt2",
+            "-m",
+            "1",
+            "--suite",
+            "my-suite",
+            "--dry-run",
+            "--log-config",
+            str(log_config_path),
+        ]
+
+        # Call main
+        with ArgvContext(argv):
+            run.main()
+
+        # Check log file contents
+        assert log_path.exists(), "Log file was not created"
+        log_contents = log_path.read_text()
+
+        # Test that log file was written to disk as requested
+        print("Log Contents")
+        print("------------")
+        print(log_contents)
+
+        assert (
+            "mscoco" in log_contents or "huggingface" in log_contents or "dry-run" in log_contents
+        ), "Expected log content not found in log file:\n"
+
+
+if __name__ == "__main__":
+    test_run_with_custom_logging_config()