crfm-helm 0.5.5__py3-none-any.whl → 0.5.7__py3-none-any.whl

helm/common/general.py

@@ -42,6 +42,13 @@ def ensure_directory_exists(path: str):
     os.makedirs(path, exist_ok=True)
 
 
+def check_file_exists(path: str, msg: Optional[str] = None):
+    """Checks that `path` exists, raises FileNotFoundError if it doesn't."""
+    if not os.path.exists(path):
+        error_msg = msg if msg else f"Required file not found: {path}"
+        raise FileNotFoundError(error_msg)
+
+
 def parse_hocon(text: str):
     """Parse `text` (in HOCON format) into a dict-like object."""
     return pyhocon.ConfigFactory.parse_string(text)
@@ -156,7 +163,7 @@ def format_split(split: str) -> str:
 
 
 def asdict_without_nones(obj: Any) -> Dict[str, Any]:
-    if not is_dataclass(obj):
+    if not is_dataclass(obj) or isinstance(obj, type):
         raise ValueError(f"Expected dataclass, got '{obj}'")
     return asdict(obj, dict_factory=lambda x: {k: v for (k, v) in x if v is not None})
 
@@ -178,7 +185,7 @@ def binarize_dict(d: Dict[str, int]) -> Dict[str, int]:
 
 def serialize(obj: Any) -> List[str]:
     """Takes in a dataclass and outputs all of its fields and values in a list."""
-    if not is_dataclass(obj):
+    if not is_dataclass(obj) or isinstance(obj, type):
         raise ValueError(f"Expected dataclass, got '{obj}'")
     return [f"{key}: {json.dumps(value)}" for key, value in asdict(obj).items()]
 

helm/common/hierarchical_logger.py

@@ -1,6 +1,11 @@
+import logging
+import logging.config
+import yaml
+import os
 import sys
 import time
 from typing import Any, Callable, List, Optional
+from colorlog import ColoredFormatter
 
 
 class HierarchicalLogger(object):
@@ -20,24 +25,43 @@ class HierarchicalLogger(object):
         } [0s]
     """
 
+    # Far too much effort to unwind every call to hlog to go via logging,
+    # And is a terrible idea to inspect the stack every time hlog is called
+    # to figure out the caller,
+    # So just log everything under "helm".
+    logger = logging.getLogger("helm")
+
     def __init__(self) -> None:
         self.start_times: List[float] = []
 
     def indent(self) -> str:
         return "  " * len(self.start_times)
 
-    def track_begin(self, x: Any) -> None:
-        print(self.indent() + str(x) + " {")
+    def track_begin(self, x: Any, **kwargs) -> None:
+        kwargs["stacklevel"] = kwargs.get("stacklevel", 1) + 1
+        self.logger.info(self.indent() + str(x) + " {", **kwargs)
         sys.stdout.flush()
         self.start_times.append(time.time())
 
-    def track_end(self) -> None:
+    def track_end(self, **kwargs) -> None:
+        kwargs["stacklevel"] = kwargs.get("stacklevel", 1) + 1
         t = time.time() - self.start_times.pop()
-        print(self.indent() + "} [%s]" % (format_time(t)))
+        self.logger.info(self.indent() + "} [%s]" % (format_time(t)), **kwargs)
+        sys.stdout.flush()
+
+    def log(self, x: Any, **kwargs) -> None:
+        kwargs["stacklevel"] = kwargs.get("stacklevel", 1) + 1
+        self.logger.info(self.indent() + str(x), **kwargs)
         sys.stdout.flush()
 
-    def log(self, x: Any) -> None:
-        print(self.indent() + str(x))
+    def debug(self, x: Any, **kwargs) -> None:
+        kwargs["stacklevel"] = kwargs.get("stacklevel", 1) + 1
+        self.logger.debug(self.indent() + str(x), **kwargs)
+        sys.stdout.flush()
+
+    def warn(self, x: Any, **kwargs) -> None:
+        kwargs["stacklevel"] = kwargs.get("stacklevel", 1) + 1
+        self.logger.warning(self.indent() + str(x), **kwargs)
         sys.stdout.flush()
 
 
@@ -57,19 +81,31 @@ singleton = HierarchicalLogger()
 # Exposed public methods
 
 
-def hlog(x: Any) -> None:
-    singleton.log(x)
+def hdebug(x: Any, **kwargs) -> None:
+    kwargs["stacklevel"] = kwargs.get("stacklevel", 1) + 1
+    singleton.debug(x, **kwargs)
+
+
+def hlog(x: Any, **kwargs) -> None:
+    kwargs["stacklevel"] = kwargs.get("stacklevel", 1) + 1
+    singleton.log(x, **kwargs)
+
+
+def hwarn(x: Any, **kwargs) -> None:
+    kwargs["stacklevel"] = kwargs.get("stacklevel", 1) + 1
+    singleton.warn(x, **kwargs)
 
 
 class htrack_block:
-    def __init__(self, x: Any) -> None:
+    def __init__(self, x: Any, stacklevel=1) -> None:
+        self._stacklevel = stacklevel + 1
         self.x = x
 
     def __enter__(self) -> None:
-        singleton.track_begin(self.x)
+        singleton.track_begin(self.x, stacklevel=self._stacklevel)
 
     def __exit__(self, tpe: Any, value: Any, callback: Any) -> None:
-        singleton.track_end()
+        singleton.track_end(stacklevel=self._stacklevel)
 
 
 class htrack:
@@ -100,7 +136,63 @@ class htrack:
                     description = description.replace("$" + k, str(v))
             else:
                 description = ""
-            with htrack_block(parent + fn.__name__ + description):
+            with htrack_block(parent + fn.__name__ + description, stacklevel=2):
                 return fn(*args, **kwargs)
 
         return wrapper
+
+
+def setup_default_logging(config_path: Optional[str] = None):
+    """
+    Setup Python logging for HELM
+
+    Priority:
+    1. External config file (YAML or JSON).
+    2. ENV var LOG_LEVEL.
+    3. a default logger to STDOUT
+    """
+    logger = logging.getLogger("helm")
+    logger.propagate = False
+
+    if config_path and os.path.exists(config_path):
+        with open(config_path, "r") as f:
+            config = yaml.safe_load(f)
+        logging.config.dictConfig(config)
+        hdebug("setup custom HELM logging")
+        return
+
+    log_level = (os.getenv("HELM_LOG_LEVEL") or os.getenv("LOG_LEVEL") or "INFO").upper()
+    try:
+        logger.setLevel(getattr(logging, log_level))
+    except AttributeError:
+        logger.setLevel(logging.INFO)
+
+    # Set formatter
+    formatter: Optional[logging.Formatter] = None
+    if sys.stdout.isatty():
+        try:
+            formatter = ColoredFormatter(
+                "%(bold_black)s%(asctime)s%(reset)s %(log_color)s%(levelname)-8s%(reset)s %(message)s",
+                datefmt="%Y-%m-%dT%H:%M:%S",
+                reset=True,
+                log_colors={
+                    "DEBUG": "cyan",
+                    "INFO": "green",
+                    "WARNING": "yellow",
+                    "ERROR": "red",
+                    "CRITICAL": "red,bg_white",
+                },
+                style="%",
+            )
+        except ImportError:
+            pass
+
+    if formatter is None:
+        # fallback
+        formatter = logging.Formatter("%(asctime)s %(levelname)-8s %(message)s")
+
+    # Add default stdout handler
+    handler = logging.StreamHandler(sys.stdout)
+    handler.setFormatter(formatter)
+    logger.addHandler(handler)
+    hdebug("setup default HELM logging")

helm/common/local_context.py

@@ -0,0 +1,140 @@
+import dataclasses
+import os
+from typing import Optional
+
+from helm.common.context import Context
+from helm.common.cache import CacheConfig
+from helm.common.cache_backend_config import CacheBackendConfig, BlackHoleCacheBackendConfig
+from helm.common.critique_request import CritiqueRequest, CritiqueRequestResult
+from helm.common.moderations_api_request import ModerationAPIRequest, ModerationAPIRequestResult
+from helm.common.clip_score_request import CLIPScoreRequest, CLIPScoreResult
+from helm.common.nudity_check_request import NudityCheckRequest, NudityCheckResult
+from helm.common.file_upload_request import FileUploadRequest, FileUploadResult
+from helm.common.general import ensure_directory_exists, parse_hocon, get_credentials
+from helm.common.perspective_api_request import PerspectiveAPIRequest, PerspectiveAPIRequestResult
+from helm.common.tokenization_request import (
+    TokenizationRequest,
+    TokenizationRequestResult,
+    DecodeRequest,
+    DecodeRequestResult,
+)
+from helm.common.request import Request, RequestResult
+from helm.clients.auto_client import AutoClient
+from helm.clients.moderation_api_client import ModerationAPIClient
+from helm.clients.image_generation.nudity_check_client import NudityCheckClient
+from helm.clients.gcs_client import GCSClient
+from helm.clients.clip_score_client import CLIPScoreClient
+from helm.clients.toxicity_classifier_client import ToxicityClassifierClient
+from helm.proxy.example_queries import example_queries
+from helm.benchmark.model_metadata_registry import ALL_MODELS_METADATA
+from helm.proxy.query import Query, QueryResult
+from helm.proxy.retry import retry_request
+from helm.tokenizers.auto_tokenizer import AutoTokenizer
+from helm.proxy.services.service import (
+    CACHE_DIR,
+    GeneralInfo,
+    VERSION,
+    expand_environments,
+    synthesize_request,
+)
+
+
+class LocalContext(Context):
+    """
+    Main class that supports various functionality for the server.
+    """
+
+    def __init__(
+        self,
+        base_path: str = "prod_env",
+        cache_backend_config: CacheBackendConfig = BlackHoleCacheBackendConfig(),
+    ):
+        ensure_directory_exists(base_path)
+        client_file_storage_path = os.path.join(base_path, CACHE_DIR)
+        ensure_directory_exists(client_file_storage_path)
+
+        credentials = get_credentials(base_path)
+
+        self.cache_backend_config = cache_backend_config
+        self.client = AutoClient(credentials, client_file_storage_path, cache_backend_config)
+        self.tokenizer = AutoTokenizer(credentials, cache_backend_config)
+
+        # Lazily instantiate the following clients
+        self.moderation_api_client: Optional[ModerationAPIClient] = None
+        self.toxicity_classifier_client: Optional[ToxicityClassifierClient] = None
+        self.perspective_api_client: Optional[ToxicityClassifierClient] = None
+        self.nudity_check_client: Optional[NudityCheckClient] = None
+        self.clip_score_client: Optional[CLIPScoreClient] = None
+        self.gcs_client: Optional[GCSClient] = None
+
+    def get_general_info(self) -> GeneralInfo:
+        # Can't send release_dates in ModelMetadata bacause dates cannot be round-tripped to and from JSON easily.
+        # TODO(#2158): Either fix this or delete get_general_info.
+        all_models = [dataclasses.replace(model_metadata, release_date=None) for model_metadata in ALL_MODELS_METADATA]
+        return GeneralInfo(version=VERSION, example_queries=example_queries, all_models=all_models)
+
+    def expand_query(self, query: Query) -> QueryResult:
+        """Turn the `query` into requests."""
+        prompt = query.prompt
+        settings = query.settings
+        environments = parse_hocon(query.environments)
+        requests = []
+        for environment in expand_environments(environments):
+            request = synthesize_request(prompt, settings, environment)
+            requests.append(request)
+        return QueryResult(requests=requests)
+
+    def make_request(self, request: Request) -> RequestResult:
+        """Actually make a request to an API."""
+        return self.client.make_request(request)
+
+    def tokenize(self, request: TokenizationRequest) -> TokenizationRequestResult:
+        return self.tokenizer.tokenize(request)
+
+    def decode(self, request: DecodeRequest) -> DecodeRequestResult:
+        return self.tokenizer.decode(request)
+
+    def upload(self, request: FileUploadRequest) -> FileUploadResult:
+        if not self.gcs_client:
+            self.gcs_client = self.client.get_gcs_client()
+
+        assert self.gcs_client
+        return self.gcs_client.upload(request)
+
+    def check_nudity(self, request: NudityCheckRequest) -> NudityCheckResult:
+        if not self.nudity_check_client:
+            self.nudity_check_client = self.client.get_nudity_check_client()
+
+        assert self.nudity_check_client
+        return self.nudity_check_client.check_nudity(request)
+
+    def compute_clip_score(self, request: CLIPScoreRequest) -> CLIPScoreResult:
+        if not self.clip_score_client:
+            self.clip_score_client = self.client.get_clip_score_client()
+
+        assert self.clip_score_client
+        return self.clip_score_client.compute_score(request)
+
+    def get_toxicity_scores(self, request: PerspectiveAPIRequest) -> PerspectiveAPIRequestResult:
+        @retry_request
+        def get_toxicity_scores_with_retry(request: PerspectiveAPIRequest) -> PerspectiveAPIRequestResult:
+            if not self.toxicity_classifier_client:
+                self.toxicity_classifier_client = self.client.get_toxicity_classifier_client()
+            return self.toxicity_classifier_client.get_toxicity_scores(request)
+
+        return get_toxicity_scores_with_retry(request)
+
+    def get_moderation_results(self, request: ModerationAPIRequest) -> ModerationAPIRequestResult:
+        @retry_request
+        def get_moderation_results_with_retry(request: ModerationAPIRequest) -> ModerationAPIRequestResult:
+            if not self.moderation_api_client:
+                self.moderation_api_client = self.client.get_moderation_api_client()
+            return self.moderation_api_client.get_moderation_results(request)
+
+        return get_moderation_results_with_retry(request)
+
+    def make_critique_request(self, request: CritiqueRequest) -> CritiqueRequestResult:
+        return self.client.get_critique_client().make_critique_request(request)
+
+    def get_cache_config(self, shard_name: str) -> CacheConfig:
+        return self.cache_backend_config.get_cache_config(shard_name)

helm/common/object_spec.py

@@ -55,14 +55,23 @@ def inject_object_spec_args(
     This is loosely based on instance (constant) bindings and provider bindings in Guice dependency injection.
 
     Example:
-
-    class MyClass:
-        def __init__(a: int, b: int, c: int, d: int = 0):
-            pass
-
-    old_object_spec = ObjectSpec(class_name="MyClass", args={"a": 11})
-    new_object_spec = inject_object_spec_args(old_object_spec, {"b": 12}, {"c": lambda: 13})
-    # new_object_spec is now ObjectSpec(class_name="MyClass", args={"a": 11, "b": 12, "c": 13})
+        >>> from helm.common.object_spec import *  # NOQA
+        >>> import sys, types
+        >>> # Given a custom class with hashable arguments
+        >>> class MyClass:
+        ...     def __init__(a: int, b: int, c: int, d: int = 0):
+        ...         pass
+        >>> #
+        >>> # <boilerplate>: make a dummy module for MyClass to make this doctest exectuable
+        >>> sys.modules["my_module"] = type("MyModule", (types.ModuleType,), {"MyClass": MyClass})("my_module")
+        >>> # </boilerplate>
+        >>> #
+        >>> # Define new style and old style object specs
+        >>> old_object_spec = ObjectSpec(class_name="my_module.MyClass", args={"a": 11})
+        >>> new_object_spec = inject_object_spec_args(old_object_spec, {"b": 12}, {"c": lambda: 13})
+        >>> # new_object_spec is now
+        >>> print(new_object_spec)
+        ObjectSpec(class_name='my_module.MyClass', args={'a': 11, 'b': 12, 'c': 13})
     """
     cls = get_class_by_name(spec.class_name)
     init_signature = inspect.signature(cls.__init__)
@@ -93,6 +102,12 @@ def parse_object_spec(description: str) -> ObjectSpec:
         <class_name>:<key>=<value>,<key>=<value>
     Usually, the description is something that's succinct and can be typed on the command-line.
     Here, value defaults to string.
+
+    Example:
+        >>> from helm.common.object_spec import *  # NOQA
+        >>> description = 'mscoco:model=huggingface_stable-diffusion-v1-4'
+        >>> parse_object_spec(description)
+        ObjectSpec(class_name='mscoco', args={'model': 'huggingface_stable-diffusion-v1-4'})
     """
 
     def parse_arg(arg: str) -> Tuple[str, Any]:

helm/common/remote_context.py

@@ -0,0 +1,61 @@
+from helm.common.context import Context
+from helm.common.cache import CacheConfig
+from helm.common.authentication import Authentication
+from helm.common.moderations_api_request import ModerationAPIRequest, ModerationAPIRequestResult
+from helm.common.critique_request import CritiqueRequest, CritiqueRequestResult
+from helm.common.nudity_check_request import NudityCheckRequest, NudityCheckResult
+from helm.common.file_upload_request import FileUploadRequest, FileUploadResult
+from helm.common.perspective_api_request import PerspectiveAPIRequest, PerspectiveAPIRequestResult
+from helm.common.clip_score_request import CLIPScoreRequest, CLIPScoreResult
+from helm.common.tokenization_request import (
+    TokenizationRequest,
+    TokenizationRequestResult,
+    DecodeRequestResult,
+    DecodeRequest,
+)
+from helm.common.request import Request, RequestResult
+from helm.proxy.query import Query, QueryResult
+from helm.proxy.services.remote_service import RemoteService
+from helm.proxy.services.service import GeneralInfo, Service
+
+
+class RemoteContext(Context):
+    def __init__(self, base_url: str, auth: Authentication):
+        self.service: Service = RemoteService(base_url)
+        self.auth = auth
+
+    def get_general_info(self) -> GeneralInfo:
+        return self.service.get_general_info()
+
+    def expand_query(self, query: Query) -> QueryResult:
+        return self.service.expand_query(query)
+
+    def make_request(self, request: Request) -> RequestResult:
+        return self.service.make_request(self.auth, request)
+
+    def tokenize(self, request: TokenizationRequest) -> TokenizationRequestResult:
+        return self.service.tokenize(self.auth, request)
+
+    def decode(self, request: DecodeRequest) -> DecodeRequestResult:
+        return self.service.decode(self.auth, request)
+
+    def upload(self, request: FileUploadRequest) -> FileUploadResult:
+        return self.service.upload(self.auth, request)
+
+    def check_nudity(self, request: NudityCheckRequest) -> NudityCheckResult:
+        return self.service.check_nudity(self.auth, request)
+
+    def compute_clip_score(self, request: CLIPScoreRequest) -> CLIPScoreResult:
+        return self.service.compute_clip_score(self.auth, request)
+
+    def get_toxicity_scores(self, request: PerspectiveAPIRequest) -> PerspectiveAPIRequestResult:
+        return self.service.get_toxicity_scores(self.auth, request)
+
+    def get_moderation_results(self, request: ModerationAPIRequest) -> ModerationAPIRequestResult:
+        return self.service.get_moderation_results(self.auth, request)
+
+    def make_critique_request(self, request: CritiqueRequest) -> CritiqueRequestResult:
+        return self.service.make_critique_request(self.auth, request)
+
+    def get_cache_config(self, shard_name: str) -> CacheConfig:
+        return self.service.get_cache_config(shard_name)

helm/common/request.py

@@ -131,6 +131,11 @@ class Token:
         ]
 
 
+@dataclass(frozen=True)
+class Thinking:
+    text: Optional[str] = None
+
+
 @dataclass(frozen=True)
 class GeneratedOutput:
     """A `GeneratedOutput` is a single generated output that may contain text or multimodal content."""
@@ -150,6 +155,9 @@ class GeneratedOutput:
     # Could be a sequence made up of multimedia content
     multimodal_content: Optional[MultimediaObject] = None
 
+    # Could be reasoning
+    thinking: Optional[Thinking] = None
+
     def __add__(self, other: "GeneratedOutput") -> "GeneratedOutput":
         return GeneratedOutput(self.text + other.text, self.logprob + other.logprob, self.tokens + other.tokens)
 

helm/common/test_logging.py

@@ -0,0 +1,94 @@
+import sys
+import tempfile
+import textwrap
+import pathlib
+from helm.benchmark import run
+from typing import List, Optional
+
+
+class ArgvContext:
+    """
+    Helper to assign a temporary value to sys.argv and then restore it
+    """
+
+    def __init__(self, argv: Optional[List[str]]):
+        self.argv = argv
+        self._original_argv: Optional[List[str]] = None
+
+    def __enter__(self):
+        self._original_argv = sys.argv[:]
+        sys.argv = self.argv or []
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        assert self._original_argv is not None  # Satisfies mypy
+        sys.argv = self._original_argv
+
+
+def test_run_with_custom_logging_config():
+    # Setup temporary directory
+    with tempfile.TemporaryDirectory(prefix="helm_test_") as tmp_dir_str:
+        tmp_dir = pathlib.Path(tmp_dir_str)
+        log_path = tmp_dir / "test.log"
+        log_config_path = tmp_dir / "test_config.yaml"
+
+        # Write custom YAML log config to file
+        log_config_text = textwrap.dedent(
+            f"""
+            version: 1
+            disable_existing_loggers: false
+            formatters:
+              simple:
+                datefmt: '%Y-%m-%dT%H:%M:%S'
+                format: '%(asctime)s %(levelname)s %(name)s %(message)s'
+            handlers:
+              file:
+                class: logging.FileHandler
+                filename: {log_path}
+                formatter: simple
+                level: DEBUG
+                mode: w
+            loggers:
+              helm:
+                handlers:
+                - file
+                level: DEBUG
+                propagate: false
+            """
+        ).strip()
+
+        log_config_path.write_text(log_config_text)
+
+        # Simulate command-line arguments
+        argv = [
+            "run.py",  # Fake script name
+            "--run-entries",
+            "mmlu:subject=philosophy,model=openai/gpt2",
+            "-m",
+            "1",
+            "--suite",
+            "my-suite",
+            "--dry-run",
+            "--log-config",
+            str(log_config_path),
+        ]
+
+        # Call main
+        with ArgvContext(argv):
+            run.main()
+
+        # Check log file contents
+        assert log_path.exists(), "Log file was not created"
+        log_contents = log_path.read_text()
+
+        # Test that log file was written to disk as requested
+        print("Log Contents")
+        print("------------")
+        print(log_contents)
+
+        assert (
+            "mscoco" in log_contents or "huggingface" in log_contents or "dry-run" in log_contents
+        ), "Expected log content not found in log file:\n"
+
+
+if __name__ == "__main__":
+    test_run_with_custom_logging_config()