crfm-helm 0.5.5__py3-none-any.whl → 0.5.6__py3-none-any.whl

helm/clients/writer_client.py

@@ -0,0 +1,102 @@
+from typing import Any, Dict, List, Mapping, Optional
+
+from helm.clients.client import CachingClient
+from helm.common.cache import CacheConfig
+from helm.common.optional_dependencies import handle_module_not_found_error
+from helm.common.request import wrap_request_time, Request, RequestResult, GeneratedOutput, Token
+
+try:
+    from writerai import Writer
+    from writerai.types.chat_completion import ChatCompletion
+except ModuleNotFoundError as e:
+    handle_module_not_found_error(e, ["openai"])
+
+
+class WriterClient(CachingClient):
+    def __init__(self, cache_config: CacheConfig, api_key: Optional[str] = None):
+        super().__init__(cache_config=cache_config)
+        self._writer_client = Writer(api_key=api_key)
+
+    def _get_messages_from_request(self, request: Request) -> List[Dict]:
+        if request.prompt and request.messages:
+            raise ValueError(f"Only one of `prompt` and `messages` may be set in request: {request}")
+        if request.multimodal_prompt:
+            raise ValueError("`multimodal_prompt` is not supported by WriterClient")
+        if request.messages:
+            return [{"role": message["role"], "content": message["content"]} for message in request.messages]
+        else:
+            return [{"role": "user", "content": request.prompt}]
+
+    def _convert_chat_completion_to_generated_outputs(
+        self, chat_completion: ChatCompletion, request: Request
+    ) -> List[GeneratedOutput]:
+        generated_outputs: List[GeneratedOutput] = []
+        for choice in chat_completion.choices:
+            raw_completion_content = choice.message.content
+            # The Writer chat completion API doesn't support echo.
+            # If `echo_prompt` is true, combine the prompt and completion.
+            text: str = request.prompt + raw_completion_content if request.echo_prompt else raw_completion_content
+            tokens: List[Token] = []
+            if choice.logprobs and choice.logprobs.content:
+                tokens = [
+                    Token(text=choice_token.token, logprob=choice_token.logprob)
+                    for choice_token in choice.logprobs.content
+                ]
+            generated_output = GeneratedOutput(
+                text=text,
+                logprob=sum(token.logprob for token in tokens) if tokens else 0.0,
+                tokens=tokens,
+                finish_reason={"reason": choice.finish_reason},
+            )
+            generated_outputs.append(generated_output)
+        return generated_outputs
+
+    def _convert_request_to_raw_request(self, request: Request) -> Dict:
+        raw_request = {
+            "messages": self._get_messages_from_request(request),
+            "model": request.model.split("/")[-1],
+            "logprobs": bool(request.top_k_per_token),
+            "max_tokens": request.max_tokens,
+            "n": request.num_completions,
+            "stop": request.stop_sequences,
+            "temperature": request.temperature,
+            "top_p": request.top_p,
+        }
+        if request.response_format and request.response_format.json_schema:
+            raw_request["response_format"] = {
+                "type": "json_schema",
+                "json_schema": {
+                    "schema": request.response_format.json_schema,
+                },
+            }
+        return raw_request
+
+    def make_request(self, request: Request) -> RequestResult:
+        raw_request = self._convert_request_to_raw_request(request)
+        cache_key: Mapping = CachingClient.make_cache_key(raw_request, request)
+
+        def do_it() -> Dict[Any, Any]:
+            return self._writer_client.chat.chat(**raw_request).model_dump()
+
+        try:
+            raw_response, cached = self.cache.get(cache_key, wrap_request_time(do_it))
+            chat_completion: ChatCompletion = ChatCompletion.model_validate(raw_response)
+        except Exception as error:
+            return RequestResult(
+                success=False,
+                cached=False,
+                error=str(error),
+                completions=[],
+                embedding=[],
+            )
+
+        generated_outputs = self._convert_chat_completion_to_generated_outputs(chat_completion, request)
+
+        return RequestResult(
+            success=True,
+            cached=cached,
+            request_time=raw_response["request_time"],
+            request_datetime=raw_response["request_datetime"],
+            completions=generated_outputs,
+            embedding=[],
+        )

helm/common/context.py

@@ -0,0 +1,80 @@
+from abc import ABC, abstractmethod
+
+from helm.common.critique_request import CritiqueRequest, CritiqueRequestResult
+from helm.common.clip_score_request import CLIPScoreRequest, CLIPScoreResult
+from helm.common.file_upload_request import FileUploadResult, FileUploadRequest
+from helm.common.nudity_check_request import NudityCheckRequest, NudityCheckResult
+from helm.common.perspective_api_request import PerspectiveAPIRequestResult, PerspectiveAPIRequest
+from helm.common.moderations_api_request import ModerationAPIRequest, ModerationAPIRequestResult
+from helm.common.tokenization_request import (
+    TokenizationRequest,
+    TokenizationRequestResult,
+    DecodeRequest,
+    DecodeRequestResult,
+)
+from helm.common.request import Request, RequestResult
+from helm.proxy.query import Query, QueryResult
+from helm.common.cache import CacheConfig
+from helm.proxy.services.service import GeneralInfo
+
+
+class Context(ABC):
+    @abstractmethod
+    def get_general_info(self) -> GeneralInfo:
+        """Get general info."""
+        pass
+
+    @abstractmethod
+    def expand_query(self, query: Query) -> QueryResult:
+        """Turn the `query` into requests."""
+        pass
+
+    @abstractmethod
+    def make_request(self, request: Request) -> RequestResult:
+        """Actually make a request to an API."""
+        pass
+
+    @abstractmethod
+    def tokenize(self, request: TokenizationRequest) -> TokenizationRequestResult:
+        """Tokenize via an API."""
+        pass
+
+    @abstractmethod
+    def decode(self, request: DecodeRequest) -> DecodeRequestResult:
+        """Decodes to text."""
+        pass
+
+    @abstractmethod
+    def upload(self, request: FileUploadRequest) -> FileUploadResult:
+        """Uploads a file to external storage."""
+        pass
+
+    @abstractmethod
+    def check_nudity(self, request: NudityCheckRequest) -> NudityCheckResult:
+        """Check for nudity for a batch of images."""
+        pass
+
+    @abstractmethod
+    def compute_clip_score(self, request: CLIPScoreRequest) -> CLIPScoreResult:
+        """Computes CLIPScore for a given caption and image."""
+        pass
+
+    @abstractmethod
+    def get_toxicity_scores(self, request: PerspectiveAPIRequest) -> PerspectiveAPIRequestResult:
+        """Get toxicity scores for a batch of text."""
+        pass
+
+    @abstractmethod
+    def get_moderation_results(self, request: ModerationAPIRequest) -> ModerationAPIRequestResult:
+        """Get OpenAI's moderation results for some text."""
+        pass
+
+    @abstractmethod
+    def make_critique_request(self, request: CritiqueRequest) -> CritiqueRequestResult:
+        """Get responses to a critique request."""
+        pass
+
+    @abstractmethod
+    def get_cache_config(self, shard_name: str) -> CacheConfig:
+        """Returns a CacheConfig"""
+        pass

helm/common/credentials_utils.py

@@ -2,7 +2,7 @@
 
 from typing import Any, Mapping, Optional
 
-from helm.common.hierarchical_logger import hlog
+from helm.common.hierarchical_logger import hlog, hwarn
 
 
 def provide_api_key(
@@ -13,16 +13,16 @@ def provide_api_key(
         hlog(f"Using host_organization api key defined in credentials.conf: {api_key_name}")
         return credentials[api_key_name]
     if "deployments" not in credentials:
-        hlog(
-            "WARNING: Could not find key 'deployments' in credentials.conf, "
+        hwarn(
+            "Could not find key 'deployments' in credentials.conf, "
             f"therefore the API key {api_key_name} should be specified."
         )
         return None
     deployment_api_keys = credentials["deployments"]
     if model is None:
-        hlog(f"WARNING: Could not find key '{host_organization}' in credentials.conf and no model provided")
+        hwarn(f"Could not find key '{host_organization}' in credentials.conf and no model provided")
         return None
     if model not in deployment_api_keys:
-        hlog(f"WARNING: Could not find key '{model}' under key 'deployments' in credentials.conf")
+        hwarn(f"Could not find key '{model}' under key 'deployments' in credentials.conf")
         return None
     return deployment_api_keys[model]

helm/common/general.py

@@ -42,6 +42,13 @@ def ensure_directory_exists(path: str):
     os.makedirs(path, exist_ok=True)
 
 
+def check_file_exists(path: str, msg: Optional[str] = None):
+    """Checks that `path` exists, raises FileNotFoundError if it doesn't."""
+    if not os.path.exists(path):
+        error_msg = msg if msg else f"Required file not found: {path}"
+        raise FileNotFoundError(error_msg)
+
+
 def parse_hocon(text: str):
     """Parse `text` (in HOCON format) into a dict-like object."""
     return pyhocon.ConfigFactory.parse_string(text)
@@ -156,7 +163,7 @@ def format_split(split: str) -> str:
 
 
 def asdict_without_nones(obj: Any) -> Dict[str, Any]:
-    if not is_dataclass(obj):
+    if not is_dataclass(obj) or isinstance(obj, type):
         raise ValueError(f"Expected dataclass, got '{obj}'")
     return asdict(obj, dict_factory=lambda x: {k: v for (k, v) in x if v is not None})
 
@@ -178,7 +185,7 @@ def binarize_dict(d: Dict[str, int]) -> Dict[str, int]:
 
 def serialize(obj: Any) -> List[str]:
     """Takes in a dataclass and outputs all of its fields and values in a list."""
-    if not is_dataclass(obj):
+    if not is_dataclass(obj) or isinstance(obj, type):
         raise ValueError(f"Expected dataclass, got '{obj}'")
     return [f"{key}: {json.dumps(value)}" for key, value in asdict(obj).items()]
 

helm/common/hierarchical_logger.py

@@ -1,6 +1,8 @@
+import logging
 import sys
 import time
 from typing import Any, Callable, List, Optional
+from colorlog import ColoredFormatter
 
 
 class HierarchicalLogger(object):
@@ -20,6 +22,12 @@ class HierarchicalLogger(object):
         } [0s]
     """
 
+    # Far too much effort to unwind every call to hlog to go via logging,
+    # And is a terrible idea to inspect the stack every time hlog is called
+    # to figure out the caller,
+    # So just log everything under "helm".
+    logger = logging.getLogger("helm")
+
     def __init__(self) -> None:
         self.start_times: List[float] = []
 
@@ -27,17 +35,21 @@ class HierarchicalLogger(object):
         return "  " * len(self.start_times)
 
     def track_begin(self, x: Any) -> None:
-        print(self.indent() + str(x) + " {")
+        self.logger.info(self.indent() + str(x) + " {")
         sys.stdout.flush()
         self.start_times.append(time.time())
 
     def track_end(self) -> None:
         t = time.time() - self.start_times.pop()
-        print(self.indent() + "} [%s]" % (format_time(t)))
+        self.logger.info(self.indent() + "} [%s]" % (format_time(t)))
         sys.stdout.flush()
 
     def log(self, x: Any) -> None:
-        print(self.indent() + str(x))
+        self.logger.info(self.indent() + str(x))
+        sys.stdout.flush()
+
+    def warn(self, x: Any) -> None:
+        self.logger.warning(self.indent() + str(x))
         sys.stdout.flush()
 
 
@@ -61,6 +73,10 @@ def hlog(x: Any) -> None:
     singleton.log(x)
 
 
+def hwarn(x: Any) -> None:
+    singleton.warn(x)
+
+
 class htrack_block:
     def __init__(self, x: Any) -> None:
         self.x = x
@@ -104,3 +120,30 @@ class htrack:
                 return fn(*args, **kwargs)
 
         return wrapper
+
+
+def setup_default_logging():
+    """
+    Setup a default logger to STDOUT for HELM via Python logging
+    """
+    formatter = ColoredFormatter(
+        "%(bold_black)s%(asctime)s%(reset)s %(log_color)s%(levelname)-8s%(reset)s %(message)s",
+        datefmt="%Y-%m-%dT%H:%M:%S",
+        reset=True,
+        log_colors={
+            "DEBUG": "cyan",
+            "INFO": "green",
+            "WARNING": "yellow",
+            "ERROR": "red",
+            "CRITICAL": "red,bg_white",
+        },
+        secondary_log_colors={},
+        style="%",
+    )
+
+    logger = logging.getLogger("helm")
+    logger.setLevel(logging.INFO)
+    logger.propagate = False
+    handler = logging.StreamHandler(sys.stdout)
+    handler.setFormatter(formatter)
+    logger.addHandler(handler)

helm/common/local_context.py

@@ -0,0 +1,140 @@
+import dataclasses
+import os
+from typing import Optional
+
+from helm.common.context import Context
+from helm.common.cache import CacheConfig
+from helm.common.cache_backend_config import CacheBackendConfig, BlackHoleCacheBackendConfig
+from helm.common.critique_request import CritiqueRequest, CritiqueRequestResult
+from helm.common.moderations_api_request import ModerationAPIRequest, ModerationAPIRequestResult
+from helm.common.clip_score_request import CLIPScoreRequest, CLIPScoreResult
+from helm.common.nudity_check_request import NudityCheckRequest, NudityCheckResult
+from helm.common.file_upload_request import FileUploadRequest, FileUploadResult
+from helm.common.general import ensure_directory_exists, parse_hocon, get_credentials
+from helm.common.perspective_api_request import PerspectiveAPIRequest, PerspectiveAPIRequestResult
+from helm.common.tokenization_request import (
+    TokenizationRequest,
+    TokenizationRequestResult,
+    DecodeRequest,
+    DecodeRequestResult,
+)
+from helm.common.request import Request, RequestResult
+from helm.clients.auto_client import AutoClient
+from helm.clients.moderation_api_client import ModerationAPIClient
+from helm.clients.image_generation.nudity_check_client import NudityCheckClient
+from helm.clients.gcs_client import GCSClient
+from helm.clients.clip_score_client import CLIPScoreClient
+from helm.clients.toxicity_classifier_client import ToxicityClassifierClient
+from helm.proxy.example_queries import example_queries
+from helm.benchmark.model_metadata_registry import ALL_MODELS_METADATA
+from helm.proxy.query import Query, QueryResult
+from helm.proxy.retry import retry_request
+from helm.tokenizers.auto_tokenizer import AutoTokenizer
+from helm.proxy.services.service import (
+    CACHE_DIR,
+    GeneralInfo,
+    VERSION,
+    expand_environments,
+    synthesize_request,
+)
+
+
+class LocalContext(Context):
+    """
+    Main class that supports various functionality for the server.
+    """
+
+    def __init__(
+        self,
+        base_path: str = "prod_env",
+        cache_backend_config: CacheBackendConfig = BlackHoleCacheBackendConfig(),
+    ):
+        ensure_directory_exists(base_path)
+        client_file_storage_path = os.path.join(base_path, CACHE_DIR)
+        ensure_directory_exists(client_file_storage_path)
+
+        credentials = get_credentials(base_path)
+
+        self.cache_backend_config = cache_backend_config
+        self.client = AutoClient(credentials, client_file_storage_path, cache_backend_config)
+        self.tokenizer = AutoTokenizer(credentials, cache_backend_config)
+
+        # Lazily instantiate the following clients
+        self.moderation_api_client: Optional[ModerationAPIClient] = None
+        self.toxicity_classifier_client: Optional[ToxicityClassifierClient] = None
+        self.perspective_api_client: Optional[ToxicityClassifierClient] = None
+        self.nudity_check_client: Optional[NudityCheckClient] = None
+        self.clip_score_client: Optional[CLIPScoreClient] = None
+        self.gcs_client: Optional[GCSClient] = None
+
+    def get_general_info(self) -> GeneralInfo:
+        # Can't send release_dates in ModelMetadata bacause dates cannot be round-tripped to and from JSON easily.
+        # TODO(#2158): Either fix this or delete get_general_info.
+        all_models = [dataclasses.replace(model_metadata, release_date=None) for model_metadata in ALL_MODELS_METADATA]
+        return GeneralInfo(version=VERSION, example_queries=example_queries, all_models=all_models)
+
+    def expand_query(self, query: Query) -> QueryResult:
+        """Turn the `query` into requests."""
+        prompt = query.prompt
+        settings = query.settings
+        environments = parse_hocon(query.environments)
+        requests = []
+        for environment in expand_environments(environments):
+            request = synthesize_request(prompt, settings, environment)
+            requests.append(request)
+        return QueryResult(requests=requests)
+
+    def make_request(self, request: Request) -> RequestResult:
+        """Actually make a request to an API."""
+        return self.client.make_request(request)
+
+    def tokenize(self, request: TokenizationRequest) -> TokenizationRequestResult:
+        return self.tokenizer.tokenize(request)
+
+    def decode(self, request: DecodeRequest) -> DecodeRequestResult:
+        return self.tokenizer.decode(request)
+
+    def upload(self, request: FileUploadRequest) -> FileUploadResult:
+        if not self.gcs_client:
+            self.gcs_client = self.client.get_gcs_client()
+
+        assert self.gcs_client
+        return self.gcs_client.upload(request)
+
+    def check_nudity(self, request: NudityCheckRequest) -> NudityCheckResult:
+        if not self.nudity_check_client:
+            self.nudity_check_client = self.client.get_nudity_check_client()
+
+        assert self.nudity_check_client
+        return self.nudity_check_client.check_nudity(request)
+
+    def compute_clip_score(self, request: CLIPScoreRequest) -> CLIPScoreResult:
+        if not self.clip_score_client:
+            self.clip_score_client = self.client.get_clip_score_client()
+
+        assert self.clip_score_client
+        return self.clip_score_client.compute_score(request)
+
+    def get_toxicity_scores(self, request: PerspectiveAPIRequest) -> PerspectiveAPIRequestResult:
+        @retry_request
+        def get_toxicity_scores_with_retry(request: PerspectiveAPIRequest) -> PerspectiveAPIRequestResult:
+            if not self.toxicity_classifier_client:
+                self.toxicity_classifier_client = self.client.get_toxicity_classifier_client()
+            return self.toxicity_classifier_client.get_toxicity_scores(request)
+
+        return get_toxicity_scores_with_retry(request)
+
+    def get_moderation_results(self, request: ModerationAPIRequest) -> ModerationAPIRequestResult:
+        @retry_request
+        def get_moderation_results_with_retry(request: ModerationAPIRequest) -> ModerationAPIRequestResult:
+            if not self.moderation_api_client:
+                self.moderation_api_client = self.client.get_moderation_api_client()
+            return self.moderation_api_client.get_moderation_results(request)
+
+        return get_moderation_results_with_retry(request)
+
+    def make_critique_request(self, request: CritiqueRequest) -> CritiqueRequestResult:
+        return self.client.get_critique_client().make_critique_request(request)
+
+    def get_cache_config(self, shard_name: str) -> CacheConfig:
+        return self.cache_backend_config.get_cache_config(shard_name)

helm/common/remote_context.py

@@ -0,0 +1,61 @@
+from helm.common.context import Context
+from helm.common.cache import CacheConfig
+from helm.common.authentication import Authentication
+from helm.common.moderations_api_request import ModerationAPIRequest, ModerationAPIRequestResult
+from helm.common.critique_request import CritiqueRequest, CritiqueRequestResult
+from helm.common.nudity_check_request import NudityCheckRequest, NudityCheckResult
+from helm.common.file_upload_request import FileUploadRequest, FileUploadResult
+from helm.common.perspective_api_request import PerspectiveAPIRequest, PerspectiveAPIRequestResult
+from helm.common.clip_score_request import CLIPScoreRequest, CLIPScoreResult
+from helm.common.tokenization_request import (
+    TokenizationRequest,
+    TokenizationRequestResult,
+    DecodeRequestResult,
+    DecodeRequest,
+)
+from helm.common.request import Request, RequestResult
+from helm.proxy.query import Query, QueryResult
+from helm.proxy.services.remote_service import RemoteService
+from helm.proxy.services.service import GeneralInfo, Service
+
+
+class RemoteContext(Context):
+    def __init__(self, base_url: str, auth: Authentication):
+        self.service: Service = RemoteService(base_url)
+        self.auth = auth
+
+    def get_general_info(self) -> GeneralInfo:
+        return self.service.get_general_info()
+
+    def expand_query(self, query: Query) -> QueryResult:
+        return self.service.expand_query(query)
+
+    def make_request(self, request: Request) -> RequestResult:
+        return self.service.make_request(self.auth, request)
+
+    def tokenize(self, request: TokenizationRequest) -> TokenizationRequestResult:
+        return self.service.tokenize(self.auth, request)
+
+    def decode(self, request: DecodeRequest) -> DecodeRequestResult:
+        return self.service.decode(self.auth, request)
+
+    def upload(self, request: FileUploadRequest) -> FileUploadResult:
+        return self.service.upload(self.auth, request)
+
+    def check_nudity(self, request: NudityCheckRequest) -> NudityCheckResult:
+        return self.service.check_nudity(self.auth, request)
+
+    def compute_clip_score(self, request: CLIPScoreRequest) -> CLIPScoreResult:
+        return self.service.compute_clip_score(self.auth, request)
+
+    def get_toxicity_scores(self, request: PerspectiveAPIRequest) -> PerspectiveAPIRequestResult:
+        return self.service.get_toxicity_scores(self.auth, request)
+
+    def get_moderation_results(self, request: ModerationAPIRequest) -> ModerationAPIRequestResult:
+        return self.service.get_moderation_results(self.auth, request)
+
+    def make_critique_request(self, request: CritiqueRequest) -> CritiqueRequestResult:
+        return self.service.make_critique_request(self.auth, request)
+
+    def get_cache_config(self, shard_name: str) -> CacheConfig:
+        return self.service.get_cache_config(shard_name)

helm/common/request.py

@@ -131,6 +131,11 @@ class Token:
         ]
 
 
+@dataclass(frozen=True)
+class Thinking:
+    text: Optional[str] = None
+
+
 @dataclass(frozen=True)
 class GeneratedOutput:
     """A `GeneratedOutput` is a single generated output that may contain text or multimodal content."""
@@ -150,6 +155,9 @@ class GeneratedOutput:
     # Could be a sequence made up of multimedia content
     multimodal_content: Optional[MultimediaObject] = None
 
+    # Could be reasoning
+    thinking: Optional[Thinking] = None
+
     def __add__(self, other: "GeneratedOutput") -> "GeneratedOutput":
         return GeneratedOutput(self.text + other.text, self.logprob + other.logprob, self.tokens + other.tokens)