PyPI - crfm-helm - Versions diffs - 0.5.8__py3-none-any.whl → 0.5.9__py3-none-any.whl - Mend

crfm-helm 0.5.8py3-none-any.whl → 0.5.9py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.

This version of crfm-helm might be problematic. Click here for more details.

Files changed (121) hide show

helm/benchmark/static_build/index.html CHANGED Viewed

@@ -7,14 +7,13 @@
     <title>Holistic Evaluation of Language Models (HELM)</title>
     <meta name="description" content="The Holistic Evaluation of Language Models (HELM) serves as a living benchmark for transparency in language models. Providing broad coverage and recognizing incompleteness, multi-metric measurements, and standardization. All data and analysis are freely accessible on the website for exploration and study." />
     <script type="text/javascript" src="./config.js"></script>
-    <script type="module" crossorigin src="./assets/index-671a5e06.js"></script>
-    <link rel="modulepreload" crossorigin href="./assets/react-f82877fd.js">
-    <link rel="modulepreload" crossorigin href="./assets/recharts-4037aff0.js">
-    <link rel="modulepreload" crossorigin href="./assets/tremor-38a10867.js">
-    <link rel="stylesheet" href="./assets/index-9352595e.css">
+    <script type="module" crossorigin src="./assets/index-qOFpOyHb.js"></script>
+    <link rel="modulepreload" crossorigin href="./assets/react-BteFIppM.js">
+    <link rel="modulepreload" crossorigin href="./assets/recharts-DxuQtTOs.js">
+    <link rel="modulepreload" crossorigin href="./assets/tremor-DR4fE7ko.js">
+    <link rel="stylesheet" crossorigin href="./assets/index-oIeiQW2g.css">
   </head>
   <body class="block">
     <div id="root"></div>
   </body>
 </html>

helm/clients/ai21_client.py CHANGED Viewed

@@ -2,6 +2,7 @@ from typing import Dict, List, Optional, TypedDict
 import requests
 from helm.common.cache import CacheConfig
+from helm.common.hierarchical_logger import hexception
 from helm.common.optional_dependencies import handle_module_not_found_error
 from helm.common.request import (
     wrap_request_time,
@@ -76,6 +77,7 @@ class AI21Client(CachingClient):
             cache_key = CachingClient.make_cache_key({"engine": request.model_engine, **raw_request}, request)
             response, cached = self.cache.get(cache_key, wrap_request_time(do_it))
         except AI21RequestError as e:
+            hexception(e)
             return RequestResult(success=False, cached=False, error=str(e), completions=[], embedding=[])
         def fix_text(x: str, first: bool) -> str:

helm/clients/aleph_alpha_client.py CHANGED Viewed

@@ -1,6 +1,7 @@
 from typing import List
 from helm.common.cache import CacheConfig
+from helm.common.hierarchical_logger import hexception
 from helm.common.media_object import TEXT_TYPE
 from helm.common.optional_dependencies import handle_module_not_found_error
 from helm.common.request import wrap_request_time, Request, RequestResult, GeneratedOutput, Token
@@ -76,6 +77,7 @@ class AlephAlphaClient(CachingClient):
             cache_key = CachingClient.make_cache_key({"model": model, "prompt": prompt_key, **parameters}, request)
             response, cached = self.cache.get(cache_key, wrap_request_time(do_it))
         except Exception as e:
+            hexception(e)
             error: str = f"AlephAlphaClient error: {e}"
             return RequestResult(success=False, cached=False, error=error, completions=[], embedding=[])

helm/clients/anthropic_client.py CHANGED Viewed

@@ -8,7 +8,7 @@ import time
 import urllib.parse
 from helm.common.cache import CacheConfig
-from helm.common.hierarchical_logger import htrack_block, hlog, hwarn
+from helm.common.hierarchical_logger import hexception, htrack_block, hlog, hwarn
 from helm.common.media_object import IMAGE_TYPE, TEXT_TYPE
 from helm.common.optional_dependencies import handle_module_not_found_error
 from helm.common.request import (
@@ -184,6 +184,7 @@ class AnthropicClient(CachingClient):
                         embedding=[],
                         error_flags=ErrorFlags(is_retriable=False, is_fatal=False),
                     )
+                hexception(error)
                 return RequestResult(success=False, cached=False, error=str(error), completions=[], embedding=[])
             # Post process the completion.
@@ -385,6 +386,10 @@ class AnthropicMessagesClient(CachingClient):
             # Avoid error:
             # `top_k` must be unset when thinking is enabled. Please consult our documentation at https://docs.anthropic.com/en/docs/build-with-claude/extended-thinking#important-considerations-when-using-extended-thinking  # noqa: E501
             del raw_request["top_k"]
+        if raw_request["model"].startswith("claude-sonnet-4-5"):
+            # Avoid error:
+            # `temperature` and `top_p` cannot both be specified for this model. Please use only one.
+            del raw_request["top_p"]
         completions: List[GeneratedOutput] = []
@@ -696,6 +701,7 @@ class AnthropicLegacyClient(CachingClient):
                 )
                 response, cached = self.cache.get(cache_key, wrap_request_time(do_it))
             except AnthropicRequestError as error:
+                hexception(error)
                 return RequestResult(success=False, cached=False, error=str(error), completions=[], embedding=[])
             sequence_logprob: float = 0

helm/clients/audio_language/diva_llama_client.py CHANGED Viewed

@@ -6,6 +6,7 @@ from transformers import AutoModel, PreTrainedModel
 from helm.clients.client import CachingClient
 from helm.common.cache import CacheConfig
+from helm.common.hierarchical_logger import hexception
 from helm.common.media_object import TEXT_TYPE
 from helm.common.request import (
     GeneratedOutput,
@@ -105,6 +106,7 @@ class DivaLlamaClient(CachingClient):
             cache_key = CachingClient.make_cache_key(raw_request, request)
             response, cached = self.cache.get(cache_key, wrap_request_time(do_it))
         except Exception as e:  # Do something if error is encountered.
+            hexception(e)
             error: str = f"HuggingFace error: {e}"
             return RequestResult(success=False, cached=False, error=error, completions=[], embedding=[])

helm/clients/audio_language/llama_omni_client.py CHANGED Viewed

@@ -12,7 +12,7 @@ from helm.clients.audio_language.llama_omni.preprocess import tokenizer_speech_t
 from helm.common.cache import CacheConfig
 from helm.common.gpu_utils import get_torch_device_name
-from helm.common.hierarchical_logger import hlog, htrack_block
+from helm.common.hierarchical_logger import hexception, hlog, htrack_block
 from helm.common.media_object import TEXT_TYPE
 from helm.common.request import Request, RequestResult, GeneratedOutput, Token
 from helm.common.request import wrap_request_time
@@ -170,6 +170,7 @@ class LlamaOmniAudioLMClient(CachingClient):
                     )
                     result, cached = self.cache.get(cache_key, wrap_request_time(do_it))
                 except RuntimeError as model_error:
+                    hexception(model_error)
                     return RequestResult(
                         success=False, cached=False, error=str(model_error), completions=[], embedding=[]
                     )

helm/clients/audio_language/qwen2_5_omni_client.py CHANGED Viewed

@@ -9,7 +9,7 @@ from helm.clients.audio_language.qwen_omni.qwen2_5_omni_utils.v2_5 import proces
 from helm.common.cache import CacheConfig
 from helm.common.gpu_utils import get_torch_device_name
-from helm.common.hierarchical_logger import hlog, htrack_block
+from helm.common.hierarchical_logger import hexception, hlog, htrack_block
 from helm.common.media_object import TEXT_TYPE
 from helm.common.request import Request, RequestResult, GeneratedOutput, Token
 from helm.common.request import wrap_request_time
@@ -180,6 +180,7 @@ class Qwen2_5OmniAudioLMClient(CachingClient):
                     )
                     result, cached = self.cache.get(cache_key, wrap_request_time(do_it))
                 except RuntimeError as model_error:
+                    hexception(model_error)
                     return RequestResult(
                         success=False, cached=False, error=str(model_error), completions=[], embedding=[]
                     )

helm/clients/audio_language/qwen2_audiolm_client.py CHANGED Viewed

@@ -7,7 +7,7 @@ from transformers import Qwen2AudioForConditionalGeneration, AutoProcessor
 from helm.common.cache import CacheConfig
 from helm.common.gpu_utils import get_torch_device_name
-from helm.common.hierarchical_logger import hlog, htrack_block
+from helm.common.hierarchical_logger import hexception, hlog, htrack_block
 from helm.common.media_object import TEXT_TYPE
 from helm.common.request import Request, RequestResult, GeneratedOutput, Token
 from helm.common.request import wrap_request_time
@@ -161,6 +161,7 @@ class Qwen2AudioLMClient(CachingClient):
                     )
                     result, cached = self.cache.get(cache_key, wrap_request_time(do_it))
                 except RuntimeError as model_error:
+                    hexception(model_error)
                     return RequestResult(
                         success=False, cached=False, error=str(model_error), completions=[], embedding=[]
                     )

helm/clients/audio_language/qwen_audiolm_client.py CHANGED Viewed

@@ -6,7 +6,7 @@ from transformers import AutoModelForCausalLM, AutoTokenizer
 from helm.common.cache import CacheConfig
 from helm.common.gpu_utils import get_torch_device_name
-from helm.common.hierarchical_logger import hlog, htrack_block
+from helm.common.hierarchical_logger import hexception, hlog, htrack_block
 from helm.common.media_object import TEXT_TYPE
 from helm.common.request import Request, RequestResult, GeneratedOutput, Token
 from helm.common.request import wrap_request_time
@@ -124,6 +124,7 @@ class QwenAudioLMClient(CachingClient):
                     )
                     result, cached = self.cache.get(cache_key, wrap_request_time(do_it))
                 except RuntimeError as model_error:
+                    hexception(model_error)
                     return RequestResult(
                         success=False, cached=False, error=str(model_error), completions=[], embedding=[]
                     )

helm/clients/bedrock_client.py CHANGED Viewed

@@ -7,6 +7,7 @@ from datetime import datetime
 from helm.common.cache import CacheConfig
 from helm.clients.client import CachingClient, truncate_and_tokenize_response_text
+from helm.common.hierarchical_logger import hexception
 from helm.common.request import Request, RequestResult, GeneratedOutput, wrap_request_time
 from helm.clients.bedrock_utils import get_bedrock_client, get_bedrock_client_v1
 from helm.tokenizers.tokenizer import Tokenizer
@@ -75,6 +76,7 @@ class BedrockClient(CachingClient):
             response, cached = self.cache.get(cache_key, wrap_request_time(do_it))
         except Exception as error:
+            hexception(error)
             return RequestResult(
                 success=False,
                 cached=False,

helm/clients/cohere_client.py CHANGED Viewed

@@ -3,6 +3,7 @@ import requests
 from typing import List, Optional, Sequence, TypedDict
 from helm.common.cache import CacheConfig
+from helm.common.hierarchical_logger import hexception
 from helm.common.optional_dependencies import handle_module_not_found_error
 from helm.common.request import (
     wrap_request_time,
@@ -123,6 +124,7 @@ class CohereClient(CachingClient):
             response, cached = self.cache.get(raw_request, wrap_request_time(do_it))
         except (requests.exceptions.RequestException, AssertionError) as e:
+            hexception(e)
             error: str = f"CohereClient error: {e}"
             return RequestResult(success=False, cached=False, error=error, completions=[], embedding=[])
@@ -232,6 +234,7 @@ class CohereChatClient(CachingClient):
             response, cached = self.cache.get(raw_request, wrap_request_time(do_it))
         except (requests.exceptions.RequestException, AssertionError) as e:
+            hexception(e)
             error: str = f"CohereClient error: {e}"
             return RequestResult(success=False, cached=False, error=error, completions=[], embedding=[])

helm/clients/google_client.py CHANGED Viewed

@@ -1,6 +1,7 @@
 from typing import List, Dict
 from helm.common.cache import CacheConfig
+from helm.common.hierarchical_logger import hexception
 from helm.common.request import Request, RequestResult, GeneratedOutput, Token
 from helm.clients.client import CachingClient, truncate_sequence
@@ -44,6 +45,7 @@ class GoogleClient(CachingClient):
             # If results are not cached for a given query, fail fast
             response, cached = self.cache.get(cache_key, fail)
         except RuntimeError as e:
+            hexception(e)
             error: str = f"GoogleClient error: {e}"
             return RequestResult(success=False, cached=False, error=error, completions=[], embedding=[])

helm/clients/http_model_client.py CHANGED Viewed

@@ -3,6 +3,7 @@ from dataclasses import asdict
 from typing import Any, Dict
 from helm.common.cache import CacheConfig
+from helm.common.hierarchical_logger import hexception
 from helm.common.request import (
     wrap_request_time,
     Request,
@@ -76,5 +77,6 @@ class HTTPModelClient(CachingClient):
                 request_time=response["request_time"],
             )
         except requests.exceptions.RequestException as e:
+            hexception(e)
             error: str = f"Request error: {e}"
             return RequestResult(success=False, cached=False, error=error, completions=[], embedding=[])

helm/clients/huggingface_client.py CHANGED Viewed

@@ -8,7 +8,7 @@ from transformers.generation.stopping_criteria import (
 from typing import Any, Dict, List, Optional, TypedDict
 from helm.common.cache import CacheConfig
-from helm.common.hierarchical_logger import htrack_block, hlog, hwarn
+from helm.common.hierarchical_logger import hexception, htrack_block, hlog, hwarn
 from helm.common.optional_dependencies import handle_module_not_found_error
 from helm.common.request import (
     wrap_request_time,
@@ -345,6 +345,7 @@ class HuggingFaceClient(CachingClient):
             response, cached = self.cache.get(cache_key, wrap_request_time(do_it))
         except Exception as e:  # Do something if error is encountered.
             error: str = f"HuggingFace error: {e}"
+            hexception(e)
             return RequestResult(success=False, cached=False, error=error, completions=[], embedding=[])
         completions = []

helm/clients/ibm_client.py CHANGED Viewed

@@ -1,7 +1,7 @@
 from abc import ABC
 from abc import abstractmethod
-from helm.common.hierarchical_logger import hlog
+from helm.common.hierarchical_logger import hexception, hlog
 from helm.common.cache import CacheConfig
 from helm.common.request import (
     Request,
@@ -249,6 +249,7 @@ class IbmChatClient(IbmClient):
             )
         except Exception as e:
+            hexception(e)
             error: str = f"IBM Chat client Model error: {e}"
             return RequestResult(success=False, cached=False, error=error, completions=[], embedding=[])
@@ -263,5 +264,6 @@ class IbmTextClient(IbmClient):
                 inference_handler=GenerateInferenceHandler(inference_engine=self.inference_engine), request=request
             )
         except Exception as e:
+            hexception(e)
             error: str = f"IBM Text client Model error: {e}"
             return RequestResult(success=False, cached=False, error=error, completions=[], embedding=[])

helm/clients/image_generation/adobe_vision_client.py CHANGED Viewed

@@ -1,6 +1,7 @@
 from typing import List, Dict
 from helm.common.cache import Cache, CacheConfig
+from helm.common.hierarchical_logger import hexception
 from helm.common.request import Request, RequestResult, GeneratedOutput
 from helm.common.tokenization_request import (
     TokenizationRequest,
@@ -54,6 +55,7 @@ class AdobeVisionClient(Client):
             response, cached = self._cache.get(cache_key, fail)
         except RuntimeError as e:
+            hexception(e)
             error: str = f"Adobe Vision Client error: {e}"
             return RequestResult(success=False, cached=False, error=error, completions=[], embedding=[])

helm/clients/image_generation/aleph_alpha_image_generation_client.py CHANGED Viewed

@@ -1,6 +1,7 @@
 from typing import List, Dict
 from helm.common.cache import Cache, CacheConfig
+from helm.common.hierarchical_logger import hexception
 from helm.common.request import Request, RequestResult, GeneratedOutput
 from helm.common.tokenization_request import (
     TokenizationRequest,
@@ -74,6 +75,7 @@ class AlephAlphaImageGenerationClient(Client):
             response, cached = self._cache.get(cache_key, fail)
         except RuntimeError as e:
+            hexception(e)
             error: str = f"AlephAlphaVisionClient error: {e}"
             return RequestResult(success=False, cached=False, error=error, completions=[], embedding=[])

helm/clients/image_generation/cogview2_client.py CHANGED Viewed

@@ -9,7 +9,7 @@ from torchvision.utils import save_image
 from helm.common.cache import CacheConfig, Cache
 from helm.common.file_caches.file_cache import FileCache
-from helm.common.hierarchical_logger import hlog, htrack_block
+from helm.common.hierarchical_logger import hexception, hlog, htrack_block
 from helm.common.optional_dependencies import handle_module_not_found_error
 from helm.common.request import Request, RequestResult, GeneratedOutput, wrap_request_time
 from helm.common.tokenization_request import (
@@ -167,6 +167,7 @@ class CogView2Client(Client):
             )
             results, cached = self._cache.get(cache_key, wrap_request_time(do_it))
         except RuntimeError as e:
+            hexception(e)
             error: str = f"CogView2Client error: {e}"
             return RequestResult(success=False, cached=False, error=error, completions=[], embedding=[])

helm/clients/image_generation/dalle2_client.py CHANGED Viewed

@@ -4,6 +4,7 @@ import base64
 from helm.common.cache import CacheConfig, Cache
 from helm.common.general import hlog
 from helm.common.file_caches.file_cache import FileCache
+from helm.common.hierarchical_logger import hexception
 from helm.common.media_object import MultimediaObject
 from helm.common.optional_dependencies import handle_module_not_found_error
 from helm.common.request import Request, RequestResult, GeneratedOutput, wrap_request_time
@@ -124,6 +125,7 @@ class DALLE2Client(Client):
             hlog(f"Failed safety check: {request.prompt}")
             return self.get_content_policy_violated_result(request)
         else:
+            hexception(error)
             return RequestResult(
                 success=False, cached=False, error=f"DALL-E error: {error}", completions=[], embedding=[]
             )

helm/clients/image_generation/dalle_mini_client.py CHANGED Viewed

@@ -5,7 +5,7 @@ from functools import partial
 from helm.common.cache import CacheConfig, Cache
 from helm.common.file_caches.file_cache import FileCache
-from helm.common.hierarchical_logger import hlog, htrack_block
+from helm.common.hierarchical_logger import hexception, hlog, htrack_block
 from helm.common.optional_dependencies import handle_module_not_found_error
 from helm.common.request import Request, RequestResult, GeneratedOutput, wrap_request_time
 from helm.common.tokenization_request import (
@@ -166,6 +166,7 @@ class DALLEMiniClient(Client):
             )
             results, cached = self._cache.get(cache_key, wrap_request_time(do_it))
         except RuntimeError as e:
+            hexception(e)
             error: str = f"DALLEMiniClient error: {e}"
             return RequestResult(success=False, cached=False, error=error, completions=[], embedding=[])

helm/clients/image_generation/deep_floyd_client.py CHANGED Viewed

@@ -1,6 +1,7 @@
 from typing import List, Dict
 from helm.common.cache import Cache, CacheConfig
+from helm.common.hierarchical_logger import hexception
 from helm.common.request import Request, RequestResult, GeneratedOutput
 from helm.common.tokenization_request import (
     TokenizationRequest,
@@ -54,6 +55,7 @@ class DeepFloydClient(Client):
             response, cached = self._cache.get(cache_key, fail)
         except RuntimeError as e:
+            hexception(e)
             error: str = f"DeepFloyd Client error: {e}"
             return RequestResult(success=False, cached=False, error=error, completions=[], embedding=[])

helm/clients/image_generation/huggingface_diffusers_client.py CHANGED Viewed

@@ -7,7 +7,7 @@ import torch
 from helm.common.cache import CacheConfig, Cache
 from helm.common.file_caches.file_cache import FileCache
 from helm.common.gpu_utils import get_torch_device_name, is_cuda_available
-from helm.common.hierarchical_logger import hlog, htrack_block
+from helm.common.hierarchical_logger import hexception, hlog, htrack_block
 from helm.common.optional_dependencies import handle_module_not_found_error
 from helm.common.request import Request, RequestResult, GeneratedOutput, wrap_request_time
 from helm.common.tokenization_request import (
@@ -178,6 +178,7 @@ class HuggingFaceDiffusersClient(Client):
             )
             results, cached = self._cache.get(cache_key, wrap_request_time(do_it))
         except RuntimeError as ex:
+            hexception(ex)
             error: str = f"HuggingFaceDiffusersClient error: {ex}"
             return RequestResult(success=False, cached=False, error=error, completions=[], embedding=[])

helm/clients/image_generation/lexica_client.py CHANGED Viewed

@@ -5,6 +5,7 @@ import urllib.parse
 from helm.common.cache import CacheConfig, Cache
 from helm.common.file_caches.file_cache import FileCache
+from helm.common.hierarchical_logger import hexception
 from helm.common.images_utils import encode_base64
 from helm.common.request import Request, RequestResult, GeneratedOutput, wrap_request_time
 from helm.common.tokenization_request import (
@@ -62,6 +63,7 @@ class LexicaClient(Client):
             response, cached = self.cache.get(cache_key, wrap_request_time(do_it))
         except RuntimeError as e:
+            hexception(e)
             error: str = f"LexicaClient error: {e}"
             return RequestResult(success=False, cached=False, error=error, completions=[], embedding=[])

helm/clients/image_generation/mindalle_client.py CHANGED Viewed

@@ -5,7 +5,7 @@ import numpy as np
 from helm.common.cache import CacheConfig, Cache
 from helm.common.file_caches.file_cache import FileCache
 from helm.common.gpu_utils import get_torch_device_name
-from helm.common.hierarchical_logger import hlog, htrack_block
+from helm.common.hierarchical_logger import hexception, hlog, htrack_block
 from helm.common.optional_dependencies import handle_module_not_found_error
 from helm.common.request import Request, RequestResult, GeneratedOutput, wrap_request_time
 from helm.common.tokenization_request import (
@@ -91,6 +91,7 @@ class MinDALLEClient(Client):
             )
             results, cached = self._cache.get(cache_key, wrap_request_time(do_it))
         except RuntimeError as ex:
+            hexception(ex)
             error: str = f"MinDALLEClient error: {ex}"
             return RequestResult(success=False, cached=False, error=error, completions=[], embedding=[])

helm/clients/image_generation/together_image_generation_client.py CHANGED Viewed

@@ -4,6 +4,7 @@ import requests
 from helm.common.cache import CacheConfig, Cache
 from helm.common.file_caches.file_cache import FileCache
+from helm.common.hierarchical_logger import hexception
 from helm.common.request import Request, RequestResult, GeneratedOutput, wrap_request_time
 from helm.common.tokenization_request import (
     TokenizationRequest,
@@ -84,6 +85,7 @@ class TogetherImageGenerationClient(Client):
             response, cached = self._cache.get(cache_key, wrap_request_time(do_it))
         except RuntimeError as e:
+            hexception(e)
             error: str = f"TogetherVisionClient error: {e}"
             return RequestResult(success=False, cached=False, error=error, completions=[], embedding=[])

helm/clients/megatron_client.py CHANGED Viewed

@@ -4,6 +4,7 @@ from typing import Any, Dict, List
 import traceback
 from helm.common.cache import CacheConfig
+from helm.common.hierarchical_logger import hexception
 from helm.common.request import (
     wrap_request_time,
     EMBEDDING_UNAVAILABLE_REQUEST_RESULT,
@@ -103,6 +104,7 @@ class MegatronClient(CachingClient):
         try:
             return self._make_request(request)
         except Exception as e:
+            hexception(e)
             return RequestResult(
                 success=False,
                 cached=False,

helm/clients/mistral_client.py CHANGED Viewed

@@ -1,6 +1,7 @@
 import requests
 from typing import Any, Dict, List, Optional, TypedDict, Union
+from helm.common.hierarchical_logger import hexception
 from helm.proxy.retry import NonRetriableException
 from helm.common.cache import CacheConfig
 from helm.common.media_object import IMAGE_TYPE, TEXT_TYPE
@@ -156,6 +157,7 @@ class MistralAIClient(CachingClient):
                 response, cached = self.cache.get(cache_key, wrap_request_time(do_it))
             except (requests.exceptions.RequestException, AssertionError) as e:
+                hexception(e)
                 error: str = f"MistralClient error: {e}"
                 return RequestResult(success=False, cached=False, error=error, completions=[], embedding=[])

helm/clients/moderation_api_client.py CHANGED Viewed

@@ -1,5 +1,6 @@
 from typing import Any, Dict
+from helm.common.hierarchical_logger import hexception
 from helm.common.request import wrap_request_time
 from helm.common.cache import Cache, CacheConfig
 from helm.common.moderations_api_request import (
@@ -64,6 +65,7 @@ class ModerationAPIClient:
             response, cached = self.cache.get(raw_request, wrap_request_time(do_it))
         except openai.OpenAIError as e:
+            hexception(e)
             error: str = f"Moderation API error: {e}"
             return ModerationAPIRequestResult(
                 success=False, cached=False, error=error, flagged=None, flagged_results=None, scores=None

helm/clients/openai_client.py CHANGED Viewed

@@ -10,7 +10,7 @@ from helm.common import multimodal_request_utils
 from helm.common.cache import CacheConfig
 from helm.common.media_object import TEXT_TYPE, MultimediaObject, MediaObject
 from helm.common.request import ErrorFlags, Thinking, wrap_request_time, Request, RequestResult, GeneratedOutput, Token
-from helm.common.hierarchical_logger import hlog, hwarn
+from helm.common.hierarchical_logger import hlog, hwarn, hexception
 from helm.common.object_spec import get_class_by_name
 from helm.common.optional_dependencies import handle_module_not_found_error
 from helm.common.tokenization_request import (
@@ -112,6 +112,7 @@ class OpenAIClientUtils:
                 error_flags=ErrorFlags(is_retriable=False, is_fatal=False),
             )
+        hexception(e)
         error: str = f"OpenAI error: {e}"
         return RequestResult(success=False, cached=False, error=error, completions=[], embedding=[])
@@ -170,6 +171,7 @@ class OpenAIClient(CachingClient):
             cache_key = self._get_cache_key(raw_request, request)
             response, cached = self.cache.get(cache_key, wrap_request_time(do_it))
         except openai.OpenAIError as e:
+            hexception(e)
             error: str = f"OpenAI error: {e}"
             return RequestResult(success=False, cached=False, error=error, completions=[], embedding=[])
@@ -436,6 +438,7 @@ class OpenAIClient(CachingClient):
             cache_key = self._get_cache_key(raw_request, request)
             response, cached = self.cache.get(cache_key, wrap_request_time(do_it))
         except openai.OpenAIError as e:
+            hexception(e)
             error: str = f"OpenAI error: {e}"
             return RequestResult(success=False, cached=False, error=error, completions=[], embedding=[])
@@ -491,6 +494,7 @@ class OpenAIClient(CachingClient):
             cache_key = self._get_cache_key({"audio": audio_path, "model": model}, request)
             response, cached = self.cache.get(cache_key, wrap_request_time(do_it))
         except openai.OpenAIError as e:
+            hexception(e)
             error: str = f"OpenAI error: {e}"
             return RequestResult(success=False, cached=False, error=error, completions=[], embedding=[])

helm/clients/palmyra_client.py CHANGED Viewed

@@ -5,7 +5,7 @@ from typing import Any, Dict, List
 from helm.clients.openai_client import OpenAIClient
 from helm.common.cache import CacheConfig
-from helm.common.hierarchical_logger import hwarn
+from helm.common.hierarchical_logger import hexception, hwarn
 from helm.common.request import wrap_request_time, Request, RequestResult, GeneratedOutput, Token, ErrorFlags
 from helm.common.tokenization_request import (
     TokenizationRequest,
@@ -99,6 +99,7 @@ class PalmyraClient(CachingClient):
                 response, cached = self.cache.get(cache_key, wrap_request_time(do_it))
             except (requests.exceptions.RequestException, AssertionError) as e:
+                hexception(e)
                 error: str = f"PalmyraClient error: {e}"
                 return RequestResult(success=False, cached=False, error=error, completions=[], embedding=[])

helm/clients/reka_client.py CHANGED Viewed

@@ -6,7 +6,7 @@ from helm.proxy.retry import NonRetriableException
 from helm.common.cache import CacheConfig
 from helm.common.media_object import TEXT_TYPE
 from helm.common.request import wrap_request_time, Request, RequestResult, GeneratedOutput
-from helm.common.hierarchical_logger import hwarn
+from helm.common.hierarchical_logger import hexception, hwarn
 from helm.common.optional_dependencies import handle_module_not_found_error
 from helm.tokenizers.tokenizer import Tokenizer
 from helm.clients.client import CachingClient, truncate_and_tokenize_response_text
@@ -167,6 +167,7 @@ class RekaClient(CachingClient):
                 response, cached = self.cache.get(raw_request, wrap_request_time(do_it))
             except (requests.exceptions.RequestException, AssertionError) as e:
+                hexception(e)
                 error: str = f"RekaClient error: {e}"
                 return RequestResult(success=False, cached=False, error=error, completions=[], embedding=[])

helm/clients/stanfordhealthcare_azure_openai_client.py CHANGED Viewed

@@ -39,7 +39,7 @@ class StanfordHealthCareAzureOpenAIClient(AzureOpenAIClient):
                 tokenizer=tokenizer,
                 tokenizer_name=tokenizer_name,
                 cache_config=cache_config,
-                api_key="unused",
+                api_key=api_key,
                 base_url=base_url,
                 azure_openai_deployment_name=openai_model_name,
                 api_version=api_version,
@@ -50,7 +50,7 @@ class StanfordHealthCareAzureOpenAIClient(AzureOpenAIClient):
                 tokenizer=tokenizer,
                 tokenizer_name=tokenizer_name,
                 cache_config=cache_config,
-                api_key="unused",
+                api_key=api_key,
                 endpoint=endpoint,
                 azure_openai_deployment_name=openai_model_name,
                 api_version=api_version,

helm/clients/stanfordhealthcare_http_model_client.py CHANGED Viewed

@@ -5,6 +5,7 @@ from dataclasses import asdict
 from typing import Any, Dict, List, Optional
 from helm.common.cache import CacheConfig
+from helm.common.hierarchical_logger import hexception
 from helm.common.request import (
     wrap_request_time,
     Request,
@@ -82,6 +83,7 @@ class StanfordHealthCareHTTPModelClient(CachingClient, ABC):
                 request_time=response["request_time"],
             )
         except requests.exceptions.RequestException as e:
+            hexception(e)
             return RequestResult(success=False, cached=False, error=f"Request error: {e}", completions=[], embedding=[])
     @abstractmethod

crfm-helm 0.5.8__py3-none-any.whl → 0.5.9__py3-none-any.whl

Potentially problematic release.

crfm-helm 0.5.8py3-none-any.whl → 0.5.9py3-none-any.whl