azure-ai-evaluation 1.5.0__py3-none-any.whl → 1.7.0__py3-none-any.whl

azure/ai/evaluation/_legacy/_batch_engine/_trace.py

@@ -4,12 +4,8 @@
 
 # Pretty much all this code will be removed
 
-import logging
-import os
 from typing import Any, Dict, Optional
 
-from ._openai_injector import inject_openai_api
-
 
 def start_trace(
     *,
@@ -17,27 +13,24 @@ def start_trace(
     collection: Optional[str] = None,
     **kwargs: Any,
 ) -> None:
-    """Promptflow instrumentation.
+    """Starts a trace.
 
     :param resource_attributes: Specify the resource attributes for current process.
     :type resource_attributes: typing.Optional[dict]
     :param collection: Specify the collection for current tracing.
     :type collection: typing.Optional[str]
     """
+    pass
 
-    logging.debug("injecting OpenAI API...")
-    inject_openai_api()
-    logging.debug("OpenAI API injected.")
-
-    res_attrs: Dict[str, str] = {"service.name": "promptflow"}
-    if resource_attributes:
-        logging.debug("specified resource attributes: %s", resource_attributes)
-        res_attrs.update(resource_attributes)
+    # res_attrs: Dict[str, str] = {"service.name": "promptflow"}
+    # if resource_attributes:
+    #     logging.debug("specified resource attributes: %s", resource_attributes)
+    #     res_attrs.update(resource_attributes)
 
-    # determine collection
-    collection_user_specified = collection is not None
-    if not collection_user_specified:
-        collection = kwargs.get("_collection", _get_collection_from_cwd())
+    # # determine collection
+    # collection_user_specified = collection is not None
+    # if not collection_user_specified:
+    #     collection = kwargs.get("_collection", _get_collection_from_cwd())
     #    logging.debug("collection is not user specified")
     #     if is_collection_writeable():
     #         # internal parameter for devkit call
@@ -53,53 +46,52 @@ def start_trace(
     #     #     logging.debug("collection is protected, will directly use that...")
     #     #     tracer_provider: TracerProvider = trace.get_tracer_provider()
     #     #     collection = tracer_provider.resource.attributes["collection"]
-    logging.info("collection: %s", collection)
-    res_attrs["collection"] = collection or "default"
-    logging.info("resource attributes: %s", res_attrs)
+    # logging.info("collection: %s", collection)
+    # res_attrs["collection"] = collection or "default"
+    # logging.info("resource attributes: %s", res_attrs)
 
-    # if user specifies collection, we will add a flag on tracer provider to avoid override
-    _set_tracer_provider(res_attrs, protected_collection=collection_user_specified)
+    # # if user specifies collection, we will add a flag on tracer provider to avoid override
+    # _set_tracer_provider(res_attrs, protected_collection=collection_user_specified)
 
     # Rest of code is removed since we are removing promptflow-devkit dependency
 
 
-def is_collection_writeable() -> bool:
-    # TODO ralphe: This has OpenTelemetry dependency. That is a future task to resolve.
-    # return not getattr(trace.get_tracer_provider(), TRACER_PROVIDER_PROTECTED_COLLECTION_ATTR, False)
-    return True
-
-
-def _get_collection_from_cwd() -> str:
-    """Try to use cwd folder name as collection name; will fall back to default value if run into exception."""
-    cur_folder_name = ""
-    try:
-        cwd = os.getcwd()
-        cur_folder_name = os.path.basename(cwd)
-    except Exception:  # pylint: disable=broad-except
-        # possible exception: PermissionError, FileNotFoundError, OSError, etc.
-        pass
-    collection = cur_folder_name or "default"
-    return collection
-
-
-def _set_tracer_provider(res_attrs: Dict[str, str], protected_collection: bool) -> None:
-    # TODO ralphe: OpenTelemetry dependency. This is a future task to resolve.
-    pass
-    # res = Resource(attributes=res_attrs)
-    # tracer_provider = TracerProvider(resource=res)
-
-    # cur_tracer_provider = trace.get_tracer_provider()
-    # if isinstance(cur_tracer_provider, TracerProvider):
-    #     logging.info("tracer provider is already set, will merge the resource attributes...")
-    #     cur_res = cur_tracer_provider.resource
-    #     logging.debug("current resource: %s", cur_res.attributes)
-    #     new_res = cur_res.merge(res)
-    #     cur_tracer_provider._resource = new_res
-    #     logging.info("tracer provider is updated with resource attributes: %s", new_res.attributes)
-    # else:
-    #     trace.set_tracer_provider(tracer_provider)
-    #     logging.info("tracer provider is set with resource attributes: %s", res.attributes)
-
-    # if protected_collection:
-    #     logging.info("user specifies collection, will add a flag on tracer provider to avoid override...")
-    #     setattr(trace.get_tracer_provider(), TRACER_PROVIDER_PROTECTED_COLLECTION_ATTR, True)
+# def is_collection_writeable() -> bool:
+#     # TODO ralphe: This has OpenTelemetry dependency. That is a future task to resolve.
+#     # return not getattr(trace.get_tracer_provider(), TRACER_PROVIDER_PROTECTED_COLLECTION_ATTR, False)
+#     return True
+
+
+# def _get_collection_from_cwd() -> str:
+#     """Try to use cwd folder name as collection name; will fall back to default value if run into exception."""
+#     cur_folder_name = ""
+#     try:
+#         cwd = os.getcwd()
+#         cur_folder_name = os.path.basename(cwd)
+#     except Exception:  # pylint: disable=broad-except
+#         # possible exception: PermissionError, FileNotFoundError, OSError, etc.
+#         pass
+#     collection = cur_folder_name or "default"
+#     return collection
+
+
+# def _set_tracer_provider(res_attrs: Dict[str, str], protected_collection: bool) -> None:
+#     # TODO ralphe: OpenTelemetry dependency. This is a future task to resolve.
+#     # res = Resource(attributes=res_attrs)
+#     # tracer_provider = TracerProvider(resource=res)
+
+#     # cur_tracer_provider = trace.get_tracer_provider()
+#     # if isinstance(cur_tracer_provider, TracerProvider):
+#     #     logging.info("tracer provider is already set, will merge the resource attributes...")
+#     #     cur_res = cur_tracer_provider.resource
+#     #     logging.debug("current resource: %s", cur_res.attributes)
+#     #     new_res = cur_res.merge(res)
+#     #     cur_tracer_provider._resource = new_res
+#     #     logging.info("tracer provider is updated with resource attributes: %s", new_res.attributes)
+#     # else:
+#     #     trace.set_tracer_provider(tracer_provider)
+#     #     logging.info("tracer provider is set with resource attributes: %s", res.attributes)
+
+#     # if protected_collection:
+#     #     logging.info("user specifies collection, will add a flag on tracer provider to avoid override...")
+#     #     setattr(trace.get_tracer_provider(), TRACER_PROVIDER_PROTECTED_COLLECTION_ATTR, True)

azure/ai/evaluation/_legacy/_batch_engine/_utils.py

@@ -2,9 +2,13 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
 
+import inspect
 import os
 import re
-from typing import Any, Mapping, Sequence, Tuple
+from typing import Any, Final, Mapping, Sequence, Tuple
+
+
+DEFAULTS_KEY: Final[str] = "$defaults$"
 
 
 def normalize_identifier_name(name: str) -> str:
@@ -80,3 +84,17 @@ def get_value_from_path(path: str, data: Mapping[str, Any]) -> Tuple[bool, Any]:
     if len(parts) == 0:
         return False, None
     return _get_value(data, parts)
+
+
+def is_async_callable(obj: Any) -> bool:
+    """Check if the object is an async callable. This will be true if the object is a coroutine function,
+    or if the object has
+
+    :param Any obj: The object to check.
+    :return: True if the object is an async callable.
+    :rtype: bool
+    """
+    return (
+        inspect.iscoroutinefunction(obj)
+        or inspect.iscoroutinefunction(getattr(obj, "__call__", None))
+    )

azure/ai/evaluation/_legacy/_common/__init__.py

@@ -0,0 +1,3 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------

azure/ai/evaluation/_legacy/_common/_async_token_provider.py

@@ -0,0 +1,124 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+import os
+from typing import Any, AsyncContextManager, Optional
+
+from azure.core.credentials import AccessToken, TokenCredential
+from azure.identity import AzureCliCredential, DefaultAzureCredential, ManagedIdentityCredential
+
+from azure.ai.evaluation._exceptions import EvaluationException, ErrorBlame, ErrorCategory, ErrorTarget
+from azure.ai.evaluation._azure._envs import AzureEnvironmentClient
+
+class AsyncAzureTokenProvider(AsyncContextManager["AsyncAzureTokenProvider"]):
+    """Asynchronous token provider for Azure services that supports non-default Azure clouds
+    (e.g. Azure China, Azure US Government, etc.)."""
+
+    def __init__(
+        self,
+        *,
+        base_url: Optional[str] = None,
+        **kwargs: Any
+    ) -> None:
+        """Initialize the AsyncAzureTokenProvider."""
+        self._credential: Optional[TokenCredential] = None
+        self._env_client: Optional[AzureEnvironmentClient] = AzureEnvironmentClient(
+            base_url=base_url,
+            **kwargs)
+
+    async def close(self) -> None:
+        if self._env_client:
+            await self._env_client.close()
+            self._env_client = None
+
+        self._credential = None
+
+    async def get_token(
+        self,
+        *scopes: str,
+        claims: Optional[str] = None,
+        tenant_id: Optional[str] = None,
+        enable_cae: bool = False,
+        **kwargs: Any,
+    ) -> AccessToken:
+        if self._credential is None:
+            self._credential = await self._initialize_async(self._env_client)
+
+        if self._credential is None:
+            raise EvaluationException(
+                f"{self.__class__.__name__} could not determine the credential to use.",
+                target=ErrorTarget.UNKNOWN,
+                category=ErrorCategory.INVALID_VALUE,
+                blame=ErrorBlame.SYSTEM_ERROR)
+
+        return self._credential.get_token(
+            *scopes,
+            claims=claims,
+            tenant_id=tenant_id,
+            enable_cae=enable_cae,
+            **kwargs)
+
+    async def __aenter__(self) -> "AsyncAzureTokenProvider":
+        self._credential = await self._initialize_async(self._env_client)
+        return self
+
+    async def __aexit__(
+        self,
+        exc_type: Optional[type] = None,
+        exc_value: Optional[BaseException] = None,
+        traceback: Optional[Any] = None
+    ) -> None:
+        await self.close()
+
+    @staticmethod
+    async def _initialize_async(client: Optional[AzureEnvironmentClient]) -> TokenCredential:
+        # Determine which credential to use based on the configured Azure cloud environment variables
+        # and possibly making network calls to Azure to get the correct Azure cloud metadata.
+        if client is None:
+            raise EvaluationException(
+                f"{AsyncAzureTokenProvider.__name__} instance has already been closed.",
+                target=ErrorTarget.UNKNOWN,
+                category=ErrorCategory.INVALID_VALUE,
+                blame=ErrorBlame.USER_ERROR)
+
+        cloud_name: str = await client.get_default_cloud_name_async()
+        if cloud_name != client.DEFAULT_AZURE_CLOUD_NAME:
+            # If the cloud name is not the default, we need to get the metadata for the specified cloud
+            # and set it in the environment client.
+            metadata = await client.get_cloud_async(cloud_name)
+            if metadata is None:
+                raise EvaluationException(
+                    f"Failed to get metadata for cloud '{cloud_name}'.",
+                    target=ErrorTarget.UNKNOWN,
+                    category=ErrorCategory.INVALID_VALUE,
+                    blame=ErrorBlame.USER_ERROR)
+
+            authority = metadata.get("active_directory_endpoint")
+            return DefaultAzureCredential(authority=authority, exclude_shared_token_cache_credential=True)
+        elif os.getenv("AZUREML_OBO_ENABLED"):
+            # using Azure on behalf of credentials requires the use of the azure-ai-ml package
+            try:
+                from azure.ai.ml.identity import AzureMLOnBehalfOfCredential
+                return AzureMLOnBehalfOfCredential()  # type: ignore
+            except (ModuleNotFoundError, ImportError):
+                raise EvaluationException(  # pylint: disable=raise-missing-from
+                    message=(
+                        "The required packages for OBO credentials are missing.\n"
+                        'To resolve this, please install them by running "pip install azure-ai-ml".'
+                    ),
+                    target=ErrorTarget.EVALUATE,
+                    category=ErrorCategory.MISSING_PACKAGE,
+                    blame=ErrorBlame.USER_ERROR,
+                )
+        elif os.environ.get("PF_USE_AZURE_CLI_CREDENTIAL", "false").lower() == "true":
+            # TODO ralphe: Is this still needed? DefaultAzureCredential already includes CLI credentials
+            #              albeit with a lower priority
+            return AzureCliCredential()
+        elif os.environ.get("IS_IN_CI_PIPELINE", "false").lower() == "true":
+            # use managed identity when executing in CI pipeline.
+            return AzureCliCredential()
+        elif identity_client_id := os.environ.get("DEFAULT_IDENTITY_CLIENT_ID"):
+            return ManagedIdentityCredential(client_id=identity_client_id)
+        else:
+            return DefaultAzureCredential()

azure/ai/evaluation/_legacy/_common/_thread_pool_executor_with_context.py

@@ -0,0 +1,15 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+import contextvars
+from concurrent.futures import ThreadPoolExecutor
+from functools import partial
+from typing_extensions import override
+
+class ThreadPoolExecutorWithContext(ThreadPoolExecutor):
+    """ThreadPoolExecutor that preserves context variables across threads."""
+    @override
+    def submit(self, fn, *args, **kwargs):
+        context = contextvars.copy_context()
+        return super().submit(context.run, partial(fn, *args, **kwargs))

azure/ai/evaluation/_legacy/prompty/_connection.py

@@ -3,40 +3,16 @@
 # ---------------------------------------------------------
 
 import os
-import re
 from abc import ABC, abstractmethod
 from dataclasses import dataclass
-from typing import Any, ClassVar, Mapping, Optional, Set, Union
+from typing import Any, ClassVar, Mapping, Optional
 
 from azure.ai.evaluation._legacy.prompty._exceptions import MissingRequiredInputError
 from azure.ai.evaluation._legacy.prompty._utils import dataclass_from_dict
 
 
-ENV_VAR_PATTERN = re.compile(r"^\$\{env:(.*)\}$")
-
-
-def _parse_environment_variable(value: Union[str, Any]) -> Union[str, Any]:
-    """Get environment variable from ${env:ENV_NAME}. If not found, return original value.
-
-    :param value: The value to parse.
-    :type value: str | Any
-    :return: The parsed value
-    :rtype: str | Any"""
-    if not isinstance(value, str):
-        return value
-
-    result = re.match(ENV_VAR_PATTERN, value)
-    if result:
-        env_name = result.groups()[0]
-        return os.environ.get(env_name, value)
-
-    return value
-
-
 def _is_empty_connection_config(connection_dict: Mapping[str, Any]) -> bool:
-    ignored_fields = set(["azure_deployment", "model"])
-    keys = {k for k, v in connection_dict.items() if v}
-    return len(keys - ignored_fields) == 0
+    return any(key not in {"azure_deployment", "model", "type"} for key in connection_dict.keys())
 
 
 @dataclass
@@ -52,16 +28,6 @@ class Connection(ABC):
         :rtype: str"""
         ...
 
-    @abstractmethod
-    def is_valid(self, missing_fields: Optional[Set[str]] = None) -> bool:
-        """Check if the connection is valid.
-
-        :param missing_fields: If set, this will be populated with the missing required fields.
-        :type missing_fields: Set[str] | None
-        :return: True if the connection is valid, False otherwise.
-        :rtype: bool"""
-        ...
-
     @staticmethod
     def parse_from_config(model_configuration: Mapping[str, Any]) -> "Connection":
         """Parse a connection from a model configuration.
@@ -71,18 +37,18 @@ class Connection(ABC):
         :return: The connection.
         :rtype: Connection
         """
-        connection: Connection
-        connection_dict = {k: _parse_environment_variable(v) for k, v in model_configuration.items()}
+        connection_dict = {**model_configuration}
         connection_type = connection_dict.pop("type", "")
 
+        connection: Connection
         if connection_type in [AzureOpenAIConnection.TYPE, "azure_openai"]:
-            if _is_empty_connection_config(connection_dict):
+            if not _is_empty_connection_config(connection_dict):
                 connection = AzureOpenAIConnection.from_env()
             else:
                 connection = dataclass_from_dict(AzureOpenAIConnection, connection_dict)
 
         elif connection_type in [OpenAIConnection.TYPE, "openai"]:
-            if _is_empty_connection_config(connection_dict):
+            if not _is_empty_connection_config(connection_dict):
                 connection = OpenAIConnection.from_env()
             else:
                 connection = dataclass_from_dict(OpenAIConnection, connection_dict)
@@ -94,13 +60,6 @@ class Connection(ABC):
             )
             raise MissingRequiredInputError(error_message)
 
-        missing_fields: Set[str] = set()
-        if not connection.is_valid(missing_fields):
-            raise MissingRequiredInputError(
-                f"The following required fields are missing for connection {connection.type}: "
-                f"{', '.join(missing_fields)}"
-            )
-
         return connection
 
 
@@ -109,6 +68,7 @@ class OpenAIConnection(Connection):
     """Connection class for OpenAI endpoints."""
 
     base_url: str
+    model: str
     api_key: Optional[str] = None
     organization: Optional[str] = None
 
@@ -122,29 +82,19 @@ class OpenAIConnection(Connection):
     def from_env(cls) -> "OpenAIConnection":
         return cls(
             base_url=os.environ.get("OPENAI_BASE_URL", ""),
+            model=os.environ.get("OPENAI_MODEL", ""),
             api_key=os.environ.get("OPENAI_API_KEY"),
             organization=os.environ.get("OPENAI_ORG_ID"),
         )
 
-    def is_valid(self, missing_fields: Optional[Set[str]] = None) -> bool:
-        if missing_fields is None:
-            missing_fields = set()
-        if not self.base_url:
-            missing_fields.add("base_url")
-        if not self.api_key:
-            missing_fields.add("api_key")
-        if not self.organization:
-            missing_fields.add("organization")
-        return not bool(missing_fields)
-
 
 @dataclass
 class AzureOpenAIConnection(Connection):
     """Connection class for Azure OpenAI endpoints."""
 
     azure_endpoint: str
-    api_key: Optional[str] = None  # TODO ralphe: Replace this TokenCredential to allow for more flexible authentication
-    azure_deployment: Optional[str] = None
+    azure_deployment: str
+    api_key: Optional[str] = None
     api_version: Optional[str] = None
     resource_id: Optional[str] = None
 
@@ -158,8 +108,8 @@ class AzureOpenAIConnection(Connection):
     def from_env(cls) -> "AzureOpenAIConnection":
         return cls(
             azure_endpoint=os.environ.get("AZURE_OPENAI_ENDPOINT", ""),
+            azure_deployment=os.environ.get("AZURE_OPENAI_DEPLOYMENT", ""),
             api_key=os.environ.get("AZURE_OPENAI_API_KEY"),
-            azure_deployment=os.environ.get("AZURE_OPENAI_DEPLOYMENT"),
             api_version=os.environ.get("AZURE_OPENAI_API_VERSION", "2024-02-01"),
         )
 
@@ -167,16 +117,3 @@ class AzureOpenAIConnection(Connection):
         # set default API version
         if not self.api_version:
             self.api_version = "2024-02-01"
-
-    def is_valid(self, missing_fields: Optional[Set[str]] = None) -> bool:
-        if missing_fields is None:
-            missing_fields = set()
-        if not self.azure_endpoint:
-            missing_fields.add("azure_endpoint")
-        if not self.api_key:
-            missing_fields.add("api_key")
-        if not self.azure_deployment:
-            missing_fields.add("azure_deployment")
-        if not self.api_version:
-            missing_fields.add("api_version")
-        return not bool(missing_fields)

azure/ai/evaluation/_legacy/prompty/_exceptions.py

@@ -2,6 +2,8 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
 
+from typing import Optional
+from openai import OpenAIError
 from azure.ai.evaluation._exceptions import ErrorCategory, ErrorBlame, ErrorTarget, EvaluationException
 
 
@@ -57,3 +59,81 @@ class NotSupportedError(PromptyException):
         kwargs.setdefault("target", ErrorTarget.UNKNOWN)
         kwargs.setdefault("blame", ErrorBlame.SYSTEM_ERROR)
         super().__init__(message, **kwargs)
+
+
+class WrappedOpenAIError(PromptyException):
+    """Exception raised when an OpenAI error is encountered."""
+
+    def __init__(self, *, message: Optional[str] = None, error: Optional[OpenAIError] = None, **kwargs):
+        kwargs.setdefault("category", ErrorCategory.FAILED_EXECUTION)
+        kwargs.setdefault("target", ErrorTarget.EVAL_RUN)
+        kwargs.setdefault("blame", ErrorBlame.USER_ERROR)
+
+        message = (
+            message or self.to_openai_error_message(error)
+            if error
+            else "An error occurred while executing the OpenAI API."
+        )
+        super().__init__(message, **kwargs)
+
+    @staticmethod
+    def to_openai_error_message(e: OpenAIError) -> str:
+        # TODO ralphe: Error handling that relies on string matching is fragile and should be replaced
+        #              with a more robust solution that examines the actual error type since that provies
+        #              more than enough information to handle errors.
+        ex_type = type(e).__name__
+        error_message = str(e)
+        # https://learn.microsoft.com/en-gb/azure/ai-services/openai/reference
+        if error_message == "<empty message>":
+            msg = "The api key is invalid or revoked. " "You can correct or regenerate the api key of your connection."
+            return f"OpenAI API hits {ex_type}: {msg}"
+        # for models that do not support the `functions` parameter.
+        elif "Unrecognized request argument supplied: functions" in error_message:
+            msg = (
+                "Current model does not support the `functions` parameter. If you are using openai connection, then "
+                "please use gpt-3.5-turbo, gpt-4, gpt-4-32k, gpt-3.5-turbo-0613 or gpt-4-0613. You can refer to "
+                "https://platform.openai.com/docs/guides/gpt/function-calling. If you are using azure openai "
+                "connection, then please first go to your Azure OpenAI resource, deploy model 'gpt-35-turbo' or "
+                "'gpt-4' with version 0613. You can refer to "
+                "https://learn.microsoft.com/en-us/azure/ai-services/openai/how-to/function-calling."
+            )
+            return f"OpenAI API hits {ex_type}: {msg}"
+        elif "Invalid content type. image_url is only supported by certain models" in error_message:
+            msg = (
+                "Current model does not support the image input. If you are using openai connection, then please use "
+                "gpt-4-vision-preview. You can refer to https://platform.openai.com/docs/guides/vision."
+                "If you are using azure openai connection, then please first go to your Azure OpenAI resource, "
+                'create a GPT-4 Turbo with Vision deployment by selecting model name: "gpt-4" and '
+                'model version "vision-preview". You can refer to '
+                "https://learn.microsoft.com/en-us/azure/ai-services/openai/how-to/gpt-with-vision"
+            )
+            return f"OpenAI API hits {ex_type}: {msg}"
+        elif (
+            "'response_format' of type" in error_message and "is not supported with this model." in error_message
+        ) or (
+            "Additional properties are not allowed" in error_message
+            and "unexpected) - 'response_format'" in error_message
+        ):
+            msg = (
+                'The response_format parameter needs to be a dictionary such as {"type": "text"}. '
+                "The value associated with the type key should be either 'text' or 'json_object' "
+                'If you are using openai connection, you can only set response_format to { "type": "json_object" } '
+                "when calling gpt-3.5-turbo-1106 or gpt-4-1106-preview to enable JSON mode. You can refer to "
+                "https://platform.openai.com/docs/guides/text-generation/json-mode. If you are using azure openai "
+                "connection, then please first go to your Azure OpenAI resource, compatible with GPT-4 Turbo and "
+                "all GPT-3.5 Turbo models newer than gpt-35-turbo-1106. You can refer to "
+                "https://learn.microsoft.com/en-us/azure/ai-services/openai/how-to/json-mode?tabs=python."
+            )
+            return f"OpenAI API hits {ex_type}: {msg}"
+        elif "Principal does not have access to API/Operation" in error_message:
+            msg = (
+                "Principal does not have access to API/Operation. If you are using azure openai connection, "
+                "please make sure you have proper role assignment on your azure openai resource. You can refer to "
+                "https://learn.microsoft.com/en-us/azure/ai-services/openai/how-to/role-based-access-control"
+            )
+            return f"OpenAI API hits {ex_type}: {msg}"
+        else:
+            return (
+                f"OpenAI API hits {ex_type}: {error_message} [Error reference: "
+                "https://platform.openai.com/docs/guides/error-codes/api-errors]"
+            )