azure-ai-evaluation 1.0.1__py3-none-any.whl → 1.13.3__py3-none-any.whl

azure/ai/evaluation/_legacy/_batch_engine/_trace.py

@@ -0,0 +1,97 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+# Pretty much all this code will be removed
+
+from typing import Any, Dict, Optional
+
+
+def start_trace(
+    *,
+    resource_attributes: Optional[Dict] = None,
+    collection: Optional[str] = None,
+    **kwargs: Any,
+) -> None:
+    """Starts a trace.
+
+    :param resource_attributes: Specify the resource attributes for current process.
+    :type resource_attributes: typing.Optional[dict]
+    :param collection: Specify the collection for current tracing.
+    :type collection: typing.Optional[str]
+    """
+    pass
+
+    # res_attrs: Dict[str, str] = {"service.name": "promptflow"}
+    # if resource_attributes:
+    #     logging.debug("specified resource attributes: %s", resource_attributes)
+    #     res_attrs.update(resource_attributes)
+
+    # # determine collection
+    # collection_user_specified = collection is not None
+    # if not collection_user_specified:
+    #     collection = kwargs.get("_collection", _get_collection_from_cwd())
+    #    logging.debug("collection is not user specified")
+    #     if is_collection_writeable():
+    #         # internal parameter for devkit call
+    #         _collection = kwargs.get("_collection", None)
+    #         if _collection is not None:
+    #             logging.debug("received internal parameter _collection: %s, will use this", _collection)
+    #             collection = _collection
+    #         else:
+    #             logging.debug("trying to get from current working directory...")
+    #             collection = _get_collection_from_cwd()
+    #     # TODO ralphe: OpenTelemetry dependency. This is a future task to resolve.
+    #     # else:
+    #     #     logging.debug("collection is protected, will directly use that...")
+    #     #     tracer_provider: TracerProvider = trace.get_tracer_provider()
+    #     #     collection = tracer_provider.resource.attributes["collection"]
+    # logging.info("collection: %s", collection)
+    # res_attrs["collection"] = collection or "default"
+    # logging.info("resource attributes: %s", res_attrs)
+
+    # # if user specifies collection, we will add a flag on tracer provider to avoid override
+    # _set_tracer_provider(res_attrs, protected_collection=collection_user_specified)
+
+    # Rest of code is removed since we are removing promptflow-devkit dependency
+
+
+# def is_collection_writeable() -> bool:
+#     # TODO ralphe: This has OpenTelemetry dependency. That is a future task to resolve.
+#     # return not getattr(trace.get_tracer_provider(), TRACER_PROVIDER_PROTECTED_COLLECTION_ATTR, False)
+#     return True
+
+
+# def _get_collection_from_cwd() -> str:
+#     """Try to use cwd folder name as collection name; will fall back to default value if run into exception."""
+#     cur_folder_name = ""
+#     try:
+#         cwd = os.getcwd()
+#         cur_folder_name = os.path.basename(cwd)
+#     except Exception:  # pylint: disable=broad-except
+#         # possible exception: PermissionError, FileNotFoundError, OSError, etc.
+#         pass
+#     collection = cur_folder_name or "default"
+#     return collection
+
+
+# def _set_tracer_provider(res_attrs: Dict[str, str], protected_collection: bool) -> None:
+#     # TODO ralphe: OpenTelemetry dependency. This is a future task to resolve.
+#     # res = Resource(attributes=res_attrs)
+#     # tracer_provider = TracerProvider(resource=res)
+
+#     # cur_tracer_provider = trace.get_tracer_provider()
+#     # if isinstance(cur_tracer_provider, TracerProvider):
+#     #     logging.info("tracer provider is already set, will merge the resource attributes...")
+#     #     cur_res = cur_tracer_provider.resource
+#     #     logging.debug("current resource: %s", cur_res.attributes)
+#     #     new_res = cur_res.merge(res)
+#     #     cur_tracer_provider._resource = new_res
+#     #     logging.info("tracer provider is updated with resource attributes: %s", new_res.attributes)
+#     # else:
+#     #     trace.set_tracer_provider(tracer_provider)
+#     #     logging.info("tracer provider is set with resource attributes: %s", res.attributes)
+
+#     # if protected_collection:
+#     #     logging.info("user specifies collection, will add a flag on tracer provider to avoid override...")
+#     #     setattr(trace.get_tracer_provider(), TRACER_PROVIDER_PROTECTED_COLLECTION_ATTR, True)

azure/ai/evaluation/_legacy/_batch_engine/_utils.py

@@ -0,0 +1,97 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+import inspect
+import os
+import re
+from typing import Any, Final, Mapping, Sequence, Tuple
+
+
+DEFAULTS_KEY: Final[str] = "$defaults$"
+
+
+def normalize_identifier_name(name: str) -> str:
+    """Normalize the identifier name to a valid Python variable name.
+
+    Args:
+        name (str): The identifier name to normalize.
+
+    Returns:
+        str: The normalized identifier name.
+    """
+    normalized = re.sub(r"\W", "_", name.strip())
+    if normalized[0].isdigit():
+        normalized = f"_{normalized}"
+    return normalized
+
+
+def get_int_env_var(env_var_name: str, default_value: int = 0) -> int:
+    """Get the integer value of the environment variable.
+
+    Args:
+        env_var_name (str): The name of the environment variable.
+        default_value (int): The default value if the environment variable is not set.
+
+    Returns:
+        int: The integer value of the environment variable.
+    """
+    try:
+        value = os.getenv(env_var_name, default_value)
+        return int(value)
+    except ValueError:
+        return default_value
+
+
+def get_value_from_path(path: str, data: Mapping[str, Any]) -> Tuple[bool, Any]:
+    """Tried to get a value from a mapping based on the specified path. The path is a
+    string with dot-separated keys (e.g. data.nested_1.nested_2).
+
+    This will interpret the path prioritizing a depth first search with the shortest
+    key possible at each level. If for example you had the following data:
+    {
+        "foo": {
+            "bar": {
+                "happy": 12
+            }
+        },
+        "foo.bar": {
+            "none": 14,
+            "random": { "some": 15 }
+        },
+        "foo.bar.none": 16
+    }
+    And you asked for foo.bar.none, the returned value would be 14"
+    """
+
+    def _get_value(data: Mapping[str, Any], parts: Sequence[str]) -> Tuple[bool, Any]:
+        if len(parts) == 0:
+            return True, data
+
+        for i in range(1, len(parts) + 1):
+            key = ".".join(parts[:i])
+            if isinstance(data, Mapping) and key in data:
+                found, match = _get_value(data[key], parts[i:])
+                if found:
+                    return found, match
+
+        return False, None
+
+    if path is None or data is None:
+        return False, None
+
+    parts = path.strip().split(".")
+    if len(parts) == 0:
+        return False, None
+    return _get_value(data, parts)
+
+
+def is_async_callable(obj: Any) -> bool:
+    """Check if the object is an async callable. This will be true if the object is a coroutine function,
+    or if the object has
+
+    :param Any obj: The object to check.
+    :return: True if the object is an async callable.
+    :rtype: bool
+    """
+    return inspect.iscoroutinefunction(obj) or inspect.iscoroutinefunction(getattr(obj, "__call__", None))

azure/ai/evaluation/_legacy/_batch_engine/_utils_deprecated.py

@@ -0,0 +1,131 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+import asyncio
+import contextvars
+import dataclasses
+from asyncio import Task
+from concurrent.futures import ThreadPoolExecutor
+from typing import Any, AsyncIterator, Callable, Iterator, Mapping, Optional, Sequence, Tuple, cast
+
+
+class ThreadPoolExecutorWithContext(ThreadPoolExecutor):
+    # Original source:
+    # promptflow-tracing/promptflow/tracing/_context_utils.py
+
+    def __init__(
+        self,
+        max_workers: Optional[int] = None,
+        thread_name_prefix: str = "",
+        initializer: Optional[Callable] = None,
+        initargs: Tuple[Any, ...] = (),
+    ) -> None:
+        """The ThreadPoolExecutionWithContext is an extended thread pool implementation
+        which will copy the context from the current thread to the child threads.
+        Thus the traced functions in child threads could keep parent-child relationship in the tracing system.
+        The arguments are the same as ThreadPoolExecutor.
+
+        Args:
+            max_workers: The maximum number of threads that can be used to
+                execute the given calls.
+            thread_name_prefix: An optional name prefix to give our threads.
+            initializer: A callable used to initialize worker threads.
+            initargs: A tuple of arguments to pass to the initializer.
+        """
+        current_context = contextvars.copy_context()
+        initializer_args = (current_context, initializer, initargs)
+        super().__init__(max_workers, thread_name_prefix, self.set_context_then_call, initializer_args)
+
+    @staticmethod
+    def set_context_then_call(
+        context: contextvars.Context,
+        initializer: Optional[Callable],
+        initargs: Tuple[Any, ...],
+    ) -> None:
+        for var, value in context.items():
+            var.set(value)
+        if initializer:
+            initializer(*initargs)
+
+
+def _has_running_loop() -> bool:
+    """Check if the current thread has a running event loop."""
+    # When using asyncio.get_running_loop(), a RuntimeError is raised if there is no running event loop.
+    # So, we use a try-catch block to determine whether there is currently an event loop in place.
+    #
+    # Note that this is the only way to check whether there is a running loop now, see:
+    # https://docs.python.org/3/library/asyncio-eventloop.html?highlight=get_running_loop#asyncio.get_running_loop
+    try:
+        asyncio.get_running_loop()
+        return True
+    except RuntimeError:
+        return False
+
+
+def async_run_allowing_running_loop(async_func, *args, **kwargs):
+    """Run an async function in a new thread, allowing the current thread to have a running event loop.
+
+    When run in an async environment (e.g., in a notebook), because each thread allows only one event
+    loop, using asyncio.run directly leads to a RuntimeError ("asyncio.run() cannot be called from a
+    running event loop").
+
+    To address this issue, we add a check for the event loop here. If the current thread already has an
+    event loop, we run _exec_batch in a new thread; otherwise, we run it in the current thread.
+    """
+
+    if _has_running_loop():
+        # TODO ralphe: The logic here makes absolutely no sense to me. If you already have an
+        #              async event loop running, why would you want to start up a new thread,
+        #              create a new event loop, and run the async function in a new thread?
+        #              You can just use the following to schedule the async function call on
+        #              the existing event loop:
+        # asyncio.get_running_loop().create_task(async_func(*args, *args, **kwargs)).result()
+        #              The correct thing to do here is not make these decisions here at all.
+        #              Instead, all the BatchEngine code should be async first, with the event
+        #              loop being started by the callers of that code. For now, I am keeping
+        #              this odd logic as is, and in phase 2 of the migration, this will be
+        #              refactored to be more idiomatic asyncio code.
+        with ThreadPoolExecutorWithContext() as executor:
+            return executor.submit(lambda: asyncio.run(async_func(*args, **kwargs))).result()
+    else:
+        return asyncio.run(async_func(*args, **kwargs))
+
+
+async def stringify_output_async(output: Any) -> str:
+    if isinstance(output, AsyncIterator):
+        return await stringify_output_async([v async for v in output])
+    if isinstance(output, Iterator):
+        return await stringify_output_async([v for v in output])
+    if isinstance(output, Mapping):
+        return ", ".join(
+            [f"{await stringify_output_async(k)}:{await stringify_output_async(v)}" for k, v in output.items()]
+        )
+    if isinstance(output, Sequence):
+        return "".join([await stringify_output_async(v) for v in output])
+    if isinstance(output, Task):
+        return await stringify_output_async(await output)
+
+    return str(output)
+
+
+def convert_eager_flow_output_to_dict(value: Any) -> Mapping[str, Any]:
+    """
+    Convert the output of eager flow to a dict. Since the output of eager flow
+    may not be a dict, we need to convert it to a dict in batch mode.
+
+    Examples:
+    1. If the output is a dict, return it directly:
+        value = {"output": 1} -> {"output": 1}
+    2. If the output is a dataclass, convert it to a dict:
+        value = SampleDataClass(output=1) -> {"output": 1}
+    3. If the output is not a dict or dataclass, convert it to a dict by adding a key "output":
+        value = 1 -> {"output": 1}
+    """
+
+    if isinstance(value, Mapping):
+        return value
+    elif dataclasses.is_dataclass(value):
+        return dataclasses.asdict(cast(Any, value))
+    else:
+        return {"output": value}

azure/ai/evaluation/_legacy/_common/__init__.py

@@ -0,0 +1,3 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------

azure/ai/evaluation/_legacy/_common/_async_token_provider.py

@@ -0,0 +1,117 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+import os
+from typing import Any, AsyncContextManager, Optional
+
+from azure.core.credentials import AccessToken, TokenCredential
+from azure.identity import AzureCliCredential, DefaultAzureCredential, ManagedIdentityCredential
+
+from azure.ai.evaluation._exceptions import EvaluationException, ErrorBlame, ErrorCategory, ErrorTarget
+from azure.ai.evaluation._azure._envs import AzureEnvironmentClient
+
+
+class AsyncAzureTokenProvider(AsyncContextManager["AsyncAzureTokenProvider"]):
+    """Asynchronous token provider for Azure services that supports non-default Azure clouds
+    (e.g. Azure China, Azure US Government, etc.)."""
+
+    def __init__(self, *, base_url: Optional[str] = None, **kwargs: Any) -> None:
+        """Initialize the AsyncAzureTokenProvider."""
+        self._credential: Optional[TokenCredential] = None
+        self._env_client: Optional[AzureEnvironmentClient] = AzureEnvironmentClient(base_url=base_url, **kwargs)
+
+    async def close(self) -> None:
+        if self._env_client:
+            await self._env_client.close()
+            self._env_client = None
+
+        self._credential = None
+
+    async def get_token(
+        self,
+        *scopes: str,
+        claims: Optional[str] = None,
+        tenant_id: Optional[str] = None,
+        enable_cae: bool = False,
+        **kwargs: Any,
+    ) -> AccessToken:
+        if self._credential is None:
+            self._credential = await self._initialize_async(self._env_client)
+
+        if self._credential is None:
+            raise EvaluationException(
+                f"{self.__class__.__name__} could not determine the credential to use.",
+                target=ErrorTarget.UNKNOWN,
+                category=ErrorCategory.INVALID_VALUE,
+                blame=ErrorBlame.SYSTEM_ERROR,
+            )
+
+        return self._credential.get_token(*scopes, claims=claims, tenant_id=tenant_id, enable_cae=enable_cae, **kwargs)
+
+    async def __aenter__(self) -> "AsyncAzureTokenProvider":
+        self._credential = await self._initialize_async(self._env_client)
+        return self
+
+    async def __aexit__(
+        self,
+        exc_type: Optional[type] = None,
+        exc_value: Optional[BaseException] = None,
+        traceback: Optional[Any] = None,
+    ) -> None:
+        await self.close()
+
+    @staticmethod
+    async def _initialize_async(client: Optional[AzureEnvironmentClient]) -> TokenCredential:
+        # Determine which credential to use based on the configured Azure cloud environment variables
+        # and possibly making network calls to Azure to get the correct Azure cloud metadata.
+        if client is None:
+            raise EvaluationException(
+                f"{AsyncAzureTokenProvider.__name__} instance has already been closed.",
+                target=ErrorTarget.UNKNOWN,
+                category=ErrorCategory.INVALID_VALUE,
+                blame=ErrorBlame.USER_ERROR,
+            )
+
+        cloud_name: str = await client.get_default_cloud_name_async()
+        if cloud_name != client.DEFAULT_AZURE_CLOUD_NAME:
+            # If the cloud name is not the default, we need to get the metadata for the specified cloud
+            # and set it in the environment client.
+            metadata = await client.get_cloud_async(cloud_name)
+            if metadata is None:
+                raise EvaluationException(
+                    f"Failed to get metadata for cloud '{cloud_name}'.",
+                    target=ErrorTarget.UNKNOWN,
+                    category=ErrorCategory.INVALID_VALUE,
+                    blame=ErrorBlame.USER_ERROR,
+                )
+
+            authority = metadata.get("active_directory_endpoint")
+            return DefaultAzureCredential(authority=authority, exclude_shared_token_cache_credential=True)
+        elif os.getenv("AZUREML_OBO_ENABLED"):
+            # using Azure on behalf of credentials requires the use of the azure-ai-ml package
+            try:
+                from azure.ai.ml.identity import AzureMLOnBehalfOfCredential
+
+                return AzureMLOnBehalfOfCredential()  # type: ignore
+            except (ModuleNotFoundError, ImportError):
+                raise EvaluationException(  # pylint: disable=raise-missing-from
+                    message=(
+                        "The required packages for OBO credentials are missing.\n"
+                        'To resolve this, please install them by running "pip install azure-ai-ml".'
+                    ),
+                    target=ErrorTarget.EVALUATE,
+                    category=ErrorCategory.MISSING_PACKAGE,
+                    blame=ErrorBlame.USER_ERROR,
+                )
+        elif os.environ.get("PF_USE_AZURE_CLI_CREDENTIAL", "false").lower() == "true":
+            # TODO ralphe: Is this still needed? DefaultAzureCredential already includes CLI credentials
+            #              albeit with a lower priority
+            return AzureCliCredential()
+        elif os.environ.get("IS_IN_CI_PIPELINE", "false").lower() == "true":
+            # use managed identity when executing in CI pipeline.
+            return AzureCliCredential()
+        elif identity_client_id := os.environ.get("DEFAULT_IDENTITY_CLIENT_ID"):
+            return ManagedIdentityCredential(client_id=identity_client_id)
+        else:
+            return DefaultAzureCredential()