azure-ai-evaluation 1.3.0__py3-none-any.whl → 1.5.0__py3-none-any.whl

azure/ai/evaluation/_legacy/_batch_engine/_run_submitter.py

@@ -0,0 +1,217 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+import dataclasses
+import sys
+from datetime import datetime, timezone
+from typing import Any, Callable, Dict, Mapping, Optional, Sequence, TextIO, Union
+
+from ._run import Run, RunStatus
+from ._trace import start_trace, is_collection_writeable
+from ._run_storage import AbstractRunStorage, NoOpRunStorage
+from ._logging import incremental_print, print_red_error
+from ._config import BatchEngineConfig
+from ._exceptions import BatchEngineValidationError
+from ._engine import BatchEngine, BatchEngineError, BatchResult
+
+
+class RunSubmitter:
+    """Submits run to executor
+    promptflow-devkit/promptflow/_sdk/_orchestrator/run_submitter.py
+
+    THIS WILL BE REMOVED IN A FUTURE CODE UPDATE"""
+
+    def __init__(self, config: BatchEngineConfig):
+        # self._client = PFClient instance
+        # self._config = PFClient config
+        # self.run_operations = RunOperations instance
+
+        # TODO ralphe: Use proper logger here. Old code did LoggerFactory.get_logger(__name__)
+        self._config = config
+
+    def submit(
+        self,
+        dynamic_callable: Callable,
+        inputs: Sequence[Mapping[str, Any]],
+        column_mapping: Mapping[str, str],
+        *,
+        name_prefix: Optional[str] = None,
+        created_on: Optional[datetime] = None,
+        storage_creator: Optional[Callable[[Run], AbstractRunStorage]] = None,
+        **kwargs,
+    ) -> Run:
+        # The old code always spun up two threads here using a ThreadPoolExecutor:
+        # 1. One thread essentially did nothing of value (since tracing was disabled, and we
+        #    don't care about checking for the latest PromptFlow version number now)
+        # 2. The other thread did the _run_bulk call. This was followed by a
+        #    wait(return_when=ALL_COMPLETED)
+        # This quite frankly is unnecessary complexity since the the evaluation code already
+        # calls this in the context of ThreadPoolThread. So we can just do the equivalent
+        # of the _run_bulk code here directly.
+        # In a future code refactor, all of this will be cleaned up in favour of proper
+        # async/await code.
+        run: Run = kwargs.pop("run", None) or Run(
+            dynamic_callable=dynamic_callable,
+            name_prefix=name_prefix,
+            inputs=inputs,
+            column_mapping=column_mapping,
+            created_on=created_on,
+        )
+
+        logger = self._config.logger
+        attributes: Dict[str, Any] = kwargs.get("attributes", {})
+        collection_for_run: Optional[str] = None
+
+        logger.debug("start trace for flow run...")
+        logger.debug("flow path for run.start_trace: %s", run.name)
+
+        if is_collection_writeable():
+            logger.debug("trace collection is writeable, will use flow name as collection...")
+            collection_for_run = run.name
+            logger.debug("collection for run: %s", collection_for_run)
+        else:
+            logger.debug("trace collection is protected, will honor existing collection.")
+        start_trace(attributes=attributes, run=run, _collection=collection_for_run)
+
+        self._validate_inputs(run=run)
+
+        local_storage = storage_creator(run) if storage_creator else NoOpRunStorage()
+        with local_storage.logger:
+            run._status = RunStatus.PREPARING
+
+            # unnecessary Flow loading code was removed here. Instead do direct calls to _submit_bulk_run
+            self._submit_bulk_run(run=run, local_storage=local_storage, **kwargs)
+
+        self.stream_run(run=run, storage=local_storage, raise_on_error=True)
+        return run
+
+    def _submit_bulk_run(self, run: Run, local_storage: AbstractRunStorage, **kwargs) -> None:
+        logger = self._config.logger
+
+        logger.info(f"Submitting run {run.name}, log path: {local_storage.logger.file_path}")
+
+        # Old code loaded the Flex flow, parsed input and outputs types. That logic has been
+        # removed since it is unnecessary. It also parsed and set environment variables. This
+        # has also been removed since it can be problematic in a multi-threaded environment.
+
+        self._validate_column_mapping(run.column_mapping)
+
+        run._status = RunStatus.RUNNING
+        run._start_time = datetime.now(timezone.utc)
+        batch_result: Optional[BatchResult] = None
+
+        try:
+            batch_engine = BatchEngine(
+                run.dynamic_callable,
+                storage=local_storage,
+                batch_timeout_sec=self._config.batch_timeout_seconds,
+                line_timeout_sec=self._config.run_timeout_seconds,
+                max_worker_count=self._config.max_concurrency,
+                **kwargs,
+            )
+
+            batch_result = batch_engine.run(data=run.inputs, column_mapping=run.column_mapping, id=run.name)
+            run._status = RunStatus.from_batch_result_status(batch_result.status)
+
+            error_logs: Sequence[str] = []
+            if run._status != RunStatus.COMPLETED:
+                error_logs.append(f"Run {run.name} failed with status {batch_result.status}.")
+                if batch_result.error:
+                    error_logs.append(f"Error: {str(batch_result.error)}")
+
+            if error_logs:
+                logger.warning("\n".join(error_logs))
+        except Exception as e:
+            run._status = RunStatus.FAILED
+            # when run failed in executor, store the exception in result and dump to file
+            logger.warning(f"Run {run.name} failed when executing in executor with exception {e}.")
+            # for user error, swallow stack trace and return failed run since user don't need the stack trace
+            if not isinstance(e, BatchEngineValidationError):
+                # for other errors, raise it to user to help debug root cause.
+                raise e
+            # won't raise the exception since it's already included in run object.
+        finally:
+            # persist inputs, outputs and metrics
+            local_storage.persist_result(batch_result)
+            # exceptions
+            # local_storage.dump_exception(exception=exception, batch_result=batch_result) # TODO ralphe: persist_result should handle this
+            # system metrics
+            system_metrics = {}
+            if batch_result:
+                system_metrics.update(dataclasses.asdict(batch_result.tokens))  # token related
+                system_metrics.update(
+                    {
+                        "duration": batch_result.duration.total_seconds(),
+                        # "__pf__.lines.completed": batch_result.total_lines - batch_result.failed_lines,
+                        # "__pf__.lines.failed": batch_result.failed_lines,
+                    }
+                )
+
+            run._end_time = datetime.now(timezone.utc)
+            run.metrics = system_metrics
+            run.result = batch_result
+
+    @staticmethod
+    def _validate_inputs(run: Run):
+        if not run.inputs:
+            raise BatchEngineValidationError("Data must be specified for evaluation run.")
+
+    @staticmethod
+    def _validate_column_mapping(column_mapping: Mapping[str, str]):
+        if not isinstance(column_mapping, Mapping):
+            raise BatchEngineValidationError(f"Column mapping must be a dict, got {type(column_mapping)}.")
+
+        has_mapping = any([isinstance(v, str) and v.startswith("$") for v in column_mapping.values()])
+        if not has_mapping:
+            raise BatchEngineValidationError(
+                "Column mapping must contain at least one mapping binding, "
+                f"current column mapping contains all static values: {column_mapping}"
+            )
+
+    @staticmethod
+    def stream_run(run: Run, storage: AbstractRunStorage, raise_on_error: bool) -> None:
+        """
+        Stream the output of the batch execution.
+
+        :param Run run: The run to stream.
+        :param AbstractRunStorage storage: The storage to use for the output.
+        """
+
+        # TODO ralphe: This doesn't seem to be do anything useful beyond just print
+        #              a run summary at the end. This is because by the time it gets
+        #              invoked even in the original code, the run has already completed.
+
+        if run is None or storage is None:
+            return
+
+        file_handler = sys.stdout
+        try:
+            printed = 0
+            available_logs = storage.logger.get_logs()
+            incremental_print(available_logs, printed, file_handler)
+            RunSubmitter._print_run_summary(run, file_handler)
+        except KeyboardInterrupt:
+            error_message = "The output streaming for the run was interrupted, but the run is still executing."
+            print(error_message)
+
+        if run.status == RunStatus.FAILED or run.status == RunStatus.CANCELED:
+            if run.status == RunStatus.FAILED:
+                error_message = storage.load_exception().get("message", "Run fails with unknown error.")
+            else:
+                error_message = "Run is canceled."
+            if raise_on_error:
+                raise BatchEngineError(error_message)
+            else:
+                print_red_error(error_message)
+
+    @staticmethod
+    def _print_run_summary(run: Run, text_out: Union[TextIO, Any]) -> None:
+        duration = str(run.duration)
+        text_out.write(
+            "======= Run Summary =======\n\n"
+            f'Run name: "{run.name}"\n'
+            f'Run status: "{run.status.value}"\n'
+            f'Start time: "{run.created_on}"\n'
+            f'Duration: "{duration}"\n\n'
+        )

azure/ai/evaluation/_legacy/_batch_engine/_status.py

@@ -0,0 +1,25 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+from enum import IntEnum, auto, unique
+
+
+@unique
+class BatchStatus(IntEnum):
+    NotStarted = 0
+    Running = auto()
+
+    # NOTE: DO NOT REORDER THESE ENUMS. The order is important for the is_terminated method
+    #       and other logic in the code to work properly
+    Completed = auto()
+    Canceled = auto()
+    Failed = auto()
+
+    @staticmethod
+    def is_terminated(status: "BatchStatus") -> bool:
+        return status >= BatchStatus.Completed
+
+    @staticmethod
+    def is_failed(status: "BatchStatus") -> bool:
+        return status == BatchStatus.Failed or status == BatchStatus.Canceled

azure/ai/evaluation/_legacy/_batch_engine/_trace.py

@@ -0,0 +1,105 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+# Pretty much all this code will be removed
+
+import logging
+import os
+from typing import Any, Dict, Optional
+
+from ._openai_injector import inject_openai_api
+
+
+def start_trace(
+    *,
+    resource_attributes: Optional[Dict] = None,
+    collection: Optional[str] = None,
+    **kwargs: Any,
+) -> None:
+    """Promptflow instrumentation.
+
+    :param resource_attributes: Specify the resource attributes for current process.
+    :type resource_attributes: typing.Optional[dict]
+    :param collection: Specify the collection for current tracing.
+    :type collection: typing.Optional[str]
+    """
+
+    logging.debug("injecting OpenAI API...")
+    inject_openai_api()
+    logging.debug("OpenAI API injected.")
+
+    res_attrs: Dict[str, str] = {"service.name": "promptflow"}
+    if resource_attributes:
+        logging.debug("specified resource attributes: %s", resource_attributes)
+        res_attrs.update(resource_attributes)
+
+    # determine collection
+    collection_user_specified = collection is not None
+    if not collection_user_specified:
+        collection = kwargs.get("_collection", _get_collection_from_cwd())
+    #    logging.debug("collection is not user specified")
+    #     if is_collection_writeable():
+    #         # internal parameter for devkit call
+    #         _collection = kwargs.get("_collection", None)
+    #         if _collection is not None:
+    #             logging.debug("received internal parameter _collection: %s, will use this", _collection)
+    #             collection = _collection
+    #         else:
+    #             logging.debug("trying to get from current working directory...")
+    #             collection = _get_collection_from_cwd()
+    #     # TODO ralphe: OpenTelemetry dependency. This is a future task to resolve.
+    #     # else:
+    #     #     logging.debug("collection is protected, will directly use that...")
+    #     #     tracer_provider: TracerProvider = trace.get_tracer_provider()
+    #     #     collection = tracer_provider.resource.attributes["collection"]
+    logging.info("collection: %s", collection)
+    res_attrs["collection"] = collection or "default"
+    logging.info("resource attributes: %s", res_attrs)
+
+    # if user specifies collection, we will add a flag on tracer provider to avoid override
+    _set_tracer_provider(res_attrs, protected_collection=collection_user_specified)
+
+    # Rest of code is removed since we are removing promptflow-devkit dependency
+
+
+def is_collection_writeable() -> bool:
+    # TODO ralphe: This has OpenTelemetry dependency. That is a future task to resolve.
+    # return not getattr(trace.get_tracer_provider(), TRACER_PROVIDER_PROTECTED_COLLECTION_ATTR, False)
+    return True
+
+
+def _get_collection_from_cwd() -> str:
+    """Try to use cwd folder name as collection name; will fall back to default value if run into exception."""
+    cur_folder_name = ""
+    try:
+        cwd = os.getcwd()
+        cur_folder_name = os.path.basename(cwd)
+    except Exception:  # pylint: disable=broad-except
+        # possible exception: PermissionError, FileNotFoundError, OSError, etc.
+        pass
+    collection = cur_folder_name or "default"
+    return collection
+
+
+def _set_tracer_provider(res_attrs: Dict[str, str], protected_collection: bool) -> None:
+    # TODO ralphe: OpenTelemetry dependency. This is a future task to resolve.
+    pass
+    # res = Resource(attributes=res_attrs)
+    # tracer_provider = TracerProvider(resource=res)
+
+    # cur_tracer_provider = trace.get_tracer_provider()
+    # if isinstance(cur_tracer_provider, TracerProvider):
+    #     logging.info("tracer provider is already set, will merge the resource attributes...")
+    #     cur_res = cur_tracer_provider.resource
+    #     logging.debug("current resource: %s", cur_res.attributes)
+    #     new_res = cur_res.merge(res)
+    #     cur_tracer_provider._resource = new_res
+    #     logging.info("tracer provider is updated with resource attributes: %s", new_res.attributes)
+    # else:
+    #     trace.set_tracer_provider(tracer_provider)
+    #     logging.info("tracer provider is set with resource attributes: %s", res.attributes)
+
+    # if protected_collection:
+    #     logging.info("user specifies collection, will add a flag on tracer provider to avoid override...")
+    #     setattr(trace.get_tracer_provider(), TRACER_PROVIDER_PROTECTED_COLLECTION_ATTR, True)

azure/ai/evaluation/_legacy/_batch_engine/_utils.py

@@ -0,0 +1,82 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+import os
+import re
+from typing import Any, Mapping, Sequence, Tuple
+
+
+def normalize_identifier_name(name: str) -> str:
+    """Normalize the identifier name to a valid Python variable name.
+
+    Args:
+        name (str): The identifier name to normalize.
+
+    Returns:
+        str: The normalized identifier name.
+    """
+    normalized = re.sub(r"\W", "_", name.strip())
+    if normalized[0].isdigit():
+        normalized = f"_{normalized}"
+    return normalized
+
+
+def get_int_env_var(env_var_name: str, default_value: int = 0) -> int:
+    """Get the integer value of the environment variable.
+
+    Args:
+        env_var_name (str): The name of the environment variable.
+        default_value (int): The default value if the environment variable is not set.
+
+    Returns:
+        int: The integer value of the environment variable.
+    """
+    try:
+        value = os.getenv(env_var_name, default_value)
+        return int(value)
+    except ValueError:
+        return default_value
+
+
+def get_value_from_path(path: str, data: Mapping[str, Any]) -> Tuple[bool, Any]:
+    """Tried to get a value from a mapping based on the specified path. The path is a
+    string with dot-separated keys (e.g. data.nested_1.nested_2).
+
+    This will interpret the path prioritizing a depth first search with the shortest
+    key possible at each level. If for example you had the following data:
+    {
+        "foo": {
+            "bar": {
+                "happy": 12
+            }
+        },
+        "foo.bar": {
+            "none": 14,
+            "random": { "some": 15 }
+        },
+        "foo.bar.none": 16
+    }
+    And you asked for foo.bar.none, the returned value would be 14"
+    """
+
+    def _get_value(data: Mapping[str, Any], parts: Sequence[str]) -> Tuple[bool, Any]:
+        if len(parts) == 0:
+            return True, data
+
+        for i in range(1, len(parts) + 1):
+            key = ".".join(parts[:i])
+            if isinstance(data, Mapping) and key in data:
+                found, match = _get_value(data[key], parts[i:])
+                if found:
+                    return found, match
+
+        return False, None
+
+    if path is None or data is None:
+        return False, None
+
+    parts = path.strip().split(".")
+    if len(parts) == 0:
+        return False, None
+    return _get_value(data, parts)

azure/ai/evaluation/_legacy/_batch_engine/_utils_deprecated.py

@@ -0,0 +1,131 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+import asyncio
+import contextvars
+import dataclasses
+from asyncio import Task
+from concurrent.futures import ThreadPoolExecutor
+from typing import Any, AsyncIterator, Callable, Iterator, Mapping, Optional, Sequence, Tuple, cast
+
+
+class ThreadPoolExecutorWithContext(ThreadPoolExecutor):
+    # Original source:
+    # promptflow-tracing/promptflow/tracing/_context_utils.py
+
+    def __init__(
+        self,
+        max_workers: Optional[int] = None,
+        thread_name_prefix: str = "",
+        initializer: Optional[Callable] = None,
+        initargs: Tuple[Any, ...] = (),
+    ) -> None:
+        """The ThreadPoolExecutionWithContext is an extended thread pool implementation
+        which will copy the context from the current thread to the child threads.
+        Thus the traced functions in child threads could keep parent-child relationship in the tracing system.
+        The arguments are the same as ThreadPoolExecutor.
+
+        Args:
+            max_workers: The maximum number of threads that can be used to
+                execute the given calls.
+            thread_name_prefix: An optional name prefix to give our threads.
+            initializer: A callable used to initialize worker threads.
+            initargs: A tuple of arguments to pass to the initializer.
+        """
+        current_context = contextvars.copy_context()
+        initializer_args = (current_context, initializer, initargs)
+        super().__init__(max_workers, thread_name_prefix, self.set_context_then_call, initializer_args)
+
+    @staticmethod
+    def set_context_then_call(
+        context: contextvars.Context,
+        initializer: Optional[Callable],
+        initargs: Tuple[Any, ...],
+    ) -> None:
+        for var, value in context.items():
+            var.set(value)
+        if initializer:
+            initializer(*initargs)
+
+
+def _has_running_loop() -> bool:
+    """Check if the current thread has a running event loop."""
+    # When using asyncio.get_running_loop(), a RuntimeError is raised if there is no running event loop.
+    # So, we use a try-catch block to determine whether there is currently an event loop in place.
+    #
+    # Note that this is the only way to check whether there is a running loop now, see:
+    # https://docs.python.org/3/library/asyncio-eventloop.html?highlight=get_running_loop#asyncio.get_running_loop
+    try:
+        asyncio.get_running_loop()
+        return True
+    except RuntimeError:
+        return False
+
+
+def async_run_allowing_running_loop(async_func, *args, **kwargs):
+    """Run an async function in a new thread, allowing the current thread to have a running event loop.
+
+    When run in an async environment (e.g., in a notebook), because each thread allows only one event
+    loop, using asyncio.run directly leads to a RuntimeError ("asyncio.run() cannot be called from a
+    running event loop").
+
+    To address this issue, we add a check for the event loop here. If the current thread already has an
+    event loop, we run _exec_batch in a new thread; otherwise, we run it in the current thread.
+    """
+
+    if _has_running_loop():
+        # TODO ralphe: The logic here makes absolutely no sense to me. If you already have an
+        #              async event loop running, why would you want to start up a new thread,
+        #              create a new event loop, and run the async function in a new thread?
+        #              You can just use the following to schedule the async function call on
+        #              the existing event loop:
+        # asyncio.get_running_loop().create_task(async_func(*args, *args, **kwargs)).result()
+        #              The correct thing to do here is not make these decisions here at all.
+        #              Instead, all the BatchEngine code should be async first, with the event
+        #              loop being started by the callers of that code. For now, I am keeping
+        #              this odd logic as is, and in phase 2 of the migration, this will be
+        #              refactored to be more idiomatic asyncio code.
+        with ThreadPoolExecutorWithContext() as executor:
+            return executor.submit(lambda: asyncio.run(async_func(*args, **kwargs))).result()
+    else:
+        return asyncio.run(async_func(*args, **kwargs))
+
+
+async def stringify_output_async(output: Any) -> str:
+    if isinstance(output, AsyncIterator):
+        return await stringify_output_async([v async for v in output])
+    if isinstance(output, Iterator):
+        return await stringify_output_async([v for v in output])
+    if isinstance(output, Mapping):
+        return ", ".join(
+            [f"{await stringify_output_async(k)}:{await stringify_output_async(v)}" for k, v in output.items()]
+        )
+    if isinstance(output, Sequence):
+        return "".join([await stringify_output_async(v) for v in output])
+    if isinstance(output, Task):
+        return await stringify_output_async(await output)
+
+    return str(output)
+
+
+def convert_eager_flow_output_to_dict(value: Any) -> Mapping[str, Any]:
+    """
+    Convert the output of eager flow to a dict. Since the output of eager flow
+    may not be a dict, we need to convert it to a dict in batch mode.
+
+    Examples:
+    1. If the output is a dict, return it directly:
+        value = {"output": 1} -> {"output": 1}
+    2. If the output is a dataclass, convert it to a dict:
+        value = SampleDataClass(output=1) -> {"output": 1}
+    3. If the output is not a dict or dataclass, convert it to a dict by adding a key "output":
+        value = 1 -> {"output": 1}
+    """
+
+    if isinstance(value, Mapping):
+        return value
+    elif dataclasses.is_dataclass(value):
+        return dataclasses.asdict(cast(Any, value))
+    else:
+        return {"output": value}

azure/ai/evaluation/_legacy/prompty/__init__.py

@@ -0,0 +1,36 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+from azure.ai.evaluation._legacy.prompty._prompty import AsyncPrompty
+from azure.ai.evaluation._legacy.prompty._connection import Connection, OpenAIConnection, AzureOpenAIConnection
+from azure.ai.evaluation._legacy.prompty._exceptions import (
+    PromptyException,
+    MissingRequiredInputError,
+    InvalidInputError,
+    JinjaTemplateError,
+    NotSupportedError,
+)
+
+# =========================================================================================================
+# NOTE: All of the code here is largely copy of code from Promptflow. Generally speaking, the following
+#       changes were made:
+#       - Added type annotations
+#       - Legacy or deprecated functionality has been removed (e.g. no more support for completions API)
+#       - Reworked the way images are handled to 1) Reduce the amount of code brought over, 2) remove
+#         the need to do two passes over the template to insert images, 3) remove the completely unnecessary
+#         loading of image data from the internet when it is not actually needed
+#       - Minor obvious tweaks to improve code readability, and removal of unused code paths
+# =========================================================================================================
+
+__all__ = [
+    "AsyncPrompty",
+    "Connection",
+    "AzureOpenAIConnection",
+    "OpenAIConnection",
+    "PromptyException",
+    "MissingRequiredInputError",
+    "InvalidInputError",
+    "JinjaTemplateError",
+    "NotSupportedError",
+]