azure-ai-evaluation 1.0.0b2__py3-none-any.whl → 1.13.3__py3-none-any.whl

azure/ai/evaluation/_legacy/_batch_engine/_run_submitter.py

@@ -0,0 +1,262 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+import dataclasses
+import inspect
+import sys
+import traceback
+
+from concurrent.futures import Executor
+from datetime import datetime, timezone
+from typing import Any, Callable, Dict, Mapping, Optional, Sequence, TextIO, Union
+
+from ._run import Run, RunStatus
+from ._trace import start_trace
+from ._run_storage import AbstractRunStorage, NoOpRunStorage
+from .._common._logging import incremental_print, print_red_error
+from ._config import BatchEngineConfig
+from ._exceptions import BatchEngineValidationError
+from ._engine import DEFAULTS_KEY, BatchEngine, BatchEngineError, BatchResult, BatchStatus
+
+
+class RunSubmitter:
+    """Submits run to executor
+    promptflow-devkit/promptflow/_sdk/_orchestrator/run_submitter.py
+
+    THIS WILL BE REMOVED IN A FUTURE CODE UPDATE"""
+
+    def __init__(self, config: BatchEngineConfig, executor: Optional[Executor] = None):
+        # self._client = PFClient instance
+        # self._config = PFClient config
+        # self.run_operations = RunOperations instance
+
+        # TODO ralphe: Use proper logger here. Old code did LoggerFactory.get_logger(__name__)
+        self._config = config
+        self._executor = executor
+
+    async def submit(
+        self,
+        dynamic_callable: Callable,
+        inputs: Sequence[Mapping[str, Any]],
+        column_mapping: Optional[Mapping[str, str]],
+        *,
+        name_prefix: Optional[str] = None,
+        created_on: Optional[datetime] = None,
+        storage_creator: Optional[Callable[[Run], AbstractRunStorage]] = None,
+        **kwargs,
+    ) -> Run:
+
+        # The old code always spun up two threads here using a ThreadPoolExecutor:
+        # 1. One thread essentially did nothing of value (since tracing was disabled, and we
+        #    don't care about checking for the latest PromptFlow version number now)
+        # 2. The other thread did the _run_bulk call. This was followed by a
+        #    wait(return_when=ALL_COMPLETED)
+        # This quite frankly is unnecessary complexity since the the evaluation code already
+        # calls this in the context of ThreadPoolThread. So we can just do the equivalent
+        # of the _run_bulk code here directly.
+        # In a future code refactor, all of this will be cleaned up in favour of proper
+        # async/await code.
+
+        run: Run = Run(
+            dynamic_callable=dynamic_callable,
+            name_prefix=name_prefix,
+            inputs=inputs,
+            column_mapping=column_mapping,
+            created_on=created_on,
+            run=kwargs.pop("run", None),
+        )
+
+        attributes: Dict[str, Any] = kwargs.get("attributes", {})
+        collection_for_run: str = run.name
+        start_trace(attributes=attributes, run=run, _collection=collection_for_run)
+
+        self._validate_inputs(run=run)
+
+        local_storage = storage_creator(run) if storage_creator else NoOpRunStorage()
+        with local_storage.logger:
+            run._status = RunStatus.PREPARING
+
+            # unnecessary Flow loading code was removed here. Instead do direct calls to _submit_bulk_run
+            await self._submit_bulk_run(run=run, local_storage=local_storage, **kwargs)
+
+        self.stream_run(run=run, storage=local_storage, raise_on_error=self._config.raise_on_error)
+        return run
+
+    async def _submit_bulk_run(self, run: Run, local_storage: AbstractRunStorage, **kwargs) -> None:
+        logger = self._config.logger
+
+        logger.info(f"Submitting run {run.name}, log path: {local_storage.logger.file_path}")
+
+        # Old code loaded the Flex flow, parsed input and outputs types. That logic has been
+        # removed since it is unnecessary. It also parsed and set environment variables. This
+        # has also been removed since it can be problematic in a multi-threaded environment.
+
+        if run.previous_run:
+            previous: Optional[Run] = run.previous_run
+            if previous.status != RunStatus.COMPLETED:
+                raise BatchEngineValidationError(
+                    f"Referenced run {previous.name} is not completed, got status {previous.status.value}."
+                )
+            if previous.outputs is not None:
+                if len(previous.outputs) != len(run.inputs):
+                    raise BatchEngineValidationError(
+                        f"Referenced run {previous.name} has {len(previous.outputs)} outputs, "
+                        f"but {len(run.inputs)} inputs are provided."
+                    )
+
+                # load in the previous run's outputs and inputs into the list of dictionaries to allow for
+                # the previous run's outputs to be used as inputs for the current run
+                run.inputs = [
+                    {"run.outputs": previous.outputs[i], "run.inputs": previous.inputs[i], **run.inputs[i]}
+                    for i in range(len(run.inputs))
+                ]
+
+        self._validate_column_mapping(run.column_mapping)
+
+        run._status = RunStatus.RUNNING
+        run._start_time = datetime.now(timezone.utc)
+        batch_result: Optional[BatchResult] = None
+
+        try:
+            batch_engine = BatchEngine(
+                run.dynamic_callable,
+                config=self._config,
+                storage=local_storage,
+                executor=self._executor,
+            )
+
+            batch_result = await batch_engine.run(data=run.inputs, column_mapping=run.column_mapping, id=run.name)
+            run._status = RunStatus.from_batch_result_status(batch_result.status)
+
+            error_logs: Sequence[str] = []
+            if run._status != RunStatus.COMPLETED:
+                error_logs.append(f"Run {run.name} failed with status {batch_result.status}.")
+                if batch_result.error:
+                    error_logs.append(f"Error: {str(batch_result.error)}")
+
+            if error_logs:
+                logger.warning("\n".join(error_logs))
+        except Exception as e:
+            run._status = RunStatus.FAILED
+            # when run failed in executor, store the exception in result and dump to file
+            logger.warning(f"Run {run.name} failed when executing in executor with exception {e}.")
+            if not batch_result:
+                batch_result = BatchResult(
+                    status=BatchStatus.Failed,
+                    total_lines=0,
+                    failed_lines=0,
+                    start_time=datetime.now(timezone.utc),
+                    end_time=datetime.now(timezone.utc),
+                    tokens=None,
+                    details=[],
+                )
+                batch_result.error = e
+            elif not batch_result.error:
+                batch_result.error = e
+            # for user error, swallow stack trace and return failed run since user don't need the stack trace
+            if not isinstance(e, BatchEngineValidationError):
+                # for other errors, raise it to user to help debug root cause.
+                raise e
+            # won't raise the exception since it's already included in run object.
+        finally:
+            # persist inputs, outputs and metrics
+            local_storage.persist_result(batch_result)
+            # exceptions
+            # local_storage.dump_exception(exception=exception, batch_result=batch_result) # TODO ralphe: persist_result should handle this
+            # system metrics
+            system_metrics = {}
+            if batch_result:
+                # system_metrics.update(dataclasses.asdict(batch_result.tokens))  # token related
+                system_metrics.update(
+                    {
+                        # "duration": batch_result.duration.total_seconds(),
+                        # "__pf__.lines.completed": batch_result.total_lines - batch_result.failed_lines,
+                        # "__pf__.lines.failed": batch_result.failed_lines,
+                    }
+                )
+
+            run._end_time = datetime.now(timezone.utc)
+            run.metrics = system_metrics
+            run.result = batch_result
+
+    @staticmethod
+    def _validate_inputs(run: Run):
+        if not run.inputs and not run.previous_run:
+            raise BatchEngineValidationError("Either data, or a previous run must be specified for the evaluation run.")
+
+    @staticmethod
+    def _validate_column_mapping(column_mapping: Optional[Mapping[str, str]]):
+        if not column_mapping:
+            return
+
+        if not isinstance(column_mapping, Mapping):
+            raise BatchEngineValidationError(f"Column mapping must be a dict, got {type(column_mapping)}.")
+
+        has_mapping = any([isinstance(v, str) and v.startswith("$") for v in column_mapping.values()])
+        if not has_mapping:
+            raise BatchEngineValidationError(
+                "Column mapping must contain at least one mapping binding, "
+                f"current column mapping contains all static values: {column_mapping}"
+            )
+
+    @staticmethod
+    def stream_run(run: Run, storage: AbstractRunStorage, raise_on_error: bool) -> None:
+        """
+        Stream the output of the batch execution.
+
+        :param Run run: The run to stream.
+        :param AbstractRunStorage storage: The storage to use for the output.
+        """
+
+        if run is None or storage is None:
+            return
+
+        file_handler = sys.stdout
+        error_message: Optional[str] = None
+        try:
+            printed = 0
+            available_logs = storage.logger.get_logs()
+            incremental_print(available_logs, printed, file_handler)
+            RunSubmitter._print_run_summary(run, file_handler)
+        except KeyboardInterrupt:
+            error_message = "The output streaming for the run was interrupted, but the run is still executing."
+            print(error_message)
+
+        if run.status == RunStatus.FAILED or run.status == RunStatus.CANCELED:
+            if run.status == RunStatus.FAILED:
+                # Get the first error message from the results, or use a default one
+                if run.result and run.result.error:
+                    error_message = "".join(
+                        traceback.format_exception(
+                            type(run.result.error), run.result.error, run.result.error.__traceback__
+                        )
+                    )
+                elif run.result and run.result.details:
+                    err = next((r.error for r in run.result.details if r.error), None)
+                    if err and err.exception:
+                        error_message = "".join(
+                            traceback.format_exception(type(err.exception), err.exception, err.exception.__traceback__)
+                        )
+                    elif err and err.details:
+                        error_message = err.details
+
+                if not error_message:
+                    error_message = "Run fails with unknown error."
+            else:
+                error_message = "Run is canceled."
+            if raise_on_error:
+                raise BatchEngineError(error_message)
+            else:
+                print_red_error(error_message)
+
+    @staticmethod
+    def _print_run_summary(run: Run, text_out: Union[TextIO, Any]) -> None:
+        duration = str(run.duration)
+        text_out.write(
+            "======= Run Summary =======\n\n"
+            f'Run name: "{run.name}"\n'
+            f'Run status: "{run.status.value}"\n'
+            f'Start time: "{run.created_on}"\n'
+            f'Duration: "{duration}"\n\n'
+        )

azure/ai/evaluation/_legacy/_batch_engine/_status.py

@@ -0,0 +1,25 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+from enum import IntEnum, auto, unique
+
+
+@unique
+class BatchStatus(IntEnum):
+    NotStarted = 0
+    Running = auto()
+
+    # NOTE: DO NOT REORDER THESE ENUMS. The order is important for the is_terminated method
+    #       and other logic in the code to work properly
+    Completed = auto()
+    Canceled = auto()
+    Failed = auto()
+
+    @staticmethod
+    def is_terminated(status: "BatchStatus") -> bool:
+        return status >= BatchStatus.Completed
+
+    @staticmethod
+    def is_failed(status: "BatchStatus") -> bool:
+        return status == BatchStatus.Failed or status == BatchStatus.Canceled

azure/ai/evaluation/_legacy/_batch_engine/_trace.py

@@ -0,0 +1,97 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+# Pretty much all this code will be removed
+
+from typing import Any, Dict, Optional
+
+
+def start_trace(
+    *,
+    resource_attributes: Optional[Dict] = None,
+    collection: Optional[str] = None,
+    **kwargs: Any,
+) -> None:
+    """Starts a trace.
+
+    :param resource_attributes: Specify the resource attributes for current process.
+    :type resource_attributes: typing.Optional[dict]
+    :param collection: Specify the collection for current tracing.
+    :type collection: typing.Optional[str]
+    """
+    pass
+
+    # res_attrs: Dict[str, str] = {"service.name": "promptflow"}
+    # if resource_attributes:
+    #     logging.debug("specified resource attributes: %s", resource_attributes)
+    #     res_attrs.update(resource_attributes)
+
+    # # determine collection
+    # collection_user_specified = collection is not None
+    # if not collection_user_specified:
+    #     collection = kwargs.get("_collection", _get_collection_from_cwd())
+    #    logging.debug("collection is not user specified")
+    #     if is_collection_writeable():
+    #         # internal parameter for devkit call
+    #         _collection = kwargs.get("_collection", None)
+    #         if _collection is not None:
+    #             logging.debug("received internal parameter _collection: %s, will use this", _collection)
+    #             collection = _collection
+    #         else:
+    #             logging.debug("trying to get from current working directory...")
+    #             collection = _get_collection_from_cwd()
+    #     # TODO ralphe: OpenTelemetry dependency. This is a future task to resolve.
+    #     # else:
+    #     #     logging.debug("collection is protected, will directly use that...")
+    #     #     tracer_provider: TracerProvider = trace.get_tracer_provider()
+    #     #     collection = tracer_provider.resource.attributes["collection"]
+    # logging.info("collection: %s", collection)
+    # res_attrs["collection"] = collection or "default"
+    # logging.info("resource attributes: %s", res_attrs)
+
+    # # if user specifies collection, we will add a flag on tracer provider to avoid override
+    # _set_tracer_provider(res_attrs, protected_collection=collection_user_specified)
+
+    # Rest of code is removed since we are removing promptflow-devkit dependency
+
+
+# def is_collection_writeable() -> bool:
+#     # TODO ralphe: This has OpenTelemetry dependency. That is a future task to resolve.
+#     # return not getattr(trace.get_tracer_provider(), TRACER_PROVIDER_PROTECTED_COLLECTION_ATTR, False)
+#     return True
+
+
+# def _get_collection_from_cwd() -> str:
+#     """Try to use cwd folder name as collection name; will fall back to default value if run into exception."""
+#     cur_folder_name = ""
+#     try:
+#         cwd = os.getcwd()
+#         cur_folder_name = os.path.basename(cwd)
+#     except Exception:  # pylint: disable=broad-except
+#         # possible exception: PermissionError, FileNotFoundError, OSError, etc.
+#         pass
+#     collection = cur_folder_name or "default"
+#     return collection
+
+
+# def _set_tracer_provider(res_attrs: Dict[str, str], protected_collection: bool) -> None:
+#     # TODO ralphe: OpenTelemetry dependency. This is a future task to resolve.
+#     # res = Resource(attributes=res_attrs)
+#     # tracer_provider = TracerProvider(resource=res)
+
+#     # cur_tracer_provider = trace.get_tracer_provider()
+#     # if isinstance(cur_tracer_provider, TracerProvider):
+#     #     logging.info("tracer provider is already set, will merge the resource attributes...")
+#     #     cur_res = cur_tracer_provider.resource
+#     #     logging.debug("current resource: %s", cur_res.attributes)
+#     #     new_res = cur_res.merge(res)
+#     #     cur_tracer_provider._resource = new_res
+#     #     logging.info("tracer provider is updated with resource attributes: %s", new_res.attributes)
+#     # else:
+#     #     trace.set_tracer_provider(tracer_provider)
+#     #     logging.info("tracer provider is set with resource attributes: %s", res.attributes)
+
+#     # if protected_collection:
+#     #     logging.info("user specifies collection, will add a flag on tracer provider to avoid override...")
+#     #     setattr(trace.get_tracer_provider(), TRACER_PROVIDER_PROTECTED_COLLECTION_ATTR, True)

azure/ai/evaluation/_legacy/_batch_engine/_utils.py

@@ -0,0 +1,97 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+import inspect
+import os
+import re
+from typing import Any, Final, Mapping, Sequence, Tuple
+
+
+DEFAULTS_KEY: Final[str] = "$defaults$"
+
+
+def normalize_identifier_name(name: str) -> str:
+    """Normalize the identifier name to a valid Python variable name.
+
+    Args:
+        name (str): The identifier name to normalize.
+
+    Returns:
+        str: The normalized identifier name.
+    """
+    normalized = re.sub(r"\W", "_", name.strip())
+    if normalized[0].isdigit():
+        normalized = f"_{normalized}"
+    return normalized
+
+
+def get_int_env_var(env_var_name: str, default_value: int = 0) -> int:
+    """Get the integer value of the environment variable.
+
+    Args:
+        env_var_name (str): The name of the environment variable.
+        default_value (int): The default value if the environment variable is not set.
+
+    Returns:
+        int: The integer value of the environment variable.
+    """
+    try:
+        value = os.getenv(env_var_name, default_value)
+        return int(value)
+    except ValueError:
+        return default_value
+
+
+def get_value_from_path(path: str, data: Mapping[str, Any]) -> Tuple[bool, Any]:
+    """Tried to get a value from a mapping based on the specified path. The path is a
+    string with dot-separated keys (e.g. data.nested_1.nested_2).
+
+    This will interpret the path prioritizing a depth first search with the shortest
+    key possible at each level. If for example you had the following data:
+    {
+        "foo": {
+            "bar": {
+                "happy": 12
+            }
+        },
+        "foo.bar": {
+            "none": 14,
+            "random": { "some": 15 }
+        },
+        "foo.bar.none": 16
+    }
+    And you asked for foo.bar.none, the returned value would be 14"
+    """
+
+    def _get_value(data: Mapping[str, Any], parts: Sequence[str]) -> Tuple[bool, Any]:
+        if len(parts) == 0:
+            return True, data
+
+        for i in range(1, len(parts) + 1):
+            key = ".".join(parts[:i])
+            if isinstance(data, Mapping) and key in data:
+                found, match = _get_value(data[key], parts[i:])
+                if found:
+                    return found, match
+
+        return False, None
+
+    if path is None or data is None:
+        return False, None
+
+    parts = path.strip().split(".")
+    if len(parts) == 0:
+        return False, None
+    return _get_value(data, parts)
+
+
+def is_async_callable(obj: Any) -> bool:
+    """Check if the object is an async callable. This will be true if the object is a coroutine function,
+    or if the object has
+
+    :param Any obj: The object to check.
+    :return: True if the object is an async callable.
+    :rtype: bool
+    """
+    return inspect.iscoroutinefunction(obj) or inspect.iscoroutinefunction(getattr(obj, "__call__", None))

azure/ai/evaluation/_legacy/_batch_engine/_utils_deprecated.py

@@ -0,0 +1,131 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+import asyncio
+import contextvars
+import dataclasses
+from asyncio import Task
+from concurrent.futures import ThreadPoolExecutor
+from typing import Any, AsyncIterator, Callable, Iterator, Mapping, Optional, Sequence, Tuple, cast
+
+
+class ThreadPoolExecutorWithContext(ThreadPoolExecutor):
+    # Original source:
+    # promptflow-tracing/promptflow/tracing/_context_utils.py
+
+    def __init__(
+        self,
+        max_workers: Optional[int] = None,
+        thread_name_prefix: str = "",
+        initializer: Optional[Callable] = None,
+        initargs: Tuple[Any, ...] = (),
+    ) -> None:
+        """The ThreadPoolExecutionWithContext is an extended thread pool implementation
+        which will copy the context from the current thread to the child threads.
+        Thus the traced functions in child threads could keep parent-child relationship in the tracing system.
+        The arguments are the same as ThreadPoolExecutor.
+
+        Args:
+            max_workers: The maximum number of threads that can be used to
+                execute the given calls.
+            thread_name_prefix: An optional name prefix to give our threads.
+            initializer: A callable used to initialize worker threads.
+            initargs: A tuple of arguments to pass to the initializer.
+        """
+        current_context = contextvars.copy_context()
+        initializer_args = (current_context, initializer, initargs)
+        super().__init__(max_workers, thread_name_prefix, self.set_context_then_call, initializer_args)
+
+    @staticmethod
+    def set_context_then_call(
+        context: contextvars.Context,
+        initializer: Optional[Callable],
+        initargs: Tuple[Any, ...],
+    ) -> None:
+        for var, value in context.items():
+            var.set(value)
+        if initializer:
+            initializer(*initargs)
+
+
+def _has_running_loop() -> bool:
+    """Check if the current thread has a running event loop."""
+    # When using asyncio.get_running_loop(), a RuntimeError is raised if there is no running event loop.
+    # So, we use a try-catch block to determine whether there is currently an event loop in place.
+    #
+    # Note that this is the only way to check whether there is a running loop now, see:
+    # https://docs.python.org/3/library/asyncio-eventloop.html?highlight=get_running_loop#asyncio.get_running_loop
+    try:
+        asyncio.get_running_loop()
+        return True
+    except RuntimeError:
+        return False
+
+
+def async_run_allowing_running_loop(async_func, *args, **kwargs):
+    """Run an async function in a new thread, allowing the current thread to have a running event loop.
+
+    When run in an async environment (e.g., in a notebook), because each thread allows only one event
+    loop, using asyncio.run directly leads to a RuntimeError ("asyncio.run() cannot be called from a
+    running event loop").
+
+    To address this issue, we add a check for the event loop here. If the current thread already has an
+    event loop, we run _exec_batch in a new thread; otherwise, we run it in the current thread.
+    """
+
+    if _has_running_loop():
+        # TODO ralphe: The logic here makes absolutely no sense to me. If you already have an
+        #              async event loop running, why would you want to start up a new thread,
+        #              create a new event loop, and run the async function in a new thread?
+        #              You can just use the following to schedule the async function call on
+        #              the existing event loop:
+        # asyncio.get_running_loop().create_task(async_func(*args, *args, **kwargs)).result()
+        #              The correct thing to do here is not make these decisions here at all.
+        #              Instead, all the BatchEngine code should be async first, with the event
+        #              loop being started by the callers of that code. For now, I am keeping
+        #              this odd logic as is, and in phase 2 of the migration, this will be
+        #              refactored to be more idiomatic asyncio code.
+        with ThreadPoolExecutorWithContext() as executor:
+            return executor.submit(lambda: asyncio.run(async_func(*args, **kwargs))).result()
+    else:
+        return asyncio.run(async_func(*args, **kwargs))
+
+
+async def stringify_output_async(output: Any) -> str:
+    if isinstance(output, AsyncIterator):
+        return await stringify_output_async([v async for v in output])
+    if isinstance(output, Iterator):
+        return await stringify_output_async([v for v in output])
+    if isinstance(output, Mapping):
+        return ", ".join(
+            [f"{await stringify_output_async(k)}:{await stringify_output_async(v)}" for k, v in output.items()]
+        )
+    if isinstance(output, Sequence):
+        return "".join([await stringify_output_async(v) for v in output])
+    if isinstance(output, Task):
+        return await stringify_output_async(await output)
+
+    return str(output)
+
+
+def convert_eager_flow_output_to_dict(value: Any) -> Mapping[str, Any]:
+    """
+    Convert the output of eager flow to a dict. Since the output of eager flow
+    may not be a dict, we need to convert it to a dict in batch mode.
+
+    Examples:
+    1. If the output is a dict, return it directly:
+        value = {"output": 1} -> {"output": 1}
+    2. If the output is a dataclass, convert it to a dict:
+        value = SampleDataClass(output=1) -> {"output": 1}
+    3. If the output is not a dict or dataclass, convert it to a dict by adding a key "output":
+        value = 1 -> {"output": 1}
+    """
+
+    if isinstance(value, Mapping):
+        return value
+    elif dataclasses.is_dataclass(value):
+        return dataclasses.asdict(cast(Any, value))
+    else:
+        return {"output": value}

azure/ai/evaluation/_legacy/_common/__init__.py

@@ -0,0 +1,3 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------