azure-ai-evaluation 1.0.1__py3-none-any.whl → 1.13.5__py3-none-any.whl

azure/ai/evaluation/_legacy/_batch_engine/_result.py

@@ -0,0 +1,107 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+from dataclasses import dataclass
+from datetime import datetime, timedelta
+from typing import Any, Mapping, Optional, Sequence
+
+from ._status import BatchStatus
+
+
+@dataclass
+class TokenMetrics:
+    """The token metrics of a run."""
+
+    prompt_tokens: int
+    """The number of tokens used in the prompt for the run."""
+    completion_tokens: int
+    """The number of tokens used in the completion for the run."""
+    total_tokens: int
+    """The total number of tokens used in the run."""
+
+    def update(self, other: "TokenMetrics") -> None:
+        """Update the token metrics with another set of token metrics."""
+        self.prompt_tokens += other.prompt_tokens
+        self.completion_tokens += other.completion_tokens
+        self.total_tokens += other.total_tokens
+
+
+@dataclass
+class BatchRunError:
+    """The error of a batch run."""
+
+    details: str
+    """The details of the error."""
+    exception: Optional[BaseException]
+    """The exception of the error."""
+
+
+@dataclass
+class BatchRunDetails:
+    """The error of a line in a batch run."""
+
+    id: str
+    """The ID of the line run."""
+    status: BatchStatus
+    """The status of the line run."""
+    result: Optional[Mapping[str, Any]]
+    """The result of the line run."""
+    start_time: Optional[datetime]
+    """The start time of the line run. If this was never started, this should be None."""
+    end_time: Optional[datetime]
+    """The end time of the line run. If this never completed, this should be None."""
+    tokens: TokenMetrics
+    """The token metrics of the line run."""
+    error: Optional[BatchRunError]
+    """The error of the line run. This will only be set if the status is Failed."""
+    index: int
+    """The line run index."""
+
+    @property
+    def duration(self) -> timedelta:
+        """The duration of the line run."""
+        if self.start_time is not None and self.end_time is not None:
+            return self.end_time - self.start_time
+        return timedelta(0)
+
+    @staticmethod
+    def create_id(run_id: str, index: int) -> str:
+        """Helper method to create the ID for a line run."""
+        return f"{run_id}_{index}"
+
+
+@dataclass
+class BatchResult:
+    """The result of a batch run."""
+
+    status: BatchStatus
+    """The overall status of the batch run."""
+    total_lines: int
+    """The total number of lines in the batch run."""
+    failed_lines: int
+    """The number of failed lines in the batch run."""
+    start_time: datetime
+    """The start time of the batch run."""
+    end_time: datetime
+    """The end time of the batch run."""
+    tokens: TokenMetrics
+    """The overall token metrics of the batch run."""
+    details: Sequence[BatchRunDetails]
+    """The details of each line in the batch run."""
+    error: Optional[Exception] = None
+    """The error of the batch run. This will only be set if the status does not indicate success."""
+
+    @property
+    def duration(self) -> timedelta:
+        """The duration of the batch run."""
+        if self.start_time is not None and self.end_time is not None:
+            return self.end_time - self.start_time
+        return timedelta(0)
+
+    @property
+    def results(self) -> Sequence[Optional[Mapping[str, Any]]]:
+        """The results of the batch run."""
+        if not self.details:
+            return []
+        return [d.result for d in self.details]

azure/ai/evaluation/_legacy/_batch_engine/_run.py

@@ -0,0 +1,127 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+from uuid import uuid4
+from datetime import datetime, timedelta, timezone
+from enum import Enum
+from typing import Any, Callable, Mapping, Optional, Sequence
+
+from ._utils import normalize_identifier_name
+from ._result import BatchResult
+from ._status import BatchStatus
+
+
+class RunStatus(Enum):
+    # TODO ralphe: Trim this to just the statuses we need
+    # QUEUED = "Queued"
+    NOT_STARTED = "NotStarted"
+    PREPARING = "Preparing"
+    # PROVISIONING = "Provisioning"
+    # STARTING = "Starting"
+    RUNNING = "Running"
+    # CANCEL_REQUESTED = "CancelRequested"
+    CANCELED = "Canceled"
+    # FINALIZING = "Finalizing"
+    COMPLETED = "Completed"
+    FAILED = "Failed"
+    # UNAPPROVED = "Unapproved"
+    # NOTRESPONDING = "NotResponding"
+    # PAUSING = "Pausing"
+    # PAUSED = "Paused"
+
+    @staticmethod
+    def from_batch_result_status(status: BatchStatus) -> "RunStatus":
+        if status == BatchStatus.NotStarted:
+            return RunStatus.NOT_STARTED
+        if status == BatchStatus.Running:
+            return RunStatus.RUNNING
+        if status == BatchStatus.Completed:
+            return RunStatus.COMPLETED
+        if status == BatchStatus.Canceled:
+            return RunStatus.CANCELED
+        if status == BatchStatus.Failed:
+            return RunStatus.FAILED
+
+        return RunStatus.FAILED
+
+
+class Run:
+    """The equivalent of a Promptflow Run
+    promptflow-devkit/promptflow/_sdk/entities/_run.py
+
+    THIS WILL BE REMOVED IN A FUTURE CODE UPDATE"""
+
+    def __init__(
+        self,
+        *,
+        dynamic_callable: Callable,
+        name_prefix: Optional[str],
+        inputs: Sequence[Mapping[str, Any]],
+        column_mapping: Optional[Mapping[str, str]] = None,
+        created_on: Optional[datetime] = None,
+        run: Optional["Run"] = None,
+    ):
+        self._status: RunStatus = RunStatus.NOT_STARTED
+        self._created_on = created_on or datetime.now(timezone.utc)
+        self._start_time: Optional[datetime] = None
+        self._end_time: Optional[datetime] = None
+
+        self.dynamic_callable = dynamic_callable
+        self.name = self._generate_run_name(name_prefix, self._created_on)
+        self.inputs = inputs
+        self.column_mapping: Optional[Mapping[str, str]] = column_mapping
+        self.result: Optional[BatchResult] = None
+        self.metrics: Mapping[str, Any] = {}
+        self._run = run
+
+        # self._use_remote_flow = False
+        # self._from_flex_flow = True
+        # self._from_prompty = False
+        # self.flow = path to pointless flow file
+        # self._experiment_name = name of folder containing pointless flow file
+        # self._lineage_id = basically equivalent to a hex digest of the SHA256 hash of:
+        #                    f"{uuid.getnod()}/{posix_full_path_to_pointless_folder}"
+        # self._output_path = Path("<user_folder>/.promptflow/runs/<self.name>")
+        # self._flow_name = name of pointless folder
+
+    @property
+    def status(self) -> RunStatus:
+        return self._status
+
+    @property
+    def created_on(self) -> datetime:
+        return self._created_on
+
+    @property
+    def duration(self) -> Optional[timedelta]:
+        if self._start_time is None or self._end_time is None:
+            return None
+
+        return self._end_time - self._start_time
+
+    @property
+    def outputs(self) -> Sequence[Mapping[str, Any]]:
+        if self.result is None:
+            return []
+
+        return [value or {} for value in self.result.results]
+
+    @property
+    def previous_run(self) -> Optional["Run"]:
+        return self._run
+
+    @staticmethod
+    def _generate_run_name(name_prefix: Optional[str], creation_time: datetime) -> str:
+        # The Promptflow code looked at the folder name  of the temporary folder used to
+        # store the temporary flow YAML file which was a single entry that told it look
+        # at the passed in dynamic_callable. Example folder name:
+        # azure_ai_evaluation_evaluators_common_base_eval_asyncevaluatorbase_l82059h3
+        # instead we will use the passed in name_prefix or use a UUID (which is equally
+        # opaque as what the original code did)
+        if not name_prefix:
+            name_prefix = str(uuid4())
+
+        timestamp = creation_time.strftime("%Y%m%d_%H%M%S_%f")
+        name = f"{name_prefix}_{timestamp}"
+        return normalize_identifier_name(name)

azure/ai/evaluation/_legacy/_batch_engine/_run_storage.py

@@ -0,0 +1,128 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+# Original source:
+# promptflow-devkit/promptflow/_sdk/operations/_local_storage_operations.py
+
+from abc import ABC, abstractmethod
+from contextlib import AbstractContextManager
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any, Final, Mapping, Optional, Tuple, Union
+
+from ._result import BatchResult, TokenMetrics, BatchStatus
+
+
+EVAL_USER_SUBFOLDER: Final[str] = ".evaluation"
+
+
+class AbstractRunLogger(AbstractContextManager):
+    @property
+    @abstractmethod
+    def file_path(self) -> Path:
+        """Get the file path of the logger.
+
+        :return: The file path of the logger.
+        :rtype: Path
+        """
+        ...
+
+    @abstractmethod
+    def get_logs(self) -> str:
+        """Get the logs of the run.
+
+        :return: The logs of the run.
+        :rtype: str
+        """
+        ...
+
+
+class AbstractRunStorage(ABC):
+    @property
+    @abstractmethod
+    def logger(self) -> "AbstractRunLogger":
+        """Get the logger of the run.
+
+        :return: The logger of the run.
+        :rtype: ~promptflow.contracts.run_logger.RunLogger
+        """
+        ...
+
+    @abstractmethod
+    def persist_result(self, result: Optional[BatchResult]) -> None:
+        """Persist results of a batch engine execution (including any errors).
+
+        :param Optional[BatchResult] result: The result to persist.
+        """
+        ...
+
+    @abstractmethod
+    def load_exception(self) -> Mapping[str, Any]:
+        """Load the exception from the storage. If there was no exception, an empty
+        mapping will be returned.
+
+        :return: The exception.
+        :rtype: Optional[Exception]
+        """
+        ...
+
+    @abstractmethod
+    def load_inputs_and_outputs(self) -> Tuple[Mapping[str, Any], BatchResult]:
+        """Load the inputs and outputs from the storage.
+
+        :return: The inputs and outputs.
+        :rtype: Tuple(Mapping[str, Any], BatchResult)
+        """
+        ...
+
+    @abstractmethod
+    def load_metrics(self) -> Mapping[str, Union[int, float, str]]:
+        """Load the metrics from the storage.
+
+        :return: The metrics.
+        :rtype: Mapping[str, Union[int, float, str]]
+        """
+        ...
+
+
+class NoOpRunStorage(AbstractRunStorage):
+    """A no-op implementation of the run storage."""
+
+    def __init__(self):
+        self._logger = NoOpLogger()
+        pass
+
+    @property
+    def logger(self) -> AbstractRunLogger:
+        return self._logger
+
+    def persist_result(self, result: Optional[BatchResult]) -> None:
+        pass
+
+    def load_exception(self) -> Mapping[str, Any]:
+        return {}
+
+    def load_inputs_and_outputs(self) -> Tuple[Mapping[str, Any], BatchResult]:
+        now = datetime.now(timezone.utc)
+        return {}, BatchResult(BatchStatus.NotStarted, 0, 0, now, now, TokenMetrics(0, 0, 0), [])
+
+    def load_metrics(self) -> Mapping[str, Union[int, float, str]]:
+        return {}
+
+
+class NoOpLogger(AbstractRunLogger):
+    """A no-op implementation of the run logger."""
+
+    @property
+    def file_path(self) -> Path:
+        return Path.home() / EVAL_USER_SUBFOLDER
+
+    def __enter__(self) -> None:
+        pass
+
+    def __exit__(self, *args) -> None:
+        pass
+
+    def get_logs(self) -> str:
+        return ""

azure/ai/evaluation/_legacy/_batch_engine/_run_submitter.py

@@ -0,0 +1,262 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+import dataclasses
+import inspect
+import sys
+import traceback
+
+from concurrent.futures import Executor
+from datetime import datetime, timezone
+from typing import Any, Callable, Dict, Mapping, Optional, Sequence, TextIO, Union
+
+from ._run import Run, RunStatus
+from ._trace import start_trace
+from ._run_storage import AbstractRunStorage, NoOpRunStorage
+from .._common._logging import incremental_print, print_red_error
+from ._config import BatchEngineConfig
+from ._exceptions import BatchEngineValidationError
+from ._engine import DEFAULTS_KEY, BatchEngine, BatchEngineError, BatchResult, BatchStatus
+
+
+class RunSubmitter:
+    """Submits run to executor
+    promptflow-devkit/promptflow/_sdk/_orchestrator/run_submitter.py
+
+    THIS WILL BE REMOVED IN A FUTURE CODE UPDATE"""
+
+    def __init__(self, config: BatchEngineConfig, executor: Optional[Executor] = None):
+        # self._client = PFClient instance
+        # self._config = PFClient config
+        # self.run_operations = RunOperations instance
+
+        # TODO ralphe: Use proper logger here. Old code did LoggerFactory.get_logger(__name__)
+        self._config = config
+        self._executor = executor
+
+    async def submit(
+        self,
+        dynamic_callable: Callable,
+        inputs: Sequence[Mapping[str, Any]],
+        column_mapping: Optional[Mapping[str, str]],
+        *,
+        name_prefix: Optional[str] = None,
+        created_on: Optional[datetime] = None,
+        storage_creator: Optional[Callable[[Run], AbstractRunStorage]] = None,
+        **kwargs,
+    ) -> Run:
+
+        # The old code always spun up two threads here using a ThreadPoolExecutor:
+        # 1. One thread essentially did nothing of value (since tracing was disabled, and we
+        #    don't care about checking for the latest PromptFlow version number now)
+        # 2. The other thread did the _run_bulk call. This was followed by a
+        #    wait(return_when=ALL_COMPLETED)
+        # This quite frankly is unnecessary complexity since the the evaluation code already
+        # calls this in the context of ThreadPoolThread. So we can just do the equivalent
+        # of the _run_bulk code here directly.
+        # In a future code refactor, all of this will be cleaned up in favour of proper
+        # async/await code.
+
+        run: Run = Run(
+            dynamic_callable=dynamic_callable,
+            name_prefix=name_prefix,
+            inputs=inputs,
+            column_mapping=column_mapping,
+            created_on=created_on,
+            run=kwargs.pop("run", None),
+        )
+
+        attributes: Dict[str, Any] = kwargs.get("attributes", {})
+        collection_for_run: str = run.name
+        start_trace(attributes=attributes, run=run, _collection=collection_for_run)
+
+        self._validate_inputs(run=run)
+
+        local_storage = storage_creator(run) if storage_creator else NoOpRunStorage()
+        with local_storage.logger:
+            run._status = RunStatus.PREPARING
+
+            # unnecessary Flow loading code was removed here. Instead do direct calls to _submit_bulk_run
+            await self._submit_bulk_run(run=run, local_storage=local_storage, **kwargs)
+
+        self.stream_run(run=run, storage=local_storage, raise_on_error=self._config.raise_on_error)
+        return run
+
+    async def _submit_bulk_run(self, run: Run, local_storage: AbstractRunStorage, **kwargs) -> None:
+        logger = self._config.logger
+
+        logger.info(f"Submitting run {run.name}, log path: {local_storage.logger.file_path}")
+
+        # Old code loaded the Flex flow, parsed input and outputs types. That logic has been
+        # removed since it is unnecessary. It also parsed and set environment variables. This
+        # has also been removed since it can be problematic in a multi-threaded environment.
+
+        if run.previous_run:
+            previous: Optional[Run] = run.previous_run
+            if previous.status != RunStatus.COMPLETED:
+                raise BatchEngineValidationError(
+                    f"Referenced run {previous.name} is not completed, got status {previous.status.value}."
+                )
+            if previous.outputs is not None:
+                if len(previous.outputs) != len(run.inputs):
+                    raise BatchEngineValidationError(
+                        f"Referenced run {previous.name} has {len(previous.outputs)} outputs, "
+                        f"but {len(run.inputs)} inputs are provided."
+                    )
+
+                # load in the previous run's outputs and inputs into the list of dictionaries to allow for
+                # the previous run's outputs to be used as inputs for the current run
+                run.inputs = [
+                    {"run.outputs": previous.outputs[i], "run.inputs": previous.inputs[i], **run.inputs[i]}
+                    for i in range(len(run.inputs))
+                ]
+
+        self._validate_column_mapping(run.column_mapping)
+
+        run._status = RunStatus.RUNNING
+        run._start_time = datetime.now(timezone.utc)
+        batch_result: Optional[BatchResult] = None
+
+        try:
+            batch_engine = BatchEngine(
+                run.dynamic_callable,
+                config=self._config,
+                storage=local_storage,
+                executor=self._executor,
+            )
+
+            batch_result = await batch_engine.run(data=run.inputs, column_mapping=run.column_mapping, id=run.name)
+            run._status = RunStatus.from_batch_result_status(batch_result.status)
+
+            error_logs: Sequence[str] = []
+            if run._status != RunStatus.COMPLETED:
+                error_logs.append(f"Run {run.name} failed with status {batch_result.status}.")
+                if batch_result.error:
+                    error_logs.append(f"Error: {str(batch_result.error)}")
+
+            if error_logs:
+                logger.warning("\n".join(error_logs))
+        except Exception as e:
+            run._status = RunStatus.FAILED
+            # when run failed in executor, store the exception in result and dump to file
+            logger.warning(f"Run {run.name} failed when executing in executor with exception {e}.")
+            if not batch_result:
+                batch_result = BatchResult(
+                    status=BatchStatus.Failed,
+                    total_lines=0,
+                    failed_lines=0,
+                    start_time=datetime.now(timezone.utc),
+                    end_time=datetime.now(timezone.utc),
+                    tokens=None,
+                    details=[],
+                )
+                batch_result.error = e
+            elif not batch_result.error:
+                batch_result.error = e
+            # for user error, swallow stack trace and return failed run since user don't need the stack trace
+            if not isinstance(e, BatchEngineValidationError):
+                # for other errors, raise it to user to help debug root cause.
+                raise e
+            # won't raise the exception since it's already included in run object.
+        finally:
+            # persist inputs, outputs and metrics
+            local_storage.persist_result(batch_result)
+            # exceptions
+            # local_storage.dump_exception(exception=exception, batch_result=batch_result) # TODO ralphe: persist_result should handle this
+            # system metrics
+            system_metrics = {}
+            if batch_result:
+                # system_metrics.update(dataclasses.asdict(batch_result.tokens))  # token related
+                system_metrics.update(
+                    {
+                        # "duration": batch_result.duration.total_seconds(),
+                        # "__pf__.lines.completed": batch_result.total_lines - batch_result.failed_lines,
+                        # "__pf__.lines.failed": batch_result.failed_lines,
+                    }
+                )
+
+            run._end_time = datetime.now(timezone.utc)
+            run.metrics = system_metrics
+            run.result = batch_result
+
+    @staticmethod
+    def _validate_inputs(run: Run):
+        if not run.inputs and not run.previous_run:
+            raise BatchEngineValidationError("Either data, or a previous run must be specified for the evaluation run.")
+
+    @staticmethod
+    def _validate_column_mapping(column_mapping: Optional[Mapping[str, str]]):
+        if not column_mapping:
+            return
+
+        if not isinstance(column_mapping, Mapping):
+            raise BatchEngineValidationError(f"Column mapping must be a dict, got {type(column_mapping)}.")
+
+        has_mapping = any([isinstance(v, str) and v.startswith("$") for v in column_mapping.values()])
+        if not has_mapping:
+            raise BatchEngineValidationError(
+                "Column mapping must contain at least one mapping binding, "
+                f"current column mapping contains all static values: {column_mapping}"
+            )
+
+    @staticmethod
+    def stream_run(run: Run, storage: AbstractRunStorage, raise_on_error: bool) -> None:
+        """
+        Stream the output of the batch execution.
+
+        :param Run run: The run to stream.
+        :param AbstractRunStorage storage: The storage to use for the output.
+        """
+
+        if run is None or storage is None:
+            return
+
+        file_handler = sys.stdout
+        error_message: Optional[str] = None
+        try:
+            printed = 0
+            available_logs = storage.logger.get_logs()
+            incremental_print(available_logs, printed, file_handler)
+            RunSubmitter._print_run_summary(run, file_handler)
+        except KeyboardInterrupt:
+            error_message = "The output streaming for the run was interrupted, but the run is still executing."
+            print(error_message)
+
+        if run.status == RunStatus.FAILED or run.status == RunStatus.CANCELED:
+            if run.status == RunStatus.FAILED:
+                # Get the first error message from the results, or use a default one
+                if run.result and run.result.error:
+                    error_message = "".join(
+                        traceback.format_exception(
+                            type(run.result.error), run.result.error, run.result.error.__traceback__
+                        )
+                    )
+                elif run.result and run.result.details:
+                    err = next((r.error for r in run.result.details if r.error), None)
+                    if err and err.exception:
+                        error_message = "".join(
+                            traceback.format_exception(type(err.exception), err.exception, err.exception.__traceback__)
+                        )
+                    elif err and err.details:
+                        error_message = err.details
+
+                if not error_message:
+                    error_message = "Run fails with unknown error."
+            else:
+                error_message = "Run is canceled."
+            if raise_on_error:
+                raise BatchEngineError(error_message)
+            else:
+                print_red_error(error_message)
+
+    @staticmethod
+    def _print_run_summary(run: Run, text_out: Union[TextIO, Any]) -> None:
+        duration = str(run.duration)
+        text_out.write(
+            "======= Run Summary =======\n\n"
+            f'Run name: "{run.name}"\n'
+            f'Run status: "{run.status.value}"\n'
+            f'Start time: "{run.created_on}"\n'
+            f'Duration: "{duration}"\n\n'
+        )

azure/ai/evaluation/_legacy/_batch_engine/_status.py

@@ -0,0 +1,25 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+from enum import IntEnum, auto, unique
+
+
+@unique
+class BatchStatus(IntEnum):
+    NotStarted = 0
+    Running = auto()
+
+    # NOTE: DO NOT REORDER THESE ENUMS. The order is important for the is_terminated method
+    #       and other logic in the code to work properly
+    Completed = auto()
+    Canceled = auto()
+    Failed = auto()
+
+    @staticmethod
+    def is_terminated(status: "BatchStatus") -> bool:
+        return status >= BatchStatus.Completed
+
+    @staticmethod
+    def is_failed(status: "BatchStatus") -> bool:
+        return status == BatchStatus.Failed or status == BatchStatus.Canceled