azure-ai-evaluation 1.0.0b2__py3-none-any.whl → 1.13.3__py3-none-any.whl

azure/ai/evaluation/_legacy/_batch_engine/_exceptions.py

@@ -0,0 +1,88 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+from ..._exceptions import ErrorCategory, ErrorBlame, ErrorTarget, EvaluationException
+
+
+class BatchEngineError(EvaluationException):
+    """Exception class for batch engine errors.
+
+    This exception is used to indicate that the error was caused by or in the batch engine.
+
+    :param message: The error message.
+    :type message: str
+    """
+
+    def __init__(self, message: str, **kwargs):
+        kwargs.setdefault("category", ErrorCategory.FAILED_EXECUTION)
+        kwargs.setdefault("target", ErrorTarget.EVAL_RUN)
+        kwargs.setdefault("blame", ErrorBlame.UNKNOWN)
+
+        super().__init__(message, **kwargs)
+
+
+class BatchEngineValidationError(BatchEngineError):
+    """Exception raised when validation fails
+
+    :param message: The error message.
+    :type message: str
+    """
+
+    def __init__(self, message: str, **kwargs):
+        kwargs.setdefault("category", ErrorCategory.INVALID_VALUE)
+        kwargs.setdefault("blame", ErrorBlame.USER_ERROR)
+        super().__init__(message, **kwargs)
+
+
+class BatchEngineTimeoutError(BatchEngineError):
+    """Exception raised when a batch engine operation times out.
+
+    :param message: The error message.
+    :type message: str
+    """
+
+    def __init__(self, message: str, **kwargs):
+        kwargs.setdefault("category", ErrorCategory.FAILED_EXECUTION)
+        kwargs.setdefault("blame", ErrorBlame.SYSTEM_ERROR)
+        super().__init__(message, **kwargs)
+
+
+class BatchEngineCanceledError(BatchEngineError):
+    """Exception raised when a batch engine operation is canceled.
+
+    :param message: The error message.
+    :type message: str
+    """
+
+    def __init__(self, message: str, **kwargs):
+        kwargs.setdefault("category", ErrorCategory.FAILED_EXECUTION)
+        kwargs.setdefault("blame", ErrorBlame.USER_ERROR)
+        super().__init__(message, **kwargs)
+
+
+class BatchEngineRunFailedError(BatchEngineError):
+    """Exception raised when a batch engine run fails.
+
+    :param message: The error message.
+    :type message: str
+    """
+
+    def __init__(self, message: str, **kwargs):
+        kwargs.setdefault("category", ErrorCategory.FAILED_EXECUTION)
+        kwargs.setdefault("blame", ErrorBlame.SYSTEM_ERROR)
+        super().__init__(message, **kwargs)
+
+
+class BatchEnginePartialError(BatchEngineError):
+    """Exception raised when a batch engine run has some successfull lines, mixed in
+    with some failures.
+
+    :param message: The error message.
+    :type message: str
+    """
+
+    def __init__(self, message: str, **kwargs):
+        kwargs.setdefault("category", ErrorCategory.FAILED_EXECUTION)
+        kwargs.setdefault("blame", ErrorBlame.SYSTEM_ERROR)
+        super().__init__(message, **kwargs)

azure/ai/evaluation/_legacy/_batch_engine/_openai_injector.py

@@ -0,0 +1,132 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+# Original source code: promptflow-tracing/promptflow/tracing/_integrations/_openai_injector.py
+
+import functools
+import importlib
+import logging
+
+from contextvars import ContextVar
+from typing import Any, Callable, Final, Generator, Optional, Protocol, Sequence, Tuple
+
+from azure.ai.evaluation._legacy._adapters._errors import MissingRequiredPackage
+from azure.ai.evaluation._legacy._batch_engine._result import TokenMetrics
+
+
+_token_metrics: ContextVar[TokenMetrics] = ContextVar("token_metrics", default=TokenMetrics(0, 0, 0))
+KEY_ATTR_ORIGINAL: Final[str] = "_original"
+
+
+class _TokenMetrics(Protocol):
+    """Protocol class to represent the token metrics."""
+
+    prompt_tokens: int
+    completion_tokens: int
+    total_tokens: int
+
+
+class _WithUsage(Protocol):
+    """Protocol class to represent an OpenAI object that may have a token usage property/attribute."""
+
+    usage: Optional[_TokenMetrics]
+
+
+def _wrap_openai_api_method(method: Callable, is_async: bool) -> Callable:
+    """Wraps the OpenAI API method to inject logic to run on the result of the call."""
+
+    def update_usage(result: _WithUsage) -> None:
+        if hasattr(result, "usage") and result.usage is not None:
+            usage = _token_metrics.get()
+            usage.prompt_tokens += result.usage.prompt_tokens
+            usage.completion_tokens += result.usage.completion_tokens
+            usage.total_tokens += result.usage.total_tokens
+
+    if is_async:
+
+        @functools.wraps(method)
+        async def async_wrapper(*args: Any, **kwargs: Any) -> Any:
+            result: _WithUsage = await method(*args, **kwargs)
+            update_usage(result)
+            return result
+
+        return async_wrapper
+    else:
+
+        @functools.wraps(method)
+        def sync_wrapper(*args: Any, **kwargs: Any) -> Any:
+            result: _WithUsage = method(*args, **kwargs)
+            update_usage(result)
+            return result
+
+        return sync_wrapper
+
+
+def _openai_api_list() -> Generator[Tuple[Any, Callable, bool], None, None]:
+    """Load the list of OpenAI API classes and their corresponding method names."""
+
+    apis: Sequence[Tuple[str, str, str, bool]] = [
+        ("openai.resources.chat", "Completions", "create", False),
+        ("openai.resources.chat", "AsyncCompletions", "create", True),
+        ("openai.resources", "Completions", "create", False),
+        ("openai.resources", "AsyncCompletions", "create", True),
+        ("openai.resources", "Embeddings", "create", False),
+        ("openai.resources", "AsyncEmbeddings", "create", True),
+        ("openai.resources", "Responses", "create", False),
+        ("openai.resources", "AsyncResponses", "create", True),
+    ]
+
+    for module_name, class_name, method_name, is_async in apis:
+        try:
+            module = importlib.import_module(module_name)
+            cls = getattr(module, class_name, None)
+            if cls is None:
+                continue
+            method = getattr(cls, method_name, None)
+            if method is None:
+                continue
+            yield cls, method, is_async
+        except ImportError:
+            raise MissingRequiredPackage("Please install the 'openai' package to use the Azure AI Evaluation SDK")
+        except AttributeError:
+            logging.warning(
+                "The module '%s' does not have class '%s' or method '%s'", module_name, class_name, method_name
+            )
+
+
+def inject_openai_api():
+    """This function modifies the create methods of the OpenAI API classes to inject logic
+    to enable us to collect token usage data.
+    """
+    for cls, method, is_async in _openai_api_list():
+        # Check if the create method of the openai_api class has already been modified
+        if not hasattr(method, KEY_ATTR_ORIGINAL):
+            wrapper_method: Callable = _wrap_openai_api_method(method, is_async)
+            setattr(wrapper_method, KEY_ATTR_ORIGINAL, method)
+            setattr(cls, method.__name__, wrapper_method)
+
+
+def recover_openai_api():
+    """This function restores the original create methods of the OpenAI API classes
+    by assigning them back from the _original attributes of the modified methods.
+    """
+    for cls, method, _ in _openai_api_list():
+        if hasattr(method, KEY_ATTR_ORIGINAL):
+            original_method = getattr(method, KEY_ATTR_ORIGINAL)
+            setattr(cls, method.__name__, original_method)
+
+
+class CaptureOpenAITokenUsage:
+    """Context manager to capture OpenAI token usage."""
+
+    def __init__(self):
+        self._tokens = TokenMetrics(0, 0, 0)
+
+    def __enter__(self) -> TokenMetrics:
+        _token_metrics.set(TokenMetrics(0, 0, 0))
+        return self._tokens
+
+    def __exit__(self, exc_type: Optional[Exception], exc_value: Optional[Exception], traceback: Optional[Any]) -> None:
+        captured_metrics = _token_metrics.get()
+        self._tokens.update(captured_metrics)

azure/ai/evaluation/_legacy/_batch_engine/_result.py

@@ -0,0 +1,107 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+from dataclasses import dataclass
+from datetime import datetime, timedelta
+from typing import Any, Mapping, Optional, Sequence
+
+from ._status import BatchStatus
+
+
+@dataclass
+class TokenMetrics:
+    """The token metrics of a run."""
+
+    prompt_tokens: int
+    """The number of tokens used in the prompt for the run."""
+    completion_tokens: int
+    """The number of tokens used in the completion for the run."""
+    total_tokens: int
+    """The total number of tokens used in the run."""
+
+    def update(self, other: "TokenMetrics") -> None:
+        """Update the token metrics with another set of token metrics."""
+        self.prompt_tokens += other.prompt_tokens
+        self.completion_tokens += other.completion_tokens
+        self.total_tokens += other.total_tokens
+
+
+@dataclass
+class BatchRunError:
+    """The error of a batch run."""
+
+    details: str
+    """The details of the error."""
+    exception: Optional[BaseException]
+    """The exception of the error."""
+
+
+@dataclass
+class BatchRunDetails:
+    """The error of a line in a batch run."""
+
+    id: str
+    """The ID of the line run."""
+    status: BatchStatus
+    """The status of the line run."""
+    result: Optional[Mapping[str, Any]]
+    """The result of the line run."""
+    start_time: Optional[datetime]
+    """The start time of the line run. If this was never started, this should be None."""
+    end_time: Optional[datetime]
+    """The end time of the line run. If this never completed, this should be None."""
+    tokens: TokenMetrics
+    """The token metrics of the line run."""
+    error: Optional[BatchRunError]
+    """The error of the line run. This will only be set if the status is Failed."""
+    index: int
+    """The line run index."""
+
+    @property
+    def duration(self) -> timedelta:
+        """The duration of the line run."""
+        if self.start_time is not None and self.end_time is not None:
+            return self.end_time - self.start_time
+        return timedelta(0)
+
+    @staticmethod
+    def create_id(run_id: str, index: int) -> str:
+        """Helper method to create the ID for a line run."""
+        return f"{run_id}_{index}"
+
+
+@dataclass
+class BatchResult:
+    """The result of a batch run."""
+
+    status: BatchStatus
+    """The overall status of the batch run."""
+    total_lines: int
+    """The total number of lines in the batch run."""
+    failed_lines: int
+    """The number of failed lines in the batch run."""
+    start_time: datetime
+    """The start time of the batch run."""
+    end_time: datetime
+    """The end time of the batch run."""
+    tokens: TokenMetrics
+    """The overall token metrics of the batch run."""
+    details: Sequence[BatchRunDetails]
+    """The details of each line in the batch run."""
+    error: Optional[Exception] = None
+    """The error of the batch run. This will only be set if the status does not indicate success."""
+
+    @property
+    def duration(self) -> timedelta:
+        """The duration of the batch run."""
+        if self.start_time is not None and self.end_time is not None:
+            return self.end_time - self.start_time
+        return timedelta(0)
+
+    @property
+    def results(self) -> Sequence[Optional[Mapping[str, Any]]]:
+        """The results of the batch run."""
+        if not self.details:
+            return []
+        return [d.result for d in self.details]

azure/ai/evaluation/_legacy/_batch_engine/_run.py

@@ -0,0 +1,127 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+from uuid import uuid4
+from datetime import datetime, timedelta, timezone
+from enum import Enum
+from typing import Any, Callable, Mapping, Optional, Sequence
+
+from ._utils import normalize_identifier_name
+from ._result import BatchResult
+from ._status import BatchStatus
+
+
+class RunStatus(Enum):
+    # TODO ralphe: Trim this to just the statuses we need
+    # QUEUED = "Queued"
+    NOT_STARTED = "NotStarted"
+    PREPARING = "Preparing"
+    # PROVISIONING = "Provisioning"
+    # STARTING = "Starting"
+    RUNNING = "Running"
+    # CANCEL_REQUESTED = "CancelRequested"
+    CANCELED = "Canceled"
+    # FINALIZING = "Finalizing"
+    COMPLETED = "Completed"
+    FAILED = "Failed"
+    # UNAPPROVED = "Unapproved"
+    # NOTRESPONDING = "NotResponding"
+    # PAUSING = "Pausing"
+    # PAUSED = "Paused"
+
+    @staticmethod
+    def from_batch_result_status(status: BatchStatus) -> "RunStatus":
+        if status == BatchStatus.NotStarted:
+            return RunStatus.NOT_STARTED
+        if status == BatchStatus.Running:
+            return RunStatus.RUNNING
+        if status == BatchStatus.Completed:
+            return RunStatus.COMPLETED
+        if status == BatchStatus.Canceled:
+            return RunStatus.CANCELED
+        if status == BatchStatus.Failed:
+            return RunStatus.FAILED
+
+        return RunStatus.FAILED
+
+
+class Run:
+    """The equivalent of a Promptflow Run
+    promptflow-devkit/promptflow/_sdk/entities/_run.py
+
+    THIS WILL BE REMOVED IN A FUTURE CODE UPDATE"""
+
+    def __init__(
+        self,
+        *,
+        dynamic_callable: Callable,
+        name_prefix: Optional[str],
+        inputs: Sequence[Mapping[str, Any]],
+        column_mapping: Optional[Mapping[str, str]] = None,
+        created_on: Optional[datetime] = None,
+        run: Optional["Run"] = None,
+    ):
+        self._status: RunStatus = RunStatus.NOT_STARTED
+        self._created_on = created_on or datetime.now(timezone.utc)
+        self._start_time: Optional[datetime] = None
+        self._end_time: Optional[datetime] = None
+
+        self.dynamic_callable = dynamic_callable
+        self.name = self._generate_run_name(name_prefix, self._created_on)
+        self.inputs = inputs
+        self.column_mapping: Optional[Mapping[str, str]] = column_mapping
+        self.result: Optional[BatchResult] = None
+        self.metrics: Mapping[str, Any] = {}
+        self._run = run
+
+        # self._use_remote_flow = False
+        # self._from_flex_flow = True
+        # self._from_prompty = False
+        # self.flow = path to pointless flow file
+        # self._experiment_name = name of folder containing pointless flow file
+        # self._lineage_id = basically equivalent to a hex digest of the SHA256 hash of:
+        #                    f"{uuid.getnod()}/{posix_full_path_to_pointless_folder}"
+        # self._output_path = Path("<user_folder>/.promptflow/runs/<self.name>")
+        # self._flow_name = name of pointless folder
+
+    @property
+    def status(self) -> RunStatus:
+        return self._status
+
+    @property
+    def created_on(self) -> datetime:
+        return self._created_on
+
+    @property
+    def duration(self) -> Optional[timedelta]:
+        if self._start_time is None or self._end_time is None:
+            return None
+
+        return self._end_time - self._start_time
+
+    @property
+    def outputs(self) -> Sequence[Mapping[str, Any]]:
+        if self.result is None:
+            return []
+
+        return [value or {} for value in self.result.results]
+
+    @property
+    def previous_run(self) -> Optional["Run"]:
+        return self._run
+
+    @staticmethod
+    def _generate_run_name(name_prefix: Optional[str], creation_time: datetime) -> str:
+        # The Promptflow code looked at the folder name  of the temporary folder used to
+        # store the temporary flow YAML file which was a single entry that told it look
+        # at the passed in dynamic_callable. Example folder name:
+        # azure_ai_evaluation_evaluators_common_base_eval_asyncevaluatorbase_l82059h3
+        # instead we will use the passed in name_prefix or use a UUID (which is equally
+        # opaque as what the original code did)
+        if not name_prefix:
+            name_prefix = str(uuid4())
+
+        timestamp = creation_time.strftime("%Y%m%d_%H%M%S_%f")
+        name = f"{name_prefix}_{timestamp}"
+        return normalize_identifier_name(name)

azure/ai/evaluation/_legacy/_batch_engine/_run_storage.py

@@ -0,0 +1,128 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+# Original source:
+# promptflow-devkit/promptflow/_sdk/operations/_local_storage_operations.py
+
+from abc import ABC, abstractmethod
+from contextlib import AbstractContextManager
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any, Final, Mapping, Optional, Tuple, Union
+
+from ._result import BatchResult, TokenMetrics, BatchStatus
+
+
+EVAL_USER_SUBFOLDER: Final[str] = ".evaluation"
+
+
+class AbstractRunLogger(AbstractContextManager):
+    @property
+    @abstractmethod
+    def file_path(self) -> Path:
+        """Get the file path of the logger.
+
+        :return: The file path of the logger.
+        :rtype: Path
+        """
+        ...
+
+    @abstractmethod
+    def get_logs(self) -> str:
+        """Get the logs of the run.
+
+        :return: The logs of the run.
+        :rtype: str
+        """
+        ...
+
+
+class AbstractRunStorage(ABC):
+    @property
+    @abstractmethod
+    def logger(self) -> "AbstractRunLogger":
+        """Get the logger of the run.
+
+        :return: The logger of the run.
+        :rtype: ~promptflow.contracts.run_logger.RunLogger
+        """
+        ...
+
+    @abstractmethod
+    def persist_result(self, result: Optional[BatchResult]) -> None:
+        """Persist results of a batch engine execution (including any errors).
+
+        :param Optional[BatchResult] result: The result to persist.
+        """
+        ...
+
+    @abstractmethod
+    def load_exception(self) -> Mapping[str, Any]:
+        """Load the exception from the storage. If there was no exception, an empty
+        mapping will be returned.
+
+        :return: The exception.
+        :rtype: Optional[Exception]
+        """
+        ...
+
+    @abstractmethod
+    def load_inputs_and_outputs(self) -> Tuple[Mapping[str, Any], BatchResult]:
+        """Load the inputs and outputs from the storage.
+
+        :return: The inputs and outputs.
+        :rtype: Tuple(Mapping[str, Any], BatchResult)
+        """
+        ...
+
+    @abstractmethod
+    def load_metrics(self) -> Mapping[str, Union[int, float, str]]:
+        """Load the metrics from the storage.
+
+        :return: The metrics.
+        :rtype: Mapping[str, Union[int, float, str]]
+        """
+        ...
+
+
+class NoOpRunStorage(AbstractRunStorage):
+    """A no-op implementation of the run storage."""
+
+    def __init__(self):
+        self._logger = NoOpLogger()
+        pass
+
+    @property
+    def logger(self) -> AbstractRunLogger:
+        return self._logger
+
+    def persist_result(self, result: Optional[BatchResult]) -> None:
+        pass
+
+    def load_exception(self) -> Mapping[str, Any]:
+        return {}
+
+    def load_inputs_and_outputs(self) -> Tuple[Mapping[str, Any], BatchResult]:
+        now = datetime.now(timezone.utc)
+        return {}, BatchResult(BatchStatus.NotStarted, 0, 0, now, now, TokenMetrics(0, 0, 0), [])
+
+    def load_metrics(self) -> Mapping[str, Union[int, float, str]]:
+        return {}
+
+
+class NoOpLogger(AbstractRunLogger):
+    """A no-op implementation of the run logger."""
+
+    @property
+    def file_path(self) -> Path:
+        return Path.home() / EVAL_USER_SUBFOLDER
+
+    def __enter__(self) -> None:
+        pass
+
+    def __exit__(self, *args) -> None:
+        pass
+
+    def get_logs(self) -> str:
+        return ""