azure-ai-evaluation 1.3.0__py3-none-any.whl → 1.5.0__py3-none-any.whl

azure/ai/evaluation/_legacy/_batch_engine/_logging.py

@@ -0,0 +1,292 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+# Original source:
+# - promptflow-core/promptflow/_core/log_manager.py
+# - promptflow-core/promptflow/_utils/logger_utils.py
+
+import os
+import logging
+import re
+import sys
+from re import Pattern
+from contextvars import ContextVar
+from datetime import datetime, timezone
+from dataclasses import dataclass
+from io import StringIO, TextIOBase
+from typing import Any, Dict, Final, Mapping, Optional, Set, TextIO, Tuple, Union
+
+
+valid_logging_level: Final[Set[str]] = {"CRITICAL", "FATAL", "ERROR", "WARN", "WARNING", "INFO", "DEBUG", "NOTSET"}
+
+
+def get_pf_logging_level(default=logging.INFO):
+    logging_level = os.environ.get("PF_LOGGING_LEVEL", None)
+    if logging_level not in valid_logging_level:
+        # Fall back to info if user input is invalid.
+        logging_level = default
+    return logging_level
+
+
+def _get_format_for_logger(
+    default_log_format: Optional[str] = None, default_date_format: Optional[str] = None
+) -> Tuple[str, str]:
+    """
+    Get the logging format and date format for logger.
+
+    This function attempts to find the handler of the root logger with a configured formatter.
+    If such a handler is found, it returns the format and date format used by this handler.
+    This can be configured through logging.basicConfig. If no configured formatter is found,
+    it defaults to LOG_FORMAT and DATETIME_FORMAT.
+    """
+    log_format = (
+        os.environ.get("PF_LOG_FORMAT")
+        or default_log_format
+        or "%(asctime)s %(thread)7d %(name)-18s %(levelname)-8s %(message)s"
+    )
+    datetime_format = os.environ.get("PF_LOG_DATETIME_FORMAT") or default_date_format or "%Y-%m-%d %H:%M:%S %z"
+    return log_format, datetime_format
+
+
+def get_logger(name: str) -> logging.Logger:
+    """Get logger used during execution."""
+    logger = logging.Logger(name)
+    logger.setLevel(get_pf_logging_level())
+    # logger.addHandler(FileHandlerConcurrentWrapper())
+    stdout_handler = logging.StreamHandler(sys.stdout)
+    fmt, datefmt = _get_format_for_logger()
+    # TODO ralphe: Do we need a credentials scrubber here like the old code had? We are not logging
+    #              logging anything that sensitive here.
+    stdout_handler.setFormatter(logging.Formatter(fmt=fmt, datefmt=datefmt))
+    logger.addHandler(stdout_handler)
+    return logger
+
+
+def scrub_credentials(s: str):
+    """Scrub credentials in string s.
+
+    For example, for input string: "print accountkey=accountKey", the output will be:
+    "print accountkey=**data_scrubbed**"
+    """
+    # for h in logger.handlers:
+    #     if isinstance(h, FileHandlerConcurrentWrapper):
+    #         if h.handler and h.handler._formatter:
+    #             credential_scrubber = h.handler._formatter.credential_scrubber
+    #             if credential_scrubber:
+    #                 return credential_scrubber.scrub(s)
+    return CredentialScrubber.scrub(s)
+
+
+class CredentialScrubber:
+    """Scrub sensitive information in string."""
+
+    PLACE_HOLDER = "**data_scrubbed**"
+    LENGTH_THRESHOLD = 2
+    DEFAULT_REGEX_SET: Final[Set[Pattern[str]]] = {
+        re.compile(r"(?<=sig=)[^\s;&]+", flags=re.IGNORECASE),  # Replace signature.
+        re.compile(r"(?<=key=)[^\s;&]+", flags=re.IGNORECASE),  # Replace key.
+    }
+
+    @staticmethod
+    def scrub(input: str) -> str:
+        """Replace sensitive information in input string with PLACE_HOLDER.
+
+        For example, for input string: "print accountkey=accountKey", the output will be:
+        "print accountkey=**data_scrubbed**"
+        """
+        output = input
+        for regex in CredentialScrubber.DEFAULT_REGEX_SET:
+            output = regex.sub(CredentialScrubber.PLACE_HOLDER, output)
+        return output
+
+
+# Logs by flow_logger will only be shown in flow mode.
+# These logs should contain all detailed logs from executor and runtime.
+flow_logger = get_logger("execution.flow")
+
+# Logs by bulk_logger will only be shown in bulktest and eval modes.
+# These logs should contain overall progress logs and error logs.
+bulk_logger = get_logger("execution.bulk")
+
+# Logs by logger will be shown in all the modes above,
+# such as error logs.
+logger = get_logger("execution")
+
+
+def log_progress(
+    run_start_time: datetime,
+    total_count: int,
+    current_count: int,
+    logger: logging.Logger = bulk_logger,
+    formatter="Finished {count} / {total_count} lines.",
+) -> None:
+    if current_count > 0:
+        delta = datetime.now(timezone.utc).timestamp() - run_start_time.timestamp()
+        average_execution_time = round(delta / current_count, 2)
+        estimated_execution_time = round(average_execution_time * (total_count - current_count), 2)
+        logger.info(formatter.format(count=current_count, total_count=total_count))
+        logger.info(
+            f"Average execution time for completed lines: {average_execution_time} seconds. "
+            f"Estimated time for incomplete lines: {estimated_execution_time} seconds."
+        )
+
+
+def incremental_print(log: str, printed: int, fileout: Union[TextIO, Any]) -> int:
+    count = 0
+    for line in log.splitlines():
+        if count >= printed:
+            fileout.write(line + "\n")
+            printed += 1
+        count += 1
+    return printed
+
+
+def print_red_error(message):
+    try:
+        from colorama import Fore, init
+
+        init(autoreset=True)
+        print(Fore.RED + message)
+    except ImportError:
+        print(message)
+
+
+@dataclass
+class NodeInfo:
+    run_id: str
+    node_name: str
+    line_number: int
+
+
+class NodeLogManager:
+    """Replace sys.stdout and sys.stderr with NodeLogWriter.
+
+    This class intercepts and saves logs to stdout/stderr when executing a node. For example:
+    with NodeLogManager() as log_manager:
+        print('test stdout')
+        print('test stderr', file=sys.stderr)
+
+    log_manager.get_logs() will return: {'stdout': 'test stdout\n', 'stderr': 'test stderr\n'}
+    """
+
+    def __init__(self, record_datetime: bool = True):
+        self.stdout_logger = NodeLogWriter(sys.stdout, record_datetime)
+        self.stderr_logger = NodeLogWriter(sys.stderr, record_datetime, is_stderr=True)
+
+    def __enter__(self) -> "NodeLogManager":
+        """Replace sys.stdout and sys.stderr with NodeLogWriter."""
+        self._prev_stdout = sys.stdout
+        self._prev_stderr = sys.stderr
+        sys.stdout = self.stdout_logger
+        sys.stderr = self.stderr_logger
+        return self
+
+    def __exit__(self, *args) -> None:
+        """Restore sys.stdout and sys.stderr."""
+        sys.stdout = self._prev_stdout
+        sys.stderr = self._prev_stderr
+
+    def set_node_context(self, run_id: str, node_name: str, line_number: int) -> None:
+        """Set node context."""
+        self.stdout_logger.set_node_info(run_id, node_name, line_number)
+        self.stderr_logger.set_node_info(run_id, node_name, line_number)
+
+    def clear_node_context(self, run_id: str) -> None:
+        """Clear node context."""
+        self.stdout_logger.clear_node_info(run_id)
+        self.stderr_logger.clear_node_info(run_id)
+
+    def get_logs(self, run_id: str) -> Mapping[str, str]:
+        return {
+            "stdout": self.stdout_logger.get_log(run_id),
+            "stderr": self.stderr_logger.get_log(run_id),
+        }
+
+
+class NodeLogWriter(TextIOBase):
+    """Record node run logs."""
+
+    DATETIME_FORMAT = "%Y-%m-%dT%H:%M:%S%z"
+
+    def __init__(self, prev_stdout: Union[TextIOBase, Any], record_datetime: bool = True, is_stderr: bool = False):
+        self.run_id_to_stdout: Dict[str, StringIO] = {}
+        self._context: ContextVar[Optional[NodeInfo]] = ContextVar("run_log_info", default=None)
+        self._prev_out: Union[TextIOBase, Any] = prev_stdout
+        self._record_datetime: bool = record_datetime
+        self._is_stderr: bool = is_stderr
+
+    def set_node_info(self, run_id: str, node_name: str, line_number: int) -> None:
+        """Set node info to a context variable.
+
+        After set node info, write method will write to string IO associated with this node.
+        """
+        run_log_info = NodeInfo(run_id, node_name, line_number)
+        self._context.set(run_log_info)
+        self.run_id_to_stdout.update({run_id: StringIO()})
+
+    def clear_node_info(self, run_id: str):
+        """Clear context variable associated with run id."""
+        log_info: Optional[NodeInfo] = self._context.get()
+        if log_info and log_info.run_id == run_id:
+            self._context.set(None)
+
+        if run_id in self.run_id_to_stdout:
+            self.run_id_to_stdout.pop(run_id)
+
+    def get_log(self, run_id: str) -> str:
+        """Get log associated with run id."""
+        string_io: Optional[StringIO] = self.run_id_to_stdout.get(run_id)
+        if string_io is None:
+            return ""
+
+        return string_io.getvalue()
+
+    def write(self, s: str) -> int:
+        """Override TextIO's write method and writes input string into a string IO
+
+        The written string is compliant without any credentials.
+        The string is also recorded to flow/bulk logger.
+        If node info is not set, write to previous stdout.
+        """
+        log_info: Optional[NodeInfo] = self._context.get()
+        s = scrub_credentials(s)  # Remove credential from string.
+        if log_info is None:
+            return self._prev_out.write(s)
+        else:
+            self._write_to_flow_log(log_info, s)
+            stdout: Optional[StringIO] = self.run_id_to_stdout.get(log_info.run_id)
+            # When the line execution timeout is reached, all running nodes will be cancelled and node info will
+            # be cleared. This will remove StringIO from self.run_id_to_stdout. For sync tools running in a worker
+            # thread, they can't be stopped and self._context won't change in the worker
+            # thread because it's a thread-local variable. Therefore, we need to check if StringIO is None here.
+            if stdout is None:
+                return 0
+            if self._record_datetime and s != "\n":  # For line breaker, do not add datetime prefix.
+                s = f"[{datetime.now(timezone.utc).strftime(self.DATETIME_FORMAT)}] {s}"
+            return stdout.write(s)
+
+    def flush(self):
+        """Override TextIO's flush method."""
+        node_info: Optional[NodeInfo] = self._context.get()
+        if node_info is None:
+            self._prev_out.flush()
+        else:
+            string_io = self.run_id_to_stdout.get(node_info.run_id)
+            if string_io is not None:
+                string_io.flush()
+
+    def _write_to_flow_log(self, log_info: NodeInfo, s: str):
+        """Save stdout log to flow_logger and stderr log to logger."""
+        # If user uses "print('log message.')" to log, then
+        # "write" method will be called twice and the second time input is only '\n'.
+        # For this case, should not log '\n' in flow_logger.
+        if s != "\n":
+            if self._is_stderr:
+                flow_log = f"[{str(log_info)}] stderr> " + s.rstrip("\n")
+                # Log stderr in all scenarios so we can diagnose problems.
+                logger.warning(flow_log)
+            else:
+                flow_log = f"[{str(log_info)}] stdout> " + s.rstrip("\n")
+                # Log stdout only in flow mode.
+                flow_logger.info(flow_log)

azure/ai/evaluation/_legacy/_batch_engine/_openai_injector.py

@@ -0,0 +1,23 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+# Original source code: promptflow-tracing/promptflow/tracing/_integrations/_openai_injector.py
+
+
+def inject_openai_api():
+    """This function:
+    1. Modifies the create methods of the OpenAI API classes to inject logic before calling the original methods.
+    It stores the original methods as _original attributes of the create methods.
+    2. Updates the openai api configs from environment variables.
+    """
+    # TODO ralphe: Port function?
+    pass
+
+
+def recover_openai_api():
+    """This function restores the original create methods of the OpenAI API classes
+    by assigning them back from the _original attributes of the modified methods.
+    """
+    # TODO ralphe: Port function?
+    pass

azure/ai/evaluation/_legacy/_batch_engine/_result.py

@@ -0,0 +1,99 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+from dataclasses import dataclass
+from datetime import datetime, timedelta
+from typing import Any, Mapping, Optional, Sequence
+
+from ._status import BatchStatus
+
+
+@dataclass
+class TokenMetrics:
+    """The token metrics of a run."""
+
+    prompt_tokens: int
+    """The number of tokens used in the prompt for the run."""
+    completion_tokens: int
+    """The number of tokens used in the completion for the run."""
+    total_tokens: int
+    """The total number of tokens used in the run."""
+
+
+@dataclass
+class BatchRunError:
+    """The error of a batch run."""
+
+    details: str
+    """The details of the error."""
+    exception: Optional[BaseException]
+    """The exception of the error."""
+
+
+@dataclass
+class BatchRunDetails:
+    """The error of a line in a batch run."""
+
+    id: str
+    """The ID of the line run."""
+    status: BatchStatus
+    """The status of the line run."""
+    result: Optional[Mapping[str, Any]]
+    """The result of the line run."""
+    start_time: Optional[datetime]
+    """The start time of the line run. If this was never started, this should be None."""
+    end_time: Optional[datetime]
+    """The end time of the line run. If this never completed, this should be None."""
+    tokens: TokenMetrics
+    """The token metrics of the line run."""
+    error: Optional[BatchRunError]
+    """The error of the line run. This will only be set if the status is Failed."""
+
+    @property
+    def duration(self) -> timedelta:
+        """The duration of the line run."""
+        if self.start_time is not None and self.end_time is not None:
+            return self.end_time - self.start_time
+        return timedelta(0)
+
+    @staticmethod
+    def create_id(run_id: str, index: int) -> str:
+        """Helper method to create the ID for a line run."""
+        return f"{run_id}_{index}"
+
+
+@dataclass
+class BatchResult:
+    """The result of a batch run."""
+
+    status: BatchStatus
+    """The overall status of the batch run."""
+    total_lines: int
+    """The total number of lines in the batch run."""
+    failed_lines: int
+    """The number of failed lines in the batch run."""
+    start_time: datetime
+    """The start time of the batch run."""
+    end_time: datetime
+    """The end time of the batch run."""
+    tokens: TokenMetrics
+    """The overall token metrics of the batch run."""
+    details: Sequence[BatchRunDetails]
+    """The details of each line in the batch run."""
+    error: Optional[Exception] = None
+    """The error of the batch run. This will only be set if the status does not indicate success."""
+
+    @property
+    def duration(self) -> timedelta:
+        """The duration of the batch run."""
+        if self.start_time is not None and self.end_time is not None:
+            return self.end_time - self.start_time
+        return timedelta(0)
+
+    @property
+    def results(self) -> Sequence[Optional[Mapping[str, Any]]]:
+        """The results of the batch run."""
+        if not self.details:
+            return []
+        return [d.result for d in self.details]

azure/ai/evaluation/_legacy/_batch_engine/_run.py

@@ -0,0 +1,121 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+from uuid import uuid4
+from datetime import datetime, timedelta, timezone
+from enum import Enum
+from typing import Any, Callable, Mapping, Optional, Sequence
+
+from ._utils import normalize_identifier_name
+from ._result import BatchResult
+from ._status import BatchStatus
+
+
+class RunStatus(Enum):
+    # TODO ralphe: Trim this to just the statuses we need
+    # QUEUED = "Queued"
+    NOT_STARTED = "NotStarted"
+    PREPARING = "Preparing"
+    # PROVISIONING = "Provisioning"
+    # STARTING = "Starting"
+    RUNNING = "Running"
+    # CANCEL_REQUESTED = "CancelRequested"
+    CANCELED = "Canceled"
+    # FINALIZING = "Finalizing"
+    COMPLETED = "Completed"
+    FAILED = "Failed"
+    # UNAPPROVED = "Unapproved"
+    # NOTRESPONDING = "NotResponding"
+    # PAUSING = "Pausing"
+    # PAUSED = "Paused"
+
+    @staticmethod
+    def from_batch_result_status(status: BatchStatus) -> "RunStatus":
+        if status == BatchStatus.NotStarted:
+            return RunStatus.NOT_STARTED
+        if status == BatchStatus.Running:
+            return RunStatus.RUNNING
+        if status == BatchStatus.Completed:
+            return RunStatus.COMPLETED
+        if status == BatchStatus.Canceled:
+            return RunStatus.CANCELED
+        if status == BatchStatus.Failed:
+            return RunStatus.FAILED
+
+        return RunStatus.FAILED
+
+
+class Run:
+    """The equivalent of a Promptflow Run
+    promptflow-devkit/promptflow/_sdk/entities/_run.py
+
+    THIS WILL BE REMOVED IN A FUTURE CODE UPDATE"""
+
+    def __init__(
+        self,
+        *,
+        dynamic_callable: Callable,
+        name_prefix: Optional[str],
+        inputs: Sequence[Mapping[str, Any]],
+        column_mapping: Mapping[str, str],
+        created_on: Optional[datetime] = None,
+    ):
+        self._status: RunStatus = RunStatus.NOT_STARTED
+        self._created_on = created_on or datetime.now(timezone.utc)
+        self._start_time: Optional[datetime] = None
+        self._end_time: Optional[datetime] = None
+
+        self.dynamic_callable = dynamic_callable
+        self.name = self._generate_run_name(name_prefix, self._created_on)
+        self.inputs = inputs
+        self.column_mapping = column_mapping
+        self.result: Optional[BatchResult] = None
+        self.metrics: Mapping[str, Any] = {}
+
+        # self._use_remote_flow = False
+        # self._from_flex_flow = True
+        # self._from_prompty = False
+        # self.flow = path to pointless flow file
+        # self._experiment_name = name of folder containing pointless flow file
+        # self._lineage_id = basically equivalent to a hex digest of the SHA256 hash of:
+        #                    f"{uuid.getnod()}/{posix_full_path_to_pointless_folder}"
+        # self._output_path = Path("<user_folder>/.promptflow/runs/<self.name>")
+        # self._flow_name = name of pointless folder
+
+    @property
+    def status(self) -> RunStatus:
+        return self._status
+
+    @property
+    def created_on(self) -> datetime:
+        return self._created_on
+
+    @property
+    def duration(self) -> Optional[timedelta]:
+        if self._start_time is None or self._end_time is None:
+            return None
+
+        return self._end_time - self._start_time
+
+    @property
+    def outputs(self) -> Sequence[Mapping[str, Any]]:
+        if self.result is None:
+            return []
+
+        return [value or {} for value in self.result.results]
+
+    @staticmethod
+    def _generate_run_name(name_prefix: Optional[str], creation_time: datetime) -> str:
+        # The Promptflow code looked at the folder name  of the temporary folder used to
+        # store the temporary flow YAML file which was a single entry that told it look
+        # at the passed in dynamic_callable. Example folder name:
+        # azure_ai_evaluation_evaluators_common_base_eval_asyncevaluatorbase_l82059h3
+        # instead we will use the passed in name_prefix or use a UUID (which is equally
+        # opaque as what the original code did)
+        if not name_prefix:
+            name_prefix = str(uuid4())
+
+        timestamp = creation_time.strftime("%Y%m%d_%H%M%S_%f")
+        name = f"{name_prefix}_{timestamp}"
+        return normalize_identifier_name(name)

azure/ai/evaluation/_legacy/_batch_engine/_run_storage.py

@@ -0,0 +1,128 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+# Original source:
+# promptflow-devkit/promptflow/_sdk/operations/_local_storage_operations.py
+
+from abc import ABC, abstractmethod
+from contextlib import AbstractContextManager
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any, Final, Mapping, Optional, Tuple, Union
+
+from ._result import BatchResult, TokenMetrics, BatchStatus
+
+
+EVAL_USER_SUBFOLDER: Final[str] = ".evaluation"
+
+
+class AbstractRunLogger(AbstractContextManager):
+    @property
+    @abstractmethod
+    def file_path(self) -> Path:
+        """Get the file path of the logger.
+
+        :return: The file path of the logger.
+        :rtype: Path
+        """
+        ...
+
+    @abstractmethod
+    def get_logs(self) -> str:
+        """Get the logs of the run.
+
+        :return: The logs of the run.
+        :rtype: str
+        """
+        ...
+
+
+class AbstractRunStorage(ABC):
+    @property
+    @abstractmethod
+    def logger(self) -> "AbstractRunLogger":
+        """Get the logger of the run.
+
+        :return: The logger of the run.
+        :rtype: ~promptflow.contracts.run_logger.RunLogger
+        """
+        ...
+
+    @abstractmethod
+    def persist_result(self, result: Optional[BatchResult]) -> None:
+        """Persist results of a batch engine execution (including any errors).
+
+        :param Optional[BatchResult] result: The result to persist.
+        """
+        ...
+
+    @abstractmethod
+    def load_exception(self) -> Mapping[str, Any]:
+        """Load the exception from the storage. If there was no exception, an empty
+        mapping will be returned.
+
+        :return: The exception.
+        :rtype: Optional[Exception]
+        """
+        ...
+
+    @abstractmethod
+    def load_inputs_and_outputs(self) -> Tuple[Mapping[str, Any], BatchResult]:
+        """Load the inputs and outputs from the storage.
+
+        :return: The inputs and outputs.
+        :rtype: Tuple(Mapping[str, Any], BatchResult)
+        """
+        ...
+
+    @abstractmethod
+    def load_metrics(self) -> Mapping[str, Union[int, float, str]]:
+        """Load the metrics from the storage.
+
+        :return: The metrics.
+        :rtype: Mapping[str, Union[int, float, str]]
+        """
+        ...
+
+
+class NoOpRunStorage(AbstractRunStorage):
+    """A no-op implementation of the run storage."""
+
+    def __init__(self):
+        self._logger = NoOpLogger()
+        pass
+
+    @property
+    def logger(self) -> AbstractRunLogger:
+        return self._logger
+
+    def persist_result(self, result: Optional[BatchResult]) -> None:
+        pass
+
+    def load_exception(self) -> Mapping[str, Any]:
+        return {}
+
+    def load_inputs_and_outputs(self) -> Tuple[Mapping[str, Any], BatchResult]:
+        now = datetime.now(timezone.utc)
+        return {}, BatchResult(BatchStatus.NotStarted, 0, 0, now, now, TokenMetrics(0, 0, 0), [])
+
+    def load_metrics(self) -> Mapping[str, Union[int, float, str]]:
+        return {}
+
+
+class NoOpLogger(AbstractRunLogger):
+    """A no-op implementation of the run logger."""
+
+    @property
+    def file_path(self) -> Path:
+        return Path.home() / EVAL_USER_SUBFOLDER
+
+    def __enter__(self) -> None:
+        pass
+
+    def __exit__(self, *args) -> None:
+        pass
+
+    def get_logs(self) -> str:
+        return ""