azure-ai-evaluation 1.0.0b1__py3-none-any.whl → 1.0.0b3__py3-none-any.whl

azure/ai/evaluation/__init__.py

@@ -4,10 +4,8 @@
 
 from ._evaluate._evaluate import evaluate
 from ._evaluators._bleu import BleuScoreEvaluator
-from ._evaluators._chat import ChatEvaluator
 from ._evaluators._coherence import CoherenceEvaluator
 from ._evaluators._content_safety import (
-    ContentSafetyChatEvaluator,
     ContentSafetyEvaluator,
     HateUnfairnessEvaluator,
     SelfHarmEvaluator,
@@ -22,6 +20,7 @@ from ._evaluators._meteor import MeteorScoreEvaluator
 from ._evaluators._protected_material import ProtectedMaterialEvaluator
 from ._evaluators._qa import QAEvaluator
 from ._evaluators._relevance import RelevanceEvaluator
+from ._evaluators._retrieval import RetrievalEvaluator
 from ._evaluators._rouge import RougeScoreEvaluator, RougeType
 from ._evaluators._similarity import SimilarityEvaluator
 from ._evaluators._xpia import IndirectAttackEvaluator
@@ -29,6 +28,7 @@ from ._model_configurations import (
     AzureAIProject,
     AzureOpenAIModelConfiguration,
     OpenAIModelConfiguration,
+    EvaluatorConfig,
 )
 
 __all__ = [
@@ -40,21 +40,21 @@ __all__ = [
     "RelevanceEvaluator",
     "SimilarityEvaluator",
     "QAEvaluator",
-    "ChatEvaluator",
     "ViolenceEvaluator",
     "SexualEvaluator",
     "SelfHarmEvaluator",
     "HateUnfairnessEvaluator",
     "ContentSafetyEvaluator",
-    "ContentSafetyChatEvaluator",
     "IndirectAttackEvaluator",
     "BleuScoreEvaluator",
     "GleuScoreEvaluator",
     "MeteorScoreEvaluator",
+    "RetrievalEvaluator",
     "RougeScoreEvaluator",
     "RougeType",
     "ProtectedMaterialEvaluator",
     "AzureAIProject",
     "AzureOpenAIModelConfiguration",
     "OpenAIModelConfiguration",
+    "EvaluatorConfig",
 ]

azure/ai/evaluation/_common/rai_service.py

@@ -11,12 +11,12 @@ from urllib.parse import urlparse
 
 import jwt
 import numpy as np
-from azure.core.credentials import TokenCredential
-from azure.identity import DefaultAzureCredential
 
+from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException
 from azure.ai.evaluation._http_utils import get_async_http_client
-from azure.ai.evaluation._exceptions import EvaluationException, ErrorBlame, ErrorCategory, ErrorTarget
 from azure.ai.evaluation._model_configurations import AzureAIProject
+from azure.core.credentials import TokenCredential
+from azure.identity import DefaultAzureCredential
 
 from .constants import (
     CommonConstants,
@@ -348,7 +348,7 @@ async def _get_service_discovery_url(azure_ai_project: AzureAIProject, token: st
         )
 
     if response.status_code != 200:
-        msg = f"Failed to retrieve the discovery service URL."
+        msg = "Failed to retrieve the discovery service URL."
         raise EvaluationException(
             message=msg,
             internal_message=msg,

azure/ai/evaluation/_common/utils.py

@@ -2,20 +2,16 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
 
-from typing import Optional, Union
-
-from azure.ai.evaluation._model_configurations import AzureOpenAIModelConfiguration, OpenAIModelConfiguration
+import threading
+from typing import List, Union
 
-try:
-    from . import constants
-except ImportError:
-    import constants
+import nltk
+import numpy as np
 
-from typing import List
+from azure.ai.evaluation._model_configurations import AzureOpenAIModelConfiguration, OpenAIModelConfiguration
+from azure.ai.evaluation._constants import AZURE_OPENAI_TYPE, OPENAI_TYPE
 
-import threading
-import numpy as np
-import nltk
+from . import constants
 
 _nltk_data_download_lock = threading.Lock()
 
@@ -46,7 +42,7 @@ def ensure_nltk_data_downloaded():
     """Download NLTK data packages if not already downloaded."""
     with _nltk_data_download_lock:
         try:
-            from nltk.tokenize.nist import NISTTokenizer
+            from nltk.tokenize.nist import NISTTokenizer  # pylint: disable=unused-import
         except LookupError:
             nltk.download("perluniprops")
             nltk.download("punkt")
@@ -54,12 +50,19 @@ def ensure_nltk_data_downloaded():
 
 
 def nltk_tokenize(text: str) -> List[str]:
-    """Tokenize the input text using the NLTK tokenizer."""
+    """Tokenize the input text using the NLTK tokenizer.
+
+    :param text: The text to tokenize
+    :type text: str
+    :return: A list of tokens
+    :rtype: list[str]
+    """
     ensure_nltk_data_downloaded()
 
     if not text.isascii():
         # Use NISTTokenizer for international tokenization
         from nltk.tokenize.nist import NISTTokenizer
+
         tokens = NISTTokenizer().international_tokenize(text)
     else:
         # By default, use NLTK word tokenizer
@@ -68,20 +71,32 @@ def nltk_tokenize(text: str) -> List[str]:
     return list(tokens)
 
 
-def check_and_add_api_version_for_aoai_model_config(
+def parse_model_config_type(
     model_config: Union[AzureOpenAIModelConfiguration, OpenAIModelConfiguration],
-    default_api_version: str,
 ) -> None:
-    if (
-        "azure_endpoint" in model_config or "azure_deployment" in model_config
-    ):
-        model_config["api_version"] = model_config.get("api_version", default_api_version)
+    if "azure_endpoint" in model_config or "azure_deployment" in model_config:
+        model_config["type"] = AZURE_OPENAI_TYPE
+    else:
+        model_config["type"] = OPENAI_TYPE
 
 
-def check_and_add_user_agent_for_aoai_model_config(
+def construct_prompty_model_config(
     model_config: Union[AzureOpenAIModelConfiguration, OpenAIModelConfiguration],
-    prompty_model_config: dict,
-    user_agent: Optional[str] = None,
-) -> None:
-    if user_agent and ("azure_endpoint" in model_config or "azure_deployment" in model_config):
-        prompty_model_config["parameters"]["extra_headers"].update({"x-ms-useragent": user_agent})
+    default_api_version: str,
+    user_agent: str,
+) -> dict:
+    parse_model_config_type(model_config)
+
+    if model_config["type"] == AZURE_OPENAI_TYPE:
+        model_config["api_version"] = model_config.get("api_version", default_api_version)
+
+    prompty_model_config = {"configuration": model_config, "parameters": {"extra_headers": {}}}
+
+    # Handle "RuntimeError: Event loop is closed" from httpx AsyncClient
+    # https://github.com/encode/httpx/discussions/2959
+    prompty_model_config["parameters"]["extra_headers"].update({"Connection": "close"})
+
+    if model_config["type"] == AZURE_OPENAI_TYPE and user_agent:
+        prompty_model_config["parameters"]["extra_headers"].update({"x-ms-useragent": user_agent})
+
+    return prompty_model_config

azure/ai/evaluation/_constants.py

@@ -39,6 +39,15 @@ class Prefixes:
     TSG_OUTPUTS = "__outputs."
 
 
+class DefaultOpenEncoding:
+    """Enum that captures SDK's default values for the encoding param of open(...)"""
+
+    READ = "utf-8-sig"
+    """SDK Default Encoding when reading a file"""
+    WRITE = "utf-8"
+    """SDK Default Encoding when writing a file"""
+
+
 DEFAULT_EVALUATION_RESULTS_FILE_NAME = "evaluation_results.json"
 
 CONTENT_SAFETY_DEFECT_RATE_THRESHOLD_DEFAULT = 4
@@ -48,3 +57,7 @@ PF_BATCH_TIMEOUT_SEC = "PF_BATCH_TIMEOUT_SEC"
 
 OTEL_EXPORTER_OTLP_TRACES_TIMEOUT = "OTEL_EXPORTER_OTLP_TRACES_TIMEOUT"
 OTEL_EXPORTER_OTLP_TRACES_TIMEOUT_DEFAULT = 60
+
+AZURE_OPENAI_TYPE = "azure_openai"
+
+OPENAI_TYPE = "openai"

azure/ai/evaluation/_evaluate/_batch_run_client/batch_run_context.py

@@ -5,13 +5,14 @@ import os
 
 from promptflow._sdk._constants import PF_FLOW_ENTRY_IN_TMP, PF_FLOW_META_LOAD_IN_SUBPROCESS
 from promptflow._utils.user_agent_utils import ClientUserAgentUtil
+from promptflow.tracing._integrations._openai_injector import inject_openai_api, recover_openai_api
+
 from azure.ai.evaluation._constants import (
     OTEL_EXPORTER_OTLP_TRACES_TIMEOUT,
     OTEL_EXPORTER_OTLP_TRACES_TIMEOUT_DEFAULT,
     PF_BATCH_TIMEOUT_SEC,
     PF_BATCH_TIMEOUT_SEC_DEFAULT,
 )
-from promptflow.tracing._integrations._openai_injector import inject_openai_api, recover_openai_api
 
 from ..._user_agent import USER_AGENT
 from .._utils import set_event_loop_policy

azure/ai/evaluation/_evaluate/_batch_run_client/code_client.py

@@ -4,13 +4,16 @@
 import inspect
 import json
 import logging
+import os
+from pathlib import Path
+from typing import Callable, Dict, Optional, Union
 
 import pandas as pd
-
 from promptflow.contracts.types import AttrDict
-from azure.ai.evaluation._evaluate._utils import _apply_column_mapping, _has_aggregator, get_int_env_var, load_jsonl
 from promptflow.tracing import ThreadPoolExecutorWithContext as ThreadPoolExecutor
-from azure.ai.evaluation._exceptions import EvaluationException, ErrorBlame, ErrorCategory, ErrorTarget
+
+from azure.ai.evaluation._evaluate._utils import _apply_column_mapping, _has_aggregator, get_int_env_var, load_jsonl
+from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException
 
 from ..._constants import PF_BATCH_TIMEOUT_SEC, PF_BATCH_TIMEOUT_SEC_DEFAULT
 
@@ -18,7 +21,9 @@ LOGGER = logging.getLogger(__name__)
 
 
 class CodeRun:
-    def __init__(self, run, input_data, evaluator_name=None, aggregated_metrics=None, **kwargs):
+    def __init__(
+        self, run, input_data, evaluator_name=None, aggregated_metrics=None, **kwargs  # pylint: disable=unused-argument
+    ):
         self.run = run
         self.evaluator_name = evaluator_name if evaluator_name is not None else ""
         self.input_data = input_data
@@ -40,13 +45,13 @@ class CodeRun:
                 else None
             )
         except Exception as ex:  # pylint: disable=broad-exception-caught
-            LOGGER.debug(f"Error calculating metrics for evaluator {self.evaluator_name}, failed with error {str(ex)}")
+            LOGGER.debug("Error calculating metrics for evaluator %s, failed with error %s", self.evaluator_name, ex)
             aggregated_metrics = None
 
         if not isinstance(aggregated_metrics, dict):
             LOGGER.warning(
-                f"Aggregated metrics for evaluator {self.evaluator_name}"
-                f" is not a dictionary will not be logged as metrics"
+                "Aggregated metrics for evaluator %s is not a dictionary will not be logged as metrics",
+                self.evaluator_name,
             )
 
         aggregated_metrics = aggregated_metrics if isinstance(aggregated_metrics, dict) else {}
@@ -54,11 +59,15 @@ class CodeRun:
         return aggregated_metrics
 
 
-class CodeClient:
-    def __init__(self):
+class CodeClient:  # pylint: disable=client-accepts-api-version-keyword
+    def __init__(  # pylint: disable=missing-client-constructor-parameter-credential,missing-client-constructor-parameter-kwargs
+        self,
+    ) -> None:
         self._thread_pool = ThreadPoolExecutor(thread_name_prefix="evaluators_thread")
 
-    def _calculate_metric(self, evaluator, input_df, column_mapping, evaluator_name):
+    def _calculate_metric(
+        self, evaluator: Callable, input_df: pd.DataFrame, column_mapping: Optional[Dict[str, str]], evaluator_name: str
+    ) -> pd.DataFrame:
         row_metric_futures = []
         row_metric_results = []
         input_df = _apply_column_mapping(input_df, column_mapping)
@@ -110,18 +119,25 @@ class CodeClient:
                 return aggregated_output
         except Exception as ex:  # pylint: disable=broad-exception-caught
             LOGGER.warning(
-                f"Error calculating aggregations for evaluator {run.evaluator_name}," f" failed with error {str(ex)}"
+                "Error calculating aggregations for evaluator %s, failed with error %s", run.evaluator_name, ex
             )
         return None
 
-    def run(self, flow, data, evaluator_name=None, column_mapping=None, **kwargs):
+    def run(
+        self,  # pylint: disable=unused-argument
+        flow: Callable,
+        data: Union[os.PathLike, Path, pd.DataFrame],
+        evaluator_name: Optional[str] = None,
+        column_mapping: Optional[Dict[str, str]] = None,
+        **kwargs,
+    ) -> CodeRun:
         input_df = data
         if not isinstance(input_df, pd.DataFrame):
             try:
                 json_data = load_jsonl(data)
             except json.JSONDecodeError as exc:
                 raise EvaluationException(
-                    message = f"Failed to parse data as JSON: {data}. Provide valid json lines data.",
+                    message=f"Failed to parse data as JSON: {data}. Provide valid json lines data.",
                     internal_message="Failed to parse data as JSON",
                     target=ErrorTarget.CODE_CLIENT,
                     category=ErrorCategory.INVALID_VALUE,
@@ -129,22 +145,28 @@ class CodeClient:
                 ) from exc
 
             input_df = pd.DataFrame(json_data)
-        eval_future = self._thread_pool.submit(self._calculate_metric, flow, input_df, column_mapping, evaluator_name)
+        eval_future = self._thread_pool.submit(
+            self._calculate_metric,
+            evaluator=flow,
+            input_df=input_df,
+            column_mapping=column_mapping,
+            evaluator_name=evaluator_name,
+        )
         run = CodeRun(run=eval_future, input_data=data, evaluator_name=evaluator_name, aggregated_metrics=None)
         aggregation_future = self._thread_pool.submit(self._calculate_aggregations, evaluator=flow, run=run)
         run.aggregated_metrics = aggregation_future
         return run
 
-    def get_details(self, run, all_results=False):
+    def get_details(self, run: CodeRun, all_results: bool = False) -> pd.DataFrame:
         result_df = run.get_result_df(exclude_inputs=not all_results)
         return result_df
 
-    def get_metrics(self, run):
+    def get_metrics(self, run: CodeRun) -> Optional[None]:
         try:
             aggregated_metrics = run.get_aggregated_metrics()
             print("Aggregated metrics")
             print(aggregated_metrics)
         except Exception as ex:  # pylint: disable=broad-exception-caught
-            LOGGER.debug(f"Error calculating metrics for evaluator {run.evaluator_name}, failed with error {str(ex)}")
+            LOGGER.debug("Error calculating metrics for evaluator %s, failed with error %s", run.evaluator_name, ex)
             return None
         return aggregated_metrics

azure/ai/evaluation/_evaluate/_batch_run_client/proxy_client.py

@@ -4,29 +4,40 @@
 import inspect
 import logging
 import os
+from concurrent.futures import Future
+from typing import Any, Callable, Dict, Optional, Union
 
 import numpy as np
-
+import pandas as pd
 from promptflow.client import PFClient
+from promptflow.entities import Run
 from promptflow.tracing import ThreadPoolExecutorWithContext as ThreadPoolExecutor
 
 LOGGER = logging.getLogger(__name__)
 
 
 class ProxyRun:
-    def __init__(self, run, **kwargs):
+    def __init__(self, run: Future, **kwargs) -> None:  # pylint: disable=unused-argument
         self.run = run
 
 
-class ProxyClient:
-    def __init__(self, pf_client: PFClient):
+class ProxyClient:  # pylint: disable=client-accepts-api-version-keyword
+    def __init__(  # pylint: disable=missing-client-constructor-parameter-credential,missing-client-constructor-parameter-kwargs
+        self, pf_client: PFClient
+    ) -> None:
         self._pf_client = pf_client
         self._thread_pool = ThreadPoolExecutor(thread_name_prefix="evaluators_thread")
 
-    def run(self, flow, data, column_mapping=None, **kwargs):
+    def run(
+        self,
+        flow: Union[str, os.PathLike, Callable],
+        data: Union[str, os.PathLike],
+        column_mapping: Optional[Dict[str, str]] = None,
+        **kwargs
+    ) -> ProxyRun:
         flow_to_run = flow
         if hasattr(flow, "_to_async"):
-            flow_to_run = flow._to_async()
+            flow_to_run = flow._to_async()  # pylint: disable=protected-access
 
         batch_use_async = self._should_batch_use_async(flow_to_run)
         eval_future = self._thread_pool.submit(
@@ -39,14 +50,14 @@ class ProxyClient:
         )
         return ProxyRun(run=eval_future)
 
-    def get_details(self, proxy_run, all_results=False):
-        run = proxy_run.run.result()
+    def get_details(self, proxy_run: ProxyRun, all_results: bool = False) -> pd.DataFrame:
+        run: Run = proxy_run.run.result()
         result_df = self._pf_client.get_details(run, all_results=all_results)
         result_df.replace("(Failed)", np.nan, inplace=True)
         return result_df
 
-    def get_metrics(self, proxy_run):
-        run = proxy_run.run.result()
+    def get_metrics(self, proxy_run: ProxyRun) -> Dict[str, Any]:
+        run: Run = proxy_run.run.result()
         return self._pf_client.get_metrics(run)
 
     @staticmethod
@@ -54,8 +65,7 @@ class ProxyClient:
         if os.getenv("PF_EVALS_BATCH_USE_ASYNC", "true").lower() == "true":
             if hasattr(flow, "__call__") and inspect.iscoroutinefunction(flow.__call__):
                 return True
-            elif inspect.iscoroutinefunction(flow):
+            if inspect.iscoroutinefunction(flow):
                 return True
-            else:
-                return False
+            return False
         return False

azure/ai/evaluation/_evaluate/_eval_run.py

@@ -8,17 +8,18 @@ import logging
 import os
 import posixpath
 import time
+import types
 import uuid
-from typing import Any, Dict, Optional, Set
+from typing import Any, Dict, Optional, Set, Type
 from urllib.parse import urlparse
 
-from azure.core.pipeline.policies import RetryPolicy
-from azure.core.rest import HttpResponse
-
 from promptflow._sdk.entities import Run
+
+from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException
 from azure.ai.evaluation._http_utils import get_http_client
 from azure.ai.evaluation._version import VERSION
-from azure.ai.evaluation._exceptions import EvaluationException, ErrorBlame, ErrorCategory, ErrorTarget
+from azure.core.pipeline.policies import RetryPolicy
+from azure.core.rest import HttpResponse
 
 LOGGER = logging.getLogger(__name__)
 
@@ -165,7 +166,9 @@ class EvalRun(contextlib.AbstractContextManager):  # pylint: disable=too-many-in
             self._url_base = urlparse(self._tracking_uri).netloc
             if self._promptflow_run is not None:
                 self.info = RunInfo(
-                    self._promptflow_run.name, self._promptflow_run._experiment_name, self._promptflow_run.name
+                    self._promptflow_run.name,
+                    self._promptflow_run._experiment_name,  # pylint: disable=protected-access
+                    self._promptflow_run.name,
                 )
             else:
                 url = f"https://{self._url_base}/mlflow/v2.0" f"{self._get_scope()}/api/2.0/mlflow/runs/create"
@@ -181,8 +184,10 @@ class EvalRun(contextlib.AbstractContextManager):  # pylint: disable=too-many-in
                 if response.status_code != 200:
                     self.info = RunInfo.generate(self._run_name)
                     LOGGER.warning(
-                        f"The run failed to start: {response.status_code}: {response.text()}."
-                        "The results will be saved locally, but will not be logged to Azure."
+                        "The run failed to start: %s: %s."
+                        "The results will be saved locally, but will not be logged to Azure.",
+                        response.status_code,
+                        response.text(),
                     )
                     self._status = RunStatus.BROKEN
                 else:
@@ -216,7 +221,7 @@ class EvalRun(contextlib.AbstractContextManager):  # pylint: disable=too-many-in
                 internal_message="Incorrect terminal status. Valid statuses are 'FINISHED', 'FAILED' and 'KILLED'",
                 target=ErrorTarget.EVAL_RUN,
                 category=ErrorCategory.FAILED_EXECUTION,
-                blame=ErrorBlame.UNKNOWN
+                blame=ErrorBlame.UNKNOWN,
             )
         url = f"https://{self._url_base}/mlflow/v2.0" f"{self._get_scope()}/api/2.0/mlflow/runs/update"
         body = {
@@ -239,8 +244,21 @@ class EvalRun(contextlib.AbstractContextManager):  # pylint: disable=too-many-in
         self._start_run()
         return self
 
-    def __exit__(self, exc_type, exc_value, exc_tb):
-        """The context manager exit call."""
+    def __exit__(
+        self,
+        exc_type: Optional[Type[BaseException]],
+        exc_value: Optional[BaseException],
+        exc_tb: Optional[types.TracebackType],
+    ) -> Optional[bool]:
+        """The context manager exit call.
+
+        :param exc_type: The exception type
+        :type exc_type: Optional[Type[BaseException]]
+        :param exc_value: The exception value
+        :type exc_value: Optional[BaseException]
+        :param exc_tb: The exception traceback
+        :type exc_tb: Optional[types.TracebackType]
+        """
         self._end_run("FINISHED")
 
     def get_run_history_uri(self) -> str:
@@ -280,7 +298,7 @@ class EvalRun(contextlib.AbstractContextManager):  # pylint: disable=too-many-in
         # is an optional dependency.
         from promptflow.azure._utils._token_cache import ArmTokenCache  # pylint: disable=import-error,no-name-in-module
 
-        return ArmTokenCache().get_token(self._ml_client._credential)
+        return ArmTokenCache().get_token(self._ml_client._credential)  # pylint: disable=protected-access
 
     def request_with_retry(
         self, url: str, method: str, json_dict: Dict[str, Any], headers: Optional[Dict[str, str]] = None
@@ -326,9 +344,10 @@ class EvalRun(contextlib.AbstractContextManager):  # pylint: disable=too-many-in
         :type response: HttpResponse
         """
         LOGGER.warning(
-            f"Unable to {failed_op}, "
-            f"the request failed with status code {response.status_code}, "
-            f"{response.text()=}."
+            "Unable to %s, the request failed with status code %s, response.text()=%s.",
+            failed_op,
+            response.status_code,
+            response.text(),
         )
 
     def _check_state_and_log(self, action: str, bad_states: Set[RunStatus], should_raise: bool) -> bool:
@@ -342,7 +361,8 @@ class EvalRun(contextlib.AbstractContextManager):  # pylint: disable=too-many-in
         :type bad_states: Set[RunStatus]
         :param should_raise: Should we raise an error if the bad state has been encountered
         :type should_raise: bool
-        :raises: ~azure.ai.evaluations._exceptions.EvaluationException if should_raise is True and invalid state was encountered.
+        :raises: ~azure.ai.evaluations._exceptions.EvaluationException if should_raise is True
+             and invalid state was encountered.
         :return: Whether or not run is in the correct state.
         :rtype: bool
         """
@@ -354,7 +374,7 @@ class EvalRun(contextlib.AbstractContextManager):  # pylint: disable=too-many-in
                     internal_message=msg,
                     target=ErrorTarget.EVAL_RUN,
                     category=ErrorCategory.FAILED_EXECUTION,
-                    blame=ErrorBlame.UNKNOWN
+                    blame=ErrorBlame.UNKNOWN,
                 )
             LOGGER.warning(msg)
             return False
@@ -446,7 +466,7 @@ class EvalRun(contextlib.AbstractContextManager):  # pylint: disable=too-many-in
             return credential.account_key
         if hasattr(credential, "sas_token"):
             return credential.sas_token
-        return self._ml_client.datastores._credential
+        return self._ml_client.datastores._credential  # pylint: disable=protected-access
 
     def log_metric(self, key: str, value: float) -> None:
         """