azure-ai-evaluation 1.4.0__py3-none-any.whl → 1.5.0__py3-none-any.whl

azure/ai/evaluation/_evaluate/_telemetry/__init__.py

@@ -9,11 +9,10 @@ import logging
 from typing import Callable, Dict, Literal, Optional, Union, cast
 
 import pandas as pd
-from promptflow._sdk.entities._flows import FlexFlow as flex_flow
-from promptflow._sdk.entities._flows import Prompty as prompty_sdk
-from promptflow._sdk.entities._flows.dag import Flow as dag_flow
-from promptflow.client import PFClient
-from promptflow.core import Prompty as prompty_core
+from azure.ai.evaluation._legacy._adapters._flows import FlexFlow as flex_flow
+from azure.ai.evaluation._legacy._adapters._flows import AsyncPrompty as prompty_sdk
+from azure.ai.evaluation._legacy._adapters._flows import Flow as dag_flow
+from azure.ai.evaluation._legacy._adapters.client import PFClient
 from typing_extensions import ParamSpec
 
 from azure.ai.evaluation._model_configurations import AzureAIProject, EvaluationResult
@@ -66,7 +65,7 @@ def _get_evaluator_properties(evaluator, evaluator_name):
 
     try:
         # Cover flex flow and prompty based evaluator
-        if isinstance(evaluator, (prompty_sdk, prompty_core, flex_flow)):
+        if isinstance(evaluator, (prompty_sdk, flex_flow)):
             name = evaluator.name
             pf_type = evaluator.__class__.__name__
         # Cover dag flow based evaluator
@@ -94,86 +93,3 @@ def _get_evaluator_properties(evaluator, evaluator_name):
         "type": _get_evaluator_type(evaluator),
         "alias": evaluator_name if evaluator_name else "",
     }
-
-
-# cspell:ignore isna
-def log_evaluate_activity(func: Callable[P, EvaluationResult]) -> Callable[P, EvaluationResult]:
-    """Decorator to log evaluate activity
-
-    :param func: The function to be decorated
-    :type func: Callable
-    :returns: The decorated function
-    :rtype: Callable[P, EvaluationResult]
-    """
-
-    @functools.wraps(func)
-    def wrapper(*args: P.args, **kwargs: P.kwargs) -> EvaluationResult:
-        from promptflow._sdk._telemetry import ActivityType, log_activity
-        from promptflow._sdk._telemetry.telemetry import get_telemetry_logger
-
-        evaluators = cast(Optional[Dict[str, Callable]], kwargs.get("evaluators", {})) or {}
-        azure_ai_project = cast(Optional[AzureAIProject], kwargs.get("azure_ai_project", None))
-
-        pf_client = PFClient(
-            config=(
-                {"trace.destination": _trace_destination_from_project_scope(azure_ai_project)}
-                if azure_ai_project
-                else None
-            ),
-            user_agent=USER_AGENT,
-        )
-
-        trace_destination = pf_client._config.get_trace_destination()  # pylint: disable=protected-access
-        track_in_cloud = bool(trace_destination) if trace_destination != "none" else False
-        evaluate_target = bool(kwargs.get("target", None))
-        evaluator_config = bool(kwargs.get("evaluator_config", None))
-        custom_dimensions: Dict[str, Union[str, bool]] = {
-            "track_in_cloud": track_in_cloud,
-            "evaluate_target": evaluate_target,
-            "evaluator_config": evaluator_config,
-        }
-
-        with log_activity(
-            get_telemetry_logger(),
-            "pf.evals.evaluate",
-            activity_type=ActivityType.PUBLICAPI,
-            user_agent=USER_AGENT,
-            custom_dimensions=custom_dimensions,
-        ):
-            result = func(*args, **kwargs)
-
-            try:
-                evaluators_info = []
-                for evaluator_name, evaluator in evaluators.items():
-                    evaluator_info = _get_evaluator_properties(evaluator, evaluator_name)
-                    try:
-                        evaluator_df = pd.DataFrame(result.get("rows", [])).filter(
-                            like=f"outputs.{evaluator_name}", axis=1
-                        )
-
-                        failed_rows = (
-                            evaluator_df.shape[0] if evaluator_df.empty else int(evaluator_df.isna().any(axis=1).sum())
-                        )
-                        total_rows = evaluator_df.shape[0]
-
-                        evaluator_info["failed_rows"] = failed_rows
-                        evaluator_info["total_rows"] = total_rows
-                    except Exception as e:  # pylint: disable=broad-exception-caught
-                        LOGGER.debug("Failed to collect evaluate failed row info for %s: %s", evaluator_name, e)
-                    evaluators_info.append(evaluator_info)
-
-                custom_dimensions = {"evaluators_info": json.dumps(evaluators_info)}
-                with log_activity(
-                    get_telemetry_logger(),
-                    "pf.evals.evaluate_usage_info",
-                    activity_type=ActivityType.PUBLICAPI,
-                    user_agent=USER_AGENT,
-                    custom_dimensions=custom_dimensions,
-                ):
-                    pass
-            except Exception as e:  # pylint: disable=broad-exception-caught
-                LOGGER.debug("Failed to collect evaluate usage info: %s", e)
-
-            return result
-
-    return wrapper

azure/ai/evaluation/_evaluate/_utils.py

@@ -12,7 +12,7 @@ import uuid
 import base64
 
 import pandas as pd
-from promptflow.entities import Run
+from azure.ai.evaluation._legacy._adapters.entities import Run
 
 from azure.ai.evaluation._constants import (
     DEFAULT_EVALUATION_RESULTS_FILE_NAME,
@@ -46,7 +46,7 @@ def is_none(value) -> bool:
 def extract_workspace_triad_from_trace_provider(  # pylint: disable=name-too-long
     trace_provider: str,
 ) -> AzureMLWorkspace:
-    from promptflow._cli._utils import get_workspace_triad_from_local
+    from azure.ai.evaluation._legacy._adapters.utils import get_workspace_triad_from_local
 
     match = re.match(AZURE_WORKSPACE_REGEX_FORMAT, trace_provider)
     if not match or len(match.groups()) != 5:
@@ -131,7 +131,7 @@ def _log_metrics_and_instance_results(
     metrics: Dict[str, Any],
     instance_results: pd.DataFrame,
     trace_destination: Optional[str],
-    run: Run,
+    run: Optional[Run],
     evaluation_name: Optional[str],
     **kwargs,
 ) -> Optional[str]:

azure/ai/evaluation/_evaluators/_common/_base_eval.py

@@ -6,7 +6,7 @@ import inspect
 from abc import ABC, abstractmethod
 from typing import Any, Callable, Dict, Generic, List, TypedDict, TypeVar, Union, cast, final, Optional
 
-from promptflow._utils.async_utils import async_run_allowing_running_loop
+from azure.ai.evaluation._legacy._adapters.utils import async_run_allowing_running_loop
 from typing_extensions import ParamSpec, TypeAlias, get_overloads
 
 from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException

azure/ai/evaluation/_evaluators/_common/_base_multi_eval.py

@@ -4,7 +4,7 @@
 from concurrent.futures import as_completed
 from typing import TypeVar, Dict, List
 
-from promptflow.tracing import ThreadPoolExecutorWithContext as ThreadPoolExecutor
+from azure.ai.evaluation._legacy._adapters.tracing import ThreadPoolExecutorWithContext as ThreadPoolExecutor
 from typing_extensions import override
 
 from azure.ai.evaluation._evaluators._common import EvaluatorBase

azure/ai/evaluation/_evaluators/_common/_base_prompty_eval.py

@@ -6,7 +6,7 @@ import math
 import re
 from typing import Dict, TypeVar, Union
 
-from promptflow.core import AsyncPrompty
+from azure.ai.evaluation._legacy.prompty import AsyncPrompty
 from typing_extensions import override
 
 from azure.ai.evaluation._common.constants import PROMPT_BASED_REASON_EVALUATORS

azure/ai/evaluation/_evaluators/_groundedness/_groundedness.py

@@ -5,7 +5,7 @@ import os
 from typing import Dict, List, Optional, Union
 
 from typing_extensions import overload, override
-from promptflow.core import AsyncPrompty
+from azure.ai.evaluation._legacy._adapters._flows import AsyncPrompty
 
 from azure.ai.evaluation._evaluators._common import PromptyEvaluatorBase
 from azure.ai.evaluation._model_configurations import Conversation

azure/ai/evaluation/_evaluators/_response_completeness/_response_completeness.py

@@ -15,6 +15,7 @@ from azure.ai.evaluation._model_configurations import Conversation, Message
 from azure.ai.evaluation._common._experimental import experimental
 
 
+@experimental
 class ResponseCompletenessEvaluator(PromptyEvaluatorBase[Union[str, float]]):
     """
     Evaluates the extent to which a given response contains all necessary and relevant information with respect to the

azure/ai/evaluation/_legacy/_adapters/__init__.py

@@ -0,0 +1,21 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+# NOTE: This contains adapters that make the Promptflow dependency optional. In the first phase,
+#       Promptflow will still be installed as part of the azure-ai-evaluation dependencies. This
+#       will be removed in the future once the code migration is complete.
+
+from typing import Final
+
+
+_has_legacy = False
+try:
+    from promptflow.client import PFClient
+
+    _has_legacy = True
+except ImportError:
+    pass
+
+HAS_LEGACY_SDK: Final[bool] = _has_legacy
+MISSING_LEGACY_SDK: Final[bool] = not _has_legacy

azure/ai/evaluation/_legacy/_adapters/_configuration.py

@@ -0,0 +1,45 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+from pathlib import Path
+from typing import Any, Dict, Final, Optional
+from typing_extensions import TypeAlias
+
+
+try:
+    from promptflow._sdk._configuration import Configuration as _Configuration
+except ImportError:
+    _global_config: Final[Dict[str, Any]] = {}
+
+    class _Configuration:
+        TRACE_DESTINATION: Final[str] = "trace.destination"
+        _instance = None
+
+        def __init__(self, *, override_config: Optional[Dict[str, Any]] = None) -> None:
+            self._config = override_config or {}
+
+        @classmethod
+        def get_instance(cls) -> "_Configuration":
+            """Use this to get instance to avoid multiple copies of same global config."""
+            if cls._instance is None:
+                cls._instance = Configuration(override_config=_global_config)
+            return cls._instance
+
+        def set_config(self, key: str, value: Any) -> None:
+            # Simulated config storage
+            self._config[key] = value
+
+        def get_config(self, key: str) -> Any:
+            # Simulated config storage
+            if key in self._config:
+                return self._config[key]
+            return _global_config.get(key, None)
+
+        def get_trace_destination(self, path: Optional[Path] = None) -> Optional[str]:
+            if path:
+                raise NotImplementedError("Setting trace destination with a path is not supported.")
+            return self._config.get("trace.destination", None)
+
+
+Configuration: TypeAlias = _Configuration

azure/ai/evaluation/_legacy/_adapters/_constants.py

@@ -0,0 +1,10 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+from typing import Final
+
+
+PF_FLOW_ENTRY_IN_TMP: Final[str] = "PF_FLOW_ENTRY_IN_TMP"
+PF_FLOW_META_LOAD_IN_SUBPROCESS: Final[str] = "PF_FLOW_META_LOAD_IN_SUBPROCESS"
+LINE_NUMBER: Final[str] = "line_number"

azure/ai/evaluation/_legacy/_adapters/_errors.py

@@ -0,0 +1,29 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+from typing import Any
+from typing_extensions import TypeAlias
+
+
+try:
+    from promptflow.core._errors import MissingRequiredPackage as _MissingRequiredPackage
+except ImportError:
+    from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException
+
+    class _MissingRequiredPackage(EvaluationException):
+        """Raised when a required package is missing.
+
+        :param message: A message describing the error. This is the error message the user will see.
+        :type message: str
+        """
+
+        def __init__(self, message: str, **kwargs: Any):
+            kwargs.setdefault("category", ErrorCategory.MISSING_PACKAGE)
+            kwargs.setdefault("blame", ErrorBlame.SYSTEM_ERROR)
+            kwargs.setdefault("target", ErrorTarget.EVALUATE)
+            kwargs.setdefault("internal_message", "Missing required package.")
+            super().__init__(message=message, **kwargs)
+
+
+MissingRequiredPackage: TypeAlias = _MissingRequiredPackage

azure/ai/evaluation/_legacy/_adapters/_flows.py

@@ -0,0 +1,28 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+from typing_extensions import TypeAlias
+
+
+try:
+    from promptflow._sdk.entities._flows import AsyncPrompty as _AsyncPrompty
+    from promptflow._sdk.entities._flows import FlexFlow as _FlexFlow
+    from promptflow._sdk.entities._flows.dag import Flow as _Flow
+except ImportError:
+    from azure.ai.evaluation._legacy.prompty import AsyncPrompty as _AsyncPrompty
+
+    class _FlexFlow:
+        pass
+
+    _FlexFlow.__name__ = "FlexFlow"
+
+    class _Flow:
+        name: str
+
+    _Flow.__name__ = "Flow"
+
+
+AsyncPrompty: TypeAlias = _AsyncPrompty
+FlexFlow: TypeAlias = _FlexFlow
+Flow: TypeAlias = _Flow

azure/ai/evaluation/_legacy/_adapters/_service.py

@@ -0,0 +1,16 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+from typing import Callable, Final
+
+
+try:
+    from promptflow._cli._pf._service import stop_service as _stop_service
+except ImportError:
+
+    def _stop_service() -> None:
+        pass
+
+
+stop_service: Final[Callable[[], None]] = _stop_service

azure/ai/evaluation/_legacy/_adapters/client.py

@@ -0,0 +1,51 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+from os import PathLike
+from typing import Any, Callable, Dict, Optional, Union
+from typing_extensions import TypeAlias
+
+import pandas as pd
+
+from ._errors import MissingRequiredPackage
+from ._configuration import Configuration
+from .entities import Run
+
+
+try:
+    from promptflow.client import PFClient as _PFClient
+except ImportError:
+
+    class _PFClient:
+        def __init__(self, **kwargs):
+            self._config = Configuration(override_config=kwargs.pop("config", None))
+
+        def run(
+            self,
+            flow: Union[str, PathLike, Callable],
+            *,
+            data: Union[str, PathLike],
+            run: Optional[Union[str, Run]] = None,
+            column_mapping: Optional[dict] = None,
+            variant: Optional[str] = None,
+            connections: Optional[dict] = None,
+            environment_variables: Optional[dict] = None,
+            name: Optional[str] = None,
+            display_name: Optional[str] = None,
+            tags: Optional[Dict[str, str]] = None,
+            resume_from: Optional[Union[str, Run]] = None,
+            code: Optional[Union[str, PathLike]] = None,
+            init: Optional[dict] = None,
+            **kwargs,
+        ) -> Run:
+            raise MissingRequiredPackage("Please install 'promptflow' package to use PFClient")
+
+        def get_details(self, run: Union[str, Run], max_results: int = 100, all_results: bool = False) -> pd.DataFrame:
+            return pd.DataFrame()
+
+        def get_metrics(self, run: Union[str, Run]) -> Dict[str, Any]:
+            return {}
+
+
+PFClient: TypeAlias = _PFClient

azure/ai/evaluation/_legacy/_adapters/entities.py

@@ -0,0 +1,26 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+from typing_extensions import TypeAlias
+
+
+try:
+    from promptflow._sdk.entities import Run as _Run
+except ImportError:
+    from typing_extensions import Protocol
+    from typing import Any, Dict, Optional
+    from datetime import datetime
+    from pathlib import Path
+
+    class _Run(Protocol):
+        name: str
+        status: str
+        _properties: Dict[str, Any]
+        _created_on: datetime
+        _end_time: Optional[datetime]
+        _experiment_name: Optional[str]
+        _output_path: Path
+
+
+Run: TypeAlias = _Run

azure/ai/evaluation/_legacy/_adapters/tracing.py

@@ -0,0 +1,28 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+from typing import Callable, Dict, Final, Optional
+from typing_extensions import TypeAlias
+
+
+try:
+    from promptflow.tracing import ThreadPoolExecutorWithContext as _ThreadPoolExecutorWithContext
+    from promptflow.tracing._integrations._openai_injector import (
+        inject_openai_api as _inject,
+        recover_openai_api as _recover,
+    )
+    from promptflow.tracing import _start_trace
+except ImportError:
+    from concurrent.futures import ThreadPoolExecutor as _ThreadPoolExecutorWithContext
+    from azure.ai.evaluation._legacy._batch_engine._openai_injector import (
+        inject_openai_api as _inject,
+        recover_openai_api as _recover,
+    )
+    from azure.ai.evaluation._legacy._batch_engine._trace import start_trace as _start_trace
+
+
+ThreadPoolExecutorWithContext: TypeAlias = _ThreadPoolExecutorWithContext
+inject_openai_api: Final[Callable[[], None]] = _inject
+recover_openai_api: Final[Callable[[], None]] = _recover
+start_trace: Final = _start_trace

azure/ai/evaluation/_legacy/_adapters/types.py

@@ -0,0 +1,15 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+from typing import Any
+
+
+class AttrDict(dict):
+    """A dictionary that allows attribute access to its keys."""
+
+    def __getattr__(self, key: str) -> Any:
+        return self[key]
+
+    def __setattr__(self, key: str, value: Any) -> None:
+        self[key] = value

azure/ai/evaluation/_legacy/_adapters/utils.py

@@ -0,0 +1,31 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+from typing import Final, Optional
+from typing_extensions import TypeAlias
+
+
+try:
+    from promptflow._utils.user_agent_utils import ClientUserAgentUtil as _ClientUserAgentUtil
+    from promptflow._utils.async_utils import async_run_allowing_running_loop as _async_run_allowing_running_loop
+    from promptflow._cli._utils import get_workspace_triad_from_local as _get_workspace_triad_from_local
+except ImportError:
+    from azure.ai.evaluation._legacy._batch_engine._utils_deprecated import (
+        async_run_allowing_running_loop as _async_run_allowing_running_loop,
+    )
+    from azure.ai.evaluation._evaluate._utils import AzureMLWorkspace
+
+    class _ClientUserAgentUtil:
+        @staticmethod
+        def append_user_agent(user_agent: Optional[str]):
+            # TODO ralphe: implement?
+            pass
+
+    def _get_workspace_triad_from_local() -> AzureMLWorkspace:
+        return AzureMLWorkspace("", "", "")
+
+
+ClientUserAgentUtil: TypeAlias = _ClientUserAgentUtil
+async_run_allowing_running_loop: Final = _async_run_allowing_running_loop
+get_workspace_triad_from_local: Final = _get_workspace_triad_from_local

azure/ai/evaluation/_legacy/_batch_engine/_result.py

@@ -96,4 +96,4 @@ class BatchResult:
         """The results of the batch run."""
         if not self.details:
             return []
-        return [d.result for d in self.details]
+        return [d.result for d in self.details]

azure/ai/evaluation/_legacy/_batch_engine/_status.py

@@ -22,4 +22,4 @@ class BatchStatus(IntEnum):
 
     @staticmethod
     def is_failed(status: "BatchStatus") -> bool:
-        return status == BatchStatus.Failed or status == BatchStatus.Canceled
+        return status == BatchStatus.Failed or status == BatchStatus.Canceled

azure/ai/evaluation/_version.py

@@ -3,4 +3,4 @@
 # ---------------------------------------------------------
 # represents upcoming version
 
-VERSION = "1.4.0"
+VERSION = "1.5.0"

azure/ai/evaluation/red_team/__init__.py

@@ -0,0 +1,19 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+try:
+    from ._red_team import RedTeam
+    from ._attack_strategy import AttackStrategy
+    from ._attack_objective_generator import RiskCategory
+    from ._red_team_result import RedTeamResult
+except ImportError:
+    print("[INFO] Could not import Pyrit. Please install the dependency with `pip install azure-ai-evaluation[redteam]`.")
+
+
+__all__ = [
+    "RedTeam",
+    "AttackStrategy",
+    "RiskCategory",
+    "RedTeamResult",
+]

azure/ai/evaluation/{_red_team → red_team}/_attack_objective_generator.py

@@ -7,7 +7,10 @@ import os
 from enum import Enum
 from typing import List, Optional, Dict, Any
 from pathlib import Path
+from azure.ai.evaluation._common._experimental import experimental
 
+
+@experimental
 class RiskCategory(str, Enum):
     """Risk categories for attack objectives."""
     HateUnfairness = "hate_unfairness"

azure/ai/evaluation/{_red_team → red_team}/_attack_strategy.py

@@ -3,7 +3,10 @@
 # ---------------------------------------------------------
 from enum import Enum
 from typing import List
+from azure.ai.evaluation._common._experimental import experimental
 
+
+@experimental
 class AttackStrategy(Enum):
     """Strategies for attacks."""
     EASY = "easy"