azure-ai-evaluation 1.0.1__py3-none-any.whl → 1.13.5__py3-none-any.whl

azure/ai/evaluation/_evaluators/_ungrounded_attributes/_ungrounded_attributes.py

@@ -0,0 +1,102 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+from typing_extensions import overload, override
+from typing import Dict, Union
+
+from azure.ai.evaluation._common._experimental import experimental
+from azure.ai.evaluation._common.constants import EvaluationMetrics
+from azure.ai.evaluation._evaluators._common import RaiServiceEvaluatorBase
+
+
+@experimental
+class UngroundedAttributesEvaluator(RaiServiceEvaluatorBase[Union[str, bool]]):
+    """
+    Evaluates ungrounded inference of human attributes for a given query, response, and context for a single-turn evaluation only,
+    where query represents the user query and response represents the AI system response given the provided context.
+
+    Ungrounded Attributes checks for whether a response is first, ungrounded, and checks if it contains information about protected class or
+    emotional state of a person.
+
+
+    It identifies the following attributes:
+
+    - emotional_state
+    - protected_class
+    - groundedness
+
+    :param credential: The credential for connecting to Azure AI project. Required
+    :type credential: ~azure.core.credentials.TokenCredential
+    :param azure_ai_project: The Azure AI project, which can either be a string representing the project endpoint
+        or an instance of AzureAIProject. It contains subscription id, resource group, and project name.
+    :type azure_ai_project: Union[str, ~azure.ai.evaluation.AzureAIProject]
+    :param kwargs: Additional arguments to pass to the evaluator.
+    :type kwargs: Any
+
+    .. note::
+
+        If this evaluator is supplied to the `evaluate` function, the metric
+        for the ungrounded attributes will be "ungrounded_attributes_label".
+    """
+
+    id = "azureai://built-in/evaluators/ungrounded_attributes"
+    """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
+    _OPTIONAL_PARAMS = ["query"]
+
+    @override
+    def __init__(
+        self,
+        credential,
+        azure_ai_project,
+        **kwargs,
+    ):
+        # Set default for evaluate_query if not provided
+        if "evaluate_query" not in kwargs:
+            kwargs["evaluate_query"] = True
+
+        super().__init__(
+            eval_metric=EvaluationMetrics.UNGROUNDED_ATTRIBUTES,
+            azure_ai_project=azure_ai_project,
+            credential=credential,
+            **kwargs,
+        )
+
+    @overload
+    def __call__(
+        self,
+        *,
+        query: str,
+        response: str,
+        context: str,
+    ) -> Dict[str, Union[str, float]]:
+        """Evaluate a given query/response pair and context for ungrounded attributes
+
+        :keyword query: The query to be evaluated.
+        :paramtype query: str
+        :keyword response: The response to be evaluated.
+        :paramtype response: str
+        :keyword context: The context to be used for evaluation.
+        :paramtype context: str
+        :return: The ungrounded attributes label.
+        :rtype: Dict[str, Union[str, bool]]
+        """
+
+    @override
+    def __call__(  # pylint: disable=docstring-missing-param
+        self,
+        *args,
+        **kwargs,
+    ):
+        """Evaluate a given query/response pair and context for ungrounded attributes
+
+        :keyword query: The query to be evaluated.
+        :paramtype query: str
+        :keyword response: The response to be evaluated.
+        :paramtype response: str
+        :keyword context: The context to be used for evaluation.
+        :paramtype context: str
+        :return: The ungrounded attributes label.
+        :rtype: Dict[str, Union[str, bool]]
+        """
+
+        return super().__call__(*args, **kwargs)

azure/ai/evaluation/_evaluators/_xpia/xpia.py

@@ -40,9 +40,11 @@ class IndirectAttackEvaluator(RaiServiceEvaluatorBase[Union[str, bool]]):
 
     :param credential: The credential for connecting to Azure AI project. Required
     :type credential: ~azure.core.credentials.TokenCredential
-    :param azure_ai_project: The scope of the Azure AI project. It contains subscription id, resource group, and project
-        name.
-    :type azure_ai_project: ~azure.ai.evaluation.AzureAIProject
+    :param azure_ai_project: The Azure AI project, which can either be a string representing the project endpoint
+        or an instance of AzureAIProject. It contains subscription id, resource group, and project name.
+    :type azure_ai_project: Union[str, ~azure.ai.evaluation.AzureAIProject]
+    :param threshold: The threshold for the IndirectAttack evaluator. Default is 0.
+    :type threshold: int
 
     .. admonition:: Example:
 
@@ -52,21 +54,35 @@ class IndirectAttackEvaluator(RaiServiceEvaluatorBase[Union[str, bool]]):
             :language: python
             :dedent: 8
             :caption: Initialize and call an IndirectAttackEvaluator.
+
+    .. admonition:: Example using Azure AI Project URL:
+
+        .. literalinclude:: ../samples/evaluation_samples_evaluate_fdp.py
+            :start-after: [START indirect_attack_evaluator]
+            :end-before: [END indirect_attack_evaluator]
+            :language: python
+            :dedent: 8
+            :caption: Initialize and call IndirectAttackEvaluator using Azure AI Project URL in the following format
+                https://{resource_name}.services.ai.azure.com/api/projects/{project_name}
+
     """
 
-    id = "azureml://registries/azureml/models/Indirect-Attack-Evaluator/versions/3"
+    id = "azureai://built-in/evaluators/indirect_attack"
     """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
+    _OPTIONAL_PARAMS = ["query"]
 
     @override
     def __init__(
         self,
         credential,
         azure_ai_project,
+        **kwargs,
     ):
         super().__init__(
             eval_metric=EvaluationMetrics.XPIA,
             azure_ai_project=azure_ai_project,
             credential=credential,
+            **kwargs,
         )
 
     @overload

azure/ai/evaluation/_exceptions.py

@@ -9,6 +9,15 @@ from typing import Optional
 from azure.core.exceptions import AzureError
 
 
+class ErrorMessage(Enum):
+    """Error messages to be used when raising EvaluationException.
+
+    These messages are used to provide a consistent error message format across the SDK.
+    """
+
+    MALFORMED_CONVERSATION_HISTORY = "Malformed Conversation History: Query parameter representing conversation history should have exactly one more user query than agent responses"
+
+
 class ErrorCategory(Enum):
     """Error category to be specified when using EvaluationException class.
 
@@ -38,6 +47,8 @@ class ErrorCategory(Enum):
     FAILED_REMOTE_TRACKING = "FAILED REMOTE TRACKING"
     PROJECT_ACCESS_ERROR = "PROJECT ACCESS ERROR"
     UNKNOWN = "UNKNOWN"
+    UPLOAD_ERROR = "UPLOAD ERROR"
+    NOT_APPLICABLE = "NOT APPLICABLE"
 
 
 class ErrorBlame(Enum):
@@ -62,16 +73,21 @@ class ErrorTarget(Enum):
     CODE_CLIENT = "CodeClient"
     RAI_CLIENT = "RAIClient"
     COHERENCE_EVALUATOR = "CoherenceEvaluator"
+    COMPLETENESS_EVALUATOR = "CompletenessEvaluator"
     CONTENT_SAFETY_CHAT_EVALUATOR = "ContentSafetyEvaluator"
-    CONTENT_SAFETY_MULTIMODAL_EVALUATOR = "ContentSafetyMultimodalEvaluator"
     ECI_EVALUATOR = "ECIEvaluator"
     F1_EVALUATOR = "F1Evaluator"
     GROUNDEDNESS_EVALUATOR = "GroundednessEvaluator"
+    TASK_NAVIGATION_EFFICIENCY_EVALUATOR = "_TaskNavigationEfficiencyEvaluator"
     PROTECTED_MATERIAL_EVALUATOR = "ProtectedMaterialEvaluator"
+    INTENT_RESOLUTION_EVALUATOR = "IntentResolutionEvaluator"
     RELEVANCE_EVALUATOR = "RelevanceEvaluator"
     SIMILARITY_EVALUATOR = "SimilarityEvaluator"
     FLUENCY_EVALUATOR = "FluencyEvaluator"
     RETRIEVAL_EVALUATOR = "RetrievalEvaluator"
+    TOOL_CALL_SUCCESS_EVALUATOR = "_ToolCallSuccessEvaluator"
+    TASK_ADHERENCE_EVALUATOR = "TaskAdherenceEvaluator"
+    TASK_COMPLETION_EVALUATOR = "_TaskCompletionEvaluator"
     INDIRECT_ATTACK_EVALUATOR = "IndirectAttackEvaluator"
     INDIRECT_ATTACK_SIMULATOR = "IndirectAttackSimulator"
     ADVERSARIAL_SIMULATOR = "AdversarialSimulator"
@@ -81,6 +97,13 @@ class ErrorTarget(Enum):
     MODELS = "Models"
     UNKNOWN = "Unknown"
     CONVERSATION = "Conversation"
+    TOOL_CALL_ACCURACY_EVALUATOR = "ToolCallAccuracyEvaluator"
+    TOOL_SELECTION_EVALUATOR = "_ToolSelectionEvaluator"
+    TOOL_INPUT_ACCURACY_EVALUATOR = "_ToolInputAccuracyEvaluator"
+    RED_TEAM = "RedTeam"
+    AOAI_GRADER = "AoaiGrader"
+    CONVERSATION_HISTORY_PARSING = "_get_conversation_history"
+    TOOL_OUTPUT_UTILIZATION_EVALUATOR = "ToolOutputUtilizationEvaluator"
 
 
 class EvaluationException(AzureError):

azure/ai/evaluation/_http_utils.py

@@ -7,7 +7,7 @@ from typing import Any, Dict, MutableMapping, Optional, TypedDict, cast
 
 from typing_extensions import Self, Unpack
 
-from azure.ai.evaluation._user_agent import USER_AGENT
+from azure.ai.evaluation._user_agent import UserAgentSingleton
 from azure.core.configuration import Configuration
 from azure.core.pipeline import AsyncPipeline, Pipeline
 from azure.core.pipeline.policies import (
@@ -448,19 +448,21 @@ class AsyncHttpPipeline(AsyncPipeline):
         return cast(Self, await super().__aenter__())
 
 
-def get_http_client() -> HttpPipeline:
+def get_http_client(**kwargs: Any) -> HttpPipeline:
     """Get an HttpPipeline configured with common policies.
 
     :returns: An HttpPipeline with a set of applied policies:
     :rtype: HttpPipeline
     """
-    return HttpPipeline(user_agent_policy=UserAgentPolicy(base_user_agent=USER_AGENT))
+    kwargs.setdefault("user_agent_policy", UserAgentPolicy(base_user_agent=UserAgentSingleton().value))
+    return HttpPipeline(**kwargs)
 
 
-def get_async_http_client() -> AsyncHttpPipeline:
+def get_async_http_client(**kwargs: Any) -> AsyncHttpPipeline:
     """Get an AsyncHttpPipeline configured with common policies.
 
     :returns: An AsyncHttpPipeline with a set of applied policies:
     :rtype: AsyncHttpPipeline
     """
-    return AsyncHttpPipeline(user_agent_policy=UserAgentPolicy(base_user_agent=USER_AGENT))
+    kwargs.setdefault("user_agent_policy", UserAgentPolicy(base_user_agent=UserAgentSingleton().value))
+    return AsyncHttpPipeline(**kwargs)

azure/ai/evaluation/_legacy/__init__.py

@@ -0,0 +1,3 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------

azure/ai/evaluation/_legacy/_adapters/__init__.py

@@ -0,0 +1,7 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+# NOTE: This contains adapters that make the Promptflow dependency optional. In the first phase,
+#       Promptflow will still be installed as part of the azure-ai-evaluation dependencies. This
+#       will be removed in the future once the code migration is complete.

azure/ai/evaluation/_legacy/_adapters/_check.py

@@ -0,0 +1,17 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+from typing import Final
+
+
+_has_legacy = False
+try:
+    from promptflow._constants import FlowType
+
+    _has_legacy = True
+except ImportError:
+    pass
+
+HAS_LEGACY_SDK: Final[bool] = _has_legacy
+MISSING_LEGACY_SDK: Final[bool] = not _has_legacy

azure/ai/evaluation/_legacy/_adapters/_configuration.py

@@ -0,0 +1,45 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+from pathlib import Path
+from typing import Any, Dict, Final, Optional
+from typing_extensions import TypeAlias
+
+
+try:
+    from promptflow._sdk._configuration import Configuration as _Configuration
+except ImportError:
+    _global_config: Final[Dict[str, Any]] = {}
+
+    class _Configuration:
+        TRACE_DESTINATION: Final[str] = "trace.destination"
+        _instance = None
+
+        def __init__(self, *, override_config: Optional[Dict[str, Any]] = None) -> None:
+            self._config = override_config or {}
+
+        @classmethod
+        def get_instance(cls) -> "_Configuration":
+            """Use this to get instance to avoid multiple copies of same global config."""
+            if cls._instance is None:
+                cls._instance = Configuration(override_config=_global_config)
+            return cls._instance
+
+        def set_config(self, key: str, value: Any) -> None:
+            # Simulated config storage
+            self._config[key] = value
+
+        def get_config(self, key: str) -> Any:
+            # Simulated config storage
+            if key in self._config:
+                return self._config[key]
+            return _global_config.get(key, None)
+
+        def get_trace_destination(self, path: Optional[Path] = None) -> Optional[str]:
+            if path:
+                raise NotImplementedError("Setting trace destination with a path is not supported.")
+            return self._config.get("trace.destination", None)
+
+
+Configuration: TypeAlias = _Configuration

azure/ai/evaluation/_legacy/_adapters/_constants.py

@@ -0,0 +1,10 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+from typing import Final
+
+
+PF_FLOW_ENTRY_IN_TMP: Final[str] = "PF_FLOW_ENTRY_IN_TMP"
+PF_FLOW_META_LOAD_IN_SUBPROCESS: Final[str] = "PF_FLOW_META_LOAD_IN_SUBPROCESS"
+LINE_NUMBER: Final[str] = "line_number"

azure/ai/evaluation/_legacy/_adapters/_errors.py

@@ -0,0 +1,29 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+from typing import Any
+from typing_extensions import TypeAlias
+
+
+try:
+    from promptflow.core._errors import MissingRequiredPackage as _MissingRequiredPackage
+except ImportError:
+    from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException
+
+    class _MissingRequiredPackage(EvaluationException):
+        """Raised when a required package is missing.
+
+        :param message: A message describing the error. This is the error message the user will see.
+        :type message: str
+        """
+
+        def __init__(self, message: str, **kwargs: Any):
+            kwargs.setdefault("category", ErrorCategory.MISSING_PACKAGE)
+            kwargs.setdefault("blame", ErrorBlame.SYSTEM_ERROR)
+            kwargs.setdefault("target", ErrorTarget.EVALUATE)
+            kwargs.setdefault("internal_message", "Missing required package.")
+            super().__init__(message=message, **kwargs)
+
+
+MissingRequiredPackage: TypeAlias = _MissingRequiredPackage

azure/ai/evaluation/_legacy/_adapters/_flows.py

@@ -0,0 +1,28 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+from typing_extensions import TypeAlias
+
+
+try:
+    from promptflow.core._flow import AsyncPrompty as _AsyncPrompty
+    from promptflow._sdk.entities._flows import FlexFlow as _FlexFlow
+    from promptflow._sdk.entities._flows.dag import Flow as _Flow
+except ImportError:
+    from azure.ai.evaluation._legacy.prompty import AsyncPrompty as _AsyncPrompty
+
+    class _FlexFlow:
+        pass
+
+    _FlexFlow.__name__ = "FlexFlow"
+
+    class _Flow:
+        name: str
+
+    _Flow.__name__ = "Flow"
+
+
+AsyncPrompty: TypeAlias = _AsyncPrompty
+FlexFlow: TypeAlias = _FlexFlow
+Flow: TypeAlias = _Flow

azure/ai/evaluation/_legacy/_adapters/_service.py

@@ -0,0 +1,16 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+from typing import Callable, Final
+
+
+try:
+    from promptflow._cli._pf._service import stop_service as _stop_service
+except ImportError:
+
+    def _stop_service() -> None:
+        pass
+
+
+stop_service: Final[Callable[[], None]] = _stop_service

azure/ai/evaluation/_legacy/_adapters/client.py

@@ -0,0 +1,51 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+from os import PathLike
+from typing import Any, Callable, Dict, Optional, Union
+from typing_extensions import TypeAlias
+
+import pandas as pd
+
+from ._errors import MissingRequiredPackage
+from ._configuration import Configuration
+from .entities import Run
+
+
+try:
+    from promptflow.client import PFClient as _PFClient
+except ImportError:
+
+    class _PFClient:
+        def __init__(self, **kwargs):
+            self._config = Configuration(override_config=kwargs.pop("config", None))
+
+        def run(
+            self,
+            flow: Union[str, PathLike, Callable],
+            *,
+            data: Union[str, PathLike],
+            run: Optional[Union[str, Run]] = None,
+            column_mapping: Optional[dict] = None,
+            variant: Optional[str] = None,
+            connections: Optional[dict] = None,
+            environment_variables: Optional[dict] = None,
+            name: Optional[str] = None,
+            display_name: Optional[str] = None,
+            tags: Optional[Dict[str, str]] = None,
+            resume_from: Optional[Union[str, Run]] = None,
+            code: Optional[Union[str, PathLike]] = None,
+            init: Optional[dict] = None,
+            **kwargs,
+        ) -> Run:
+            raise MissingRequiredPackage("Please install 'promptflow' package to use PFClient")
+
+        def get_details(self, run: Union[str, Run], max_results: int = 100, all_results: bool = False) -> pd.DataFrame:
+            return pd.DataFrame()
+
+        def get_metrics(self, run: Union[str, Run]) -> Dict[str, Any]:
+            return {}
+
+
+PFClient: TypeAlias = _PFClient

azure/ai/evaluation/_legacy/_adapters/entities.py

@@ -0,0 +1,26 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+from typing_extensions import TypeAlias
+
+
+try:
+    from promptflow._sdk.entities import Run as _Run
+except ImportError:
+    from typing_extensions import Protocol
+    from typing import Any, Dict, Optional
+    from datetime import datetime
+    from pathlib import Path
+
+    class _Run(Protocol):
+        name: str
+        status: str
+        _properties: Dict[str, Any]
+        _created_on: datetime
+        _end_time: Optional[datetime]
+        _experiment_name: Optional[str]
+        _output_path: Path
+
+
+Run: TypeAlias = _Run

azure/ai/evaluation/_legacy/_adapters/tracing.py

@@ -0,0 +1,28 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+from typing import Callable, Dict, Final, Optional
+from typing_extensions import TypeAlias
+
+
+try:
+    from promptflow.tracing import ThreadPoolExecutorWithContext as _ThreadPoolExecutorWithContext
+    from promptflow.tracing._integrations._openai_injector import (
+        inject_openai_api as _inject,
+        recover_openai_api as _recover,
+    )
+    from promptflow.tracing import _start_trace
+except ImportError:
+    from concurrent.futures import ThreadPoolExecutor as _ThreadPoolExecutorWithContext
+    from azure.ai.evaluation._legacy._batch_engine._openai_injector import (
+        inject_openai_api as _inject,
+        recover_openai_api as _recover,
+    )
+    from azure.ai.evaluation._legacy._batch_engine._trace import start_trace as _start_trace
+
+
+ThreadPoolExecutorWithContext: TypeAlias = _ThreadPoolExecutorWithContext
+inject_openai_api: Final[Callable[[], None]] = _inject
+recover_openai_api: Final[Callable[[], None]] = _recover
+start_trace: Final = _start_trace

azure/ai/evaluation/_legacy/_adapters/types.py

@@ -0,0 +1,15 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+from typing import Any
+
+
+class AttrDict(dict):
+    """A dictionary that allows attribute access to its keys."""
+
+    def __getattr__(self, key: str) -> Any:
+        return self[key]
+
+    def __setattr__(self, key: str, value: Any) -> None:
+        self[key] = value

azure/ai/evaluation/_legacy/_adapters/utils.py

@@ -0,0 +1,31 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+from typing import Final, Optional
+from typing_extensions import TypeAlias
+
+
+try:
+    from promptflow._utils.user_agent_utils import ClientUserAgentUtil as _ClientUserAgentUtil
+    from promptflow._utils.async_utils import async_run_allowing_running_loop as _async_run_allowing_running_loop
+    from promptflow._cli._utils import get_workspace_triad_from_local as _get_workspace_triad_from_local
+except ImportError:
+    from azure.ai.evaluation._legacy._batch_engine._utils_deprecated import (
+        async_run_allowing_running_loop as _async_run_allowing_running_loop,
+    )
+    from azure.ai.evaluation._evaluate._utils import AzureMLWorkspace
+
+    class _ClientUserAgentUtil:
+        @staticmethod
+        def append_user_agent(user_agent: Optional[str]):
+            # TODO ralphe: implement?
+            pass
+
+    def _get_workspace_triad_from_local() -> AzureMLWorkspace:
+        return AzureMLWorkspace("", "", "")
+
+
+ClientUserAgentUtil: TypeAlias = _ClientUserAgentUtil
+async_run_allowing_running_loop: Final = _async_run_allowing_running_loop
+get_workspace_triad_from_local: Final = _get_workspace_triad_from_local

azure/ai/evaluation/_legacy/_batch_engine/__init__.py

@@ -0,0 +1,9 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+# NOTE: This is a direct port of the bare minimum needed for BatchEngine functionality from
+#       the original Promptflow code. The goal here is expediency, not elegance. As such
+#       parts of this code may be a little "quirky", seem incomplete in places, or contain
+#       longer TODOs comments than usual. In a future code update, large swaths of this code
+#       will be refactored or deleted outright.

azure/ai/evaluation/_legacy/_batch_engine/_config.py

@@ -0,0 +1,48 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+from dataclasses import dataclass
+from logging import Logger
+
+from ..._constants import PF_BATCH_TIMEOUT_SEC_DEFAULT
+
+
+@dataclass
+class BatchEngineConfig:
+    """Context for a batch of evaluations. This will contain the configuration,
+    logging, and other needed information."""
+
+    logger: Logger
+    """The logger to use for logging messages."""
+
+    batch_timeout_seconds: int = PF_BATCH_TIMEOUT_SEC_DEFAULT
+    """The maximum amount of time to wait for all evaluations in the batch to complete."""
+
+    line_timeout_seconds: int = 600
+    """The maximum amount of time to wait for an evaluation to run against a single entry
+    in the data input to complete."""
+
+    max_concurrency: int = 10
+    """The maximum number of evaluations to run concurrently."""
+
+    use_async: bool = True
+    """Whether to use asynchronous evaluation."""
+
+    default_num_results: int = 100
+    """The default number of results to return if you don't ask for all results."""
+
+    raise_on_error: bool = True
+    """Whether to raise an error if an evaluation fails."""
+
+    def __post_init__(self):
+        if self.logger is None:
+            raise ValueError("logger cannot be None")
+        if self.batch_timeout_seconds <= 0:
+            raise ValueError("batch_timeout_seconds must be greater than 0")
+        if self.line_timeout_seconds <= 0:
+            raise ValueError("line_timeout_seconds must be greater than 0")
+        if self.max_concurrency <= 0:
+            raise ValueError("max_concurrency must be greater than 0")
+        if self.default_num_results <= 0:
+            raise ValueError("default_num_results must be greater than 0")