genesis-flow 1.0.0__py3-none-any.whl

mlflow/genai/evaluation/utils.py

@@ -0,0 +1,244 @@
+import json
+import logging
+from typing import TYPE_CHECKING, Any, Optional, Union
+
+from mlflow.entities import Assessment, Trace
+from mlflow.exceptions import MlflowException
+from mlflow.genai.evaluation.constant import (
+    AgentEvaluationReserverKey,
+)
+from mlflow.genai.scorers import Scorer
+from mlflow.models import EvaluationMetric
+
+try:
+    # `pandas` is not required for `mlflow-skinny`.
+    import pandas as pd
+except ImportError:
+    pass
+
+if TYPE_CHECKING:
+    from mlflow.genai.datasets import EvaluationDataset
+
+    try:
+        import pyspark.sql.dataframe
+
+        EvaluationDatasetTypes = Union[
+            pd.DataFrame, pyspark.sql.dataframe.DataFrame, list[dict], EvaluationDataset
+        ]
+    except ImportError:
+        EvaluationDatasetTypes = Union[pd.DataFrame, list[dict], EvaluationDataset]
+
+
+_logger = logging.getLogger(__name__)
+
+
+def _convert_eval_set_to_df(data: "EvaluationDatasetTypes") -> "pd.DataFrame":
+    """
+    Takes in a dataset in the format that `mlflow.genai.evaluate()` expects and
+    converts it into a pandas DataFrame.
+    """
+    if isinstance(data, list):
+        # validate that every item in the list is a dict and has inputs as key
+        for item in data:
+            if not isinstance(item, dict):
+                raise MlflowException.invalid_parameter_value(
+                    "Every item in the list must be a dictionary."
+                )
+        df = pd.DataFrame(data)
+    elif isinstance(data, pd.DataFrame):
+        # Data is already a pd DataFrame, just copy it
+        df = data.copy()
+    else:
+        try:
+            from mlflow.utils.spark_utils import get_spark_dataframe_type
+
+            if isinstance(data, get_spark_dataframe_type()):
+                df = _deserialize_inputs_and_expectations_column(data.toPandas())
+            else:
+                raise MlflowException.invalid_parameter_value(
+                    "Invalid type for parameter `data`. Expected a list of dictionaries, "
+                    f"a pandas DataFrame, or a Spark DataFrame. Got: {type(data)}"
+                )
+        except ImportError:
+            raise ImportError(
+                "The `pyspark` package is required to use mlflow.genai.evaluate() "
+                "Please install it with `pip install pyspark`."
+            )
+
+    if len(df) == 0:
+        raise MlflowException.invalid_parameter_value(
+            "The dataset is empty. Please provide a non-empty dataset."
+        )
+
+    if not any(col in df.columns for col in ("trace", "inputs")):
+        raise MlflowException.invalid_parameter_value(
+            "Either `inputs` or `trace` column is required in the dataset. Please provide inputs "
+            "for every datapoint or provide a trace."
+        )
+
+    return df
+
+
+def _convert_to_legacy_eval_set(data: "EvaluationDatasetTypes") -> "pd.DataFrame":
+    """
+    Takes in a dataset in the format that mlflow.genai.evaluate() expects and converts it into
+    to the current eval-set schema that Agent Evaluation takes in. The transformed schema should
+    be accepted by mlflow.evaluate().
+    The expected schema can be found at:
+    https://docs.databricks.com/aws/en/generative-ai/agent-evaluation/evaluation-schema
+
+    NB: The harness secretly support 'expectations' column as well. It accepts a dictionary of
+        expectations, which is same as the schema that mlflow.genai.evaluate() expects.
+        Therefore, we can simply pass through expectations column.
+    """
+    column_mapping = {
+        "inputs": "request",
+        "outputs": "response",
+    }
+
+    df = _convert_eval_set_to_df(data)
+
+    return (
+        df.rename(columns=column_mapping)
+        .pipe(_deserialize_trace_column_if_needed)
+        .pipe(_extract_request_from_trace)
+        .pipe(_extract_expectations_from_trace)
+    )
+
+
+def _deserialize_inputs_and_expectations_column(df: "pd.DataFrame") -> "pd.DataFrame":
+    """
+    Deserialize the `inputs` and `expectations` string columns from the dataframe.
+
+    When managed datasets are read as Spark DataFrames, the `inputs` and `expectations` columns
+    are loaded as string columns of JSON strings. This function deserializes these columns into
+    dictionaries expected by mlflow.genai.evaluate().
+    """
+    target_columns = ["inputs", "expectations"]
+    for col in target_columns:
+        if col not in df.columns or not isinstance(df[col][0], str):
+            continue
+
+        try:
+            df[col] = df[col].apply(json.loads)
+        except json.JSONDecodeError as e:
+            if col == "inputs":
+                msg = (
+                    "The `inputs` column must be a valid JSON string of field names and values. "
+                    "For example, `{'question': 'What is the capital of France?'}`"
+                )
+            else:
+                msg = (
+                    "The `expectations` column must be a valid JSON string of assessment names and "
+                    "values. For example, `{'expected_facts': ['fact1', 'fact2']}`"
+                )
+            raise MlflowException.invalid_parameter_value(
+                f"Failed to parse `{col}` column. Error: {e}\nHint: {msg}"
+            )
+
+    return df
+
+
+def _deserialize_trace_column_if_needed(df: "pd.DataFrame") -> "pd.DataFrame":
+    """
+    Deserialize the `trace` column from the dataframe if it is a string.
+
+    Since MLflow 3.2.0, mlflow.search_traces() returns a pandas DataFrame with a `trace`
+    column that is a trace json representation rather than the Trace object itself. This
+    function deserializes the `trace` column into a Trace object.
+    """
+    if "trace" in df.columns:
+        df["trace"] = df["trace"].apply(lambda t: Trace.from_json(t) if isinstance(t, str) else t)
+    return df
+
+
+def _extract_request_from_trace(df: "pd.DataFrame") -> "pd.DataFrame":
+    """
+    Add `request` columns to the dataframe if it is not already present.
+    This is for compatibility with mlflow.evaluate() that requires `request` column.
+    """
+    if "trace" not in df.columns:
+        return df
+
+    if "request" not in df.columns:
+        df["request"] = df["trace"].apply(lambda trace: json.loads(trace.data.request))
+    return df
+
+
+def _extract_expectations_from_trace(df: "pd.DataFrame") -> "pd.DataFrame":
+    """
+    Add `expectations` columns to the dataframe from assessments
+    stored in the traces, if the "expectations" column is not already present.
+    """
+    if "trace" not in df.columns:
+        return df
+
+    expectations_column = []
+    for trace in df["trace"]:
+        expectations = {}
+        for assessment in trace.info.assessments or []:
+            if assessment.expectation is not None:
+                expectations[assessment.name] = assessment.expectation.value
+        expectations_column.append(expectations)
+    # If no trace has assessments, not add the column
+    if all(len(expectations) == 0 for expectations in expectations_column):
+        return df
+
+    df["expectations"] = expectations_column
+    return df
+
+
+def _convert_scorer_to_legacy_metric(scorer: Scorer) -> EvaluationMetric:
+    """
+    Takes in a Scorer object and converts it into a legacy MLflow 2.x
+    Metric object.
+    """
+    try:
+        from databricks.agents.evals import metric
+    except ImportError:
+        raise ImportError(
+            "The `databricks-agents` package is required to use mlflow.genai.evaluate() "
+            "Please install it with `pip install databricks-agents`."
+        )
+
+    from mlflow.types.llm import ChatCompletionRequest
+
+    def eval_fn(
+        request_id: str,
+        request: Union[ChatCompletionRequest, str],
+        response: Optional[Any],
+        expected_response: Optional[Any],
+        trace: Optional[Trace],
+        guidelines: Optional[Union[list[str], dict[str, list[str]]]],
+        expected_facts: Optional[list[str]],
+        expected_retrieved_context: Optional[list[dict[str, str]]],
+        custom_expected: Optional[dict[str, Any]],
+        **kwargs,
+    ) -> Union[int, float, bool, str, Assessment, list[Assessment]]:
+        # Condense all expectations into a single dict
+        expectations = {}
+        if expected_response is not None:
+            expectations[AgentEvaluationReserverKey.EXPECTED_RESPONSE] = expected_response
+        if expected_facts is not None:
+            expectations[AgentEvaluationReserverKey.EXPECTED_FACTS] = expected_facts
+        if expected_retrieved_context is not None:
+            expectations[AgentEvaluationReserverKey.EXPECTED_RETRIEVED_CONTEXT] = (
+                expected_retrieved_context
+            )
+        if guidelines is not None:
+            expectations[AgentEvaluationReserverKey.GUIDELINES] = guidelines
+        if custom_expected is not None:
+            expectations.update(custom_expected)
+
+        merged = {
+            "inputs": request,
+            "outputs": response,
+            "expectations": expectations,
+            "trace": trace,
+        }
+        return scorer.run(**merged)
+
+    return metric(
+        eval_fn=eval_fn,
+        name=scorer.name,
+    )

mlflow/genai/judges/__init__.py

@@ -0,0 +1,21 @@
+from mlflow.genai.judges.databricks import (
+    CategoricalRating,
+    custom_prompt_judge,
+    is_context_relevant,
+    is_context_sufficient,
+    is_correct,
+    is_grounded,
+    is_safe,
+    meets_guidelines,
+)
+
+__all__ = [
+    "CategoricalRating",
+    "is_grounded",
+    "is_safe",
+    "is_correct",
+    "is_context_relevant",
+    "is_context_sufficient",
+    "meets_guidelines",
+    "custom_prompt_judge",
+]

mlflow/genai/judges/databricks.py

@@ -0,0 +1,404 @@
+from functools import wraps
+from typing import Any, Callable, Optional, Union
+
+from mlflow.entities.assessment import Feedback
+from mlflow.genai.utils.enum_utils import StrEnum
+from mlflow.utils.annotations import experimental
+
+# NB: User-facing name for the is_context_relevant assessment.
+_IS_CONTEXT_RELEVANT_ASSESSMENT_NAME = "relevance_to_context"
+
+
+class CategoricalRating(StrEnum):
+    """
+    A categorical rating for an assessment.
+
+    Example:
+        .. code-block:: python
+
+            from mlflow.genai.judges import CategoricalRating
+            from mlflow.entities import Feedback
+
+            # Create feedback with categorical rating
+            feedback = Feedback(
+                name="my_metric", value=CategoricalRating.YES, rationale="The metric is passing."
+            )
+    """
+
+    YES = "yes"
+    NO = "no"
+    UNKNOWN = "unknown"
+
+    @classmethod
+    def _missing_(cls, value: str):
+        value = value.lower()
+        for member in cls:
+            if member == value:
+                return member
+        return cls.UNKNOWN
+
+
+def _sanitize_feedback(feedback: Feedback) -> Feedback:
+    """Sanitize the feedback object from the databricks judges.
+
+    The judge returns a CategoricalRating class defined in the databricks-agents package.
+    This function converts it to our CategoricalRating definition above.
+
+    Args:
+        feedback: The Feedback object to convert.
+
+    Returns:
+        A new Feedback object with our CategoricalRating.
+    """
+    feedback.value = CategoricalRating(feedback.value) if feedback.value else feedback.value
+    return feedback
+
+
+def requires_databricks_agents(func):
+    """Decorator to check if the `databricks-agents` package is installed."""
+
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+        try:
+            import databricks.agents.evals.judges  # noqa: F401
+
+        except ImportError:
+            raise ImportError(
+                f"The `databricks-agents` package is required to use "
+                f"`mlflow.genai.judges.{func.__name__}`. "
+                "Please install it with `pip install databricks-agents`."
+            )
+
+        return func(*args, **kwargs)
+
+    return wrapper
+
+
+@requires_databricks_agents
+def is_context_relevant(*, request: str, context: Any, name: Optional[str] = None) -> Feedback:
+    """
+    LLM judge determines whether the given context is relevant to the input request.
+
+    Args:
+        request: Input to the application to evaluate, user's question or query.
+        context: Context to evaluate the relevance to the request.
+            Supports any JSON-serializable object.
+        name: Optional name for overriding the default name of the returned feedback.
+
+    Returns:
+        A :py:class:`mlflow.entities.assessment.Feedback~` object with a "yes" or "no" value
+        indicating whether the context is relevant to the request.
+
+    Example:
+
+        The following example shows how to evaluate whether a document retrieved by a
+        retriever is relevant to the user's question.
+
+        .. code-block:: python
+
+            from mlflow.genai.judges import is_context_relevant
+
+            feedback = is_context_relevant(
+                request="What is the capital of France?",
+                context="Paris is the capital of France.",
+            )
+            print(feedback.value)  # "yes"
+
+            feedback = is_context_relevant(
+                request="What is the capital of France?",
+                context="Paris is known for its Eiffel Tower.",
+            )
+            print(feedback.value)  # "no"
+
+    """
+    from databricks.agents.evals.judges import relevance_to_query
+
+    return _sanitize_feedback(
+        relevance_to_query(
+            request=request,
+            response=str(context),
+            # NB: User-facing name for the is_context_relevant assessment. This is required since
+            #     the existing databricks judge is called `relevance_to_query`
+            assessment_name=name or _IS_CONTEXT_RELEVANT_ASSESSMENT_NAME,
+        )
+    )
+
+
+@requires_databricks_agents
+def is_context_sufficient(
+    *,
+    request: str,
+    context: Any,
+    expected_facts: list[str],
+    expected_response: Optional[str] = None,
+    name: Optional[str] = None,
+) -> Feedback:
+    """
+    LLM judge determines whether the given context is sufficient to answer the input request.
+
+    Args:
+        request: Input to the application to evaluate, user's question or query.
+        context: Context to evaluate the sufficiency of. Supports any JSON-serializable object.
+        expected_facts: A list of expected facts that should be present in the context.
+        expected_response: The expected response from the application. Optional.
+        name: Optional name for overriding the default name of the returned feedback.
+
+    Returns:
+        A :py:class:`mlflow.entities.assessment.Feedback~` object with a "yes" or "no"
+        value indicating whether the context is sufficient to answer the request.
+
+    Example:
+
+        The following example shows how to evaluate whether the documents returned by a
+        retriever gives sufficient context to answer the user's question.
+
+        .. code-block:: python
+
+            from mlflow.genai.judges import is_context_sufficient
+
+            feedback = is_context_sufficient(
+                request="What is the capital of France?",
+                context=[
+                    {"content": "Paris is the capital of France."},
+                    {"content": "Paris is known for its Eiffel Tower."},
+                ],
+                expected_facts=["Paris is the capital of France."],
+            )
+            print(feedback.value)  # "yes"
+    """
+    from databricks.agents.evals.judges import context_sufficiency
+
+    return _sanitize_feedback(
+        context_sufficiency(
+            request=request,
+            retrieved_context=context,
+            expected_facts=expected_facts,
+            expected_response=expected_response,
+            assessment_name=name,
+        )
+    )
+
+
+@requires_databricks_agents
+def is_correct(
+    *,
+    request: str,
+    response: str,
+    expected_facts: list[str],
+    expected_response: Optional[str] = None,
+    name: Optional[str] = None,
+) -> Feedback:
+    """
+    LLM judge determines whether the given response is correct for the input request.
+
+    Args:
+        request: Input to the application to evaluate, user's question or query.
+        response: The response from the application to evaluate.
+        expected_facts: A list of expected facts that should be present in the response.
+        expected_response: The expected response from the application. Optional.
+        name: Optional name for overriding the default name of the returned feedback.
+
+    Returns:
+        A :py:class:`mlflow.entities.assessment.Feedback~` object with a "yes" or "no"
+        value indicating whether the response is correct for the request.
+    """
+    from databricks.agents.evals.judges import correctness
+
+    return _sanitize_feedback(
+        correctness(
+            request=request,
+            response=response,
+            expected_facts=expected_facts,
+            expected_response=expected_response,
+            assessment_name=name,
+        )
+    )
+
+
+@requires_databricks_agents
+def is_grounded(
+    *, request: str, response: str, context: Any, name: Optional[str] = None
+) -> Feedback:
+    """
+    LLM judge determines whether the given response is grounded in the given context.
+
+    Args:
+        request: Input to the application to evaluate, user's question or query.
+        response: The response from the application to evaluate.
+        context: Context to evaluate the response against. Supports any JSON-serializable object.
+        name: Optional name for overriding the default name of the returned feedback.
+
+    Returns:
+        A :py:class:`mlflow.entities.assessment.Feedback~` object with a "yes" or "no"
+        value indicating whether the response is grounded in the context.
+
+    Example:
+
+        The following example shows how to evaluate whether the response is grounded in
+        the context.
+
+        .. code-block:: python
+
+            from mlflow.genai.judges import is_grounded
+
+            feedback = is_grounded(
+                request="What is the capital of France?",
+                response="Paris",
+                context=[
+                    {"content": "Paris is the capital of France."},
+                    {"content": "Paris is known for its Eiffel Tower."},
+                ],
+            )
+            print(feedback.value)  # "yes"
+    """
+    from databricks.agents.evals.judges import groundedness
+
+    return _sanitize_feedback(
+        groundedness(
+            request=request,
+            response=response,
+            retrieved_context=context,
+            assessment_name=name,
+        )
+    )
+
+
+@requires_databricks_agents
+def is_safe(*, content: str, name: Optional[str] = None) -> Feedback:
+    """
+    LLM judge determines whether the given response is safe.
+
+    Args:
+        content: Text content to evaluate for safety.
+        name: Optional name for overriding the default name of the returned feedback.
+
+    Returns:
+        A :py:class:`mlflow.entities.assessment.Feedback~` object with a "yes" or "no"
+        value indicating whether the response is safe.
+
+    Example:
+
+        .. code-block:: python
+
+            from mlflow.genai.judges import is_safe
+
+            feedback = is_safe(content="I am a happy person.")
+            print(feedback.value)  # "yes"
+    """
+    from databricks.agents.evals.judges import safety
+
+    return _sanitize_feedback(safety(response=content, assessment_name=name))
+
+
+@requires_databricks_agents
+def meets_guidelines(
+    *,
+    guidelines: Union[str, list[str]],
+    context: dict[str, Any],
+    name: Optional[str] = None,
+) -> Feedback:
+    """
+    LLM judge determines whether the given response meets the given guideline(s).
+
+    Args:
+        guidelines: A single guideline or a list of guidelines.
+        context: Mapping of context to be evaluated against the guidelines. For example,
+            pass {"response": "<response text>"} to evaluate whether the response meets
+            the given guidelines.
+        name: Optional name for overriding the default name of the returned feedback.
+
+    Returns:
+        A :py:class:`mlflow.entities.assessment.Feedback~` object with a "yes" or "no"
+        value indicating whether the response meets the guideline(s).
+
+    Example:
+
+        The following example shows how to evaluate whether the response meets the given
+        guideline(s).
+
+        .. code-block:: python
+
+            from mlflow.genai.judges import meets_guidelines
+
+            feedback = meets_guidelines(
+                guidelines="Be polite and respectful.",
+                context={"response": "Hello, how are you?"},
+            )
+            print(feedback.value)  # "yes"
+
+            feedback = meets_guidelines(
+                guidelines=["Be polite and respectful.", "Must be in English."],
+                context={"response": "Hola, ¿cómo estás?"},
+            )
+            print(feedback.value)  # "no"
+    """
+    from databricks.agents.evals.judges import guideline_adherence
+
+    # Ensure guidelines is a list, as the underlying databricks judge only accepts lists
+    if isinstance(guidelines, str):
+        guidelines = [guidelines]
+
+    return _sanitize_feedback(
+        guideline_adherence(
+            guidelines=guidelines,
+            guidelines_context=context,
+            assessment_name=name,
+        )
+    )
+
+
+@experimental(version="3.0.0")
+@requires_databricks_agents
+def custom_prompt_judge(
+    *,
+    name: str,
+    prompt_template: str,
+    numeric_values: Optional[dict[str, Union[int, float]]] = None,
+) -> Callable[..., Feedback]:
+    """
+    Create a custom prompt judge that evaluates inputs using a template.
+
+    Example prompt template:
+
+    .. code-block::
+
+        You will look at the response and determine the formality of the response.
+
+        <request>{{request}}</request>
+        <response>{{response}}</response>
+
+        You must choose one of the following categories.
+
+        [[formal]]: The response is very formal.
+        [[semi_formal]]: The response is somewhat formal. The response is somewhat formal if the
+        response mentions friendship, etc.
+        [[not_formal]]: The response is not formal.
+
+    Variable names in the template should be enclosed in double curly
+    braces, e.g., `{{request}}`, `{{response}}`. They should be alphanumeric and can include
+    underscores, but should not contain spaces or special characters.
+
+    It is required for the prompt template to request choices as outputs, with each choice
+    enclosed in square brackets. Choice names should be alphanumeric and can include
+    underscores and spaces.
+
+    Args:
+        name: Name of the judge, used as the name of returned
+            :py:class`mlflow.entities.Feedback~` object.
+        prompt_template: Template string with {{var_name}} placeholders for variable substitution.
+            Should be prompted with choices as outputs.
+        numeric_values: Optional mapping from categorical values to numeric scores.
+            Useful if you want to create a custom judge that returns continuous valued outputs.
+            Defaults to None.
+
+    Returns:
+        A callable that takes keyword arguments mapping to the template variables
+        and returns an mlflow :py:class`mlflow.entities.Feedback~`.
+    """
+    from databricks.agents.evals.judges import custom_prompt_judge
+
+    return custom_prompt_judge(
+        name=name,
+        prompt_template=prompt_template,
+        numeric_values=numeric_values,
+    )