python-flexeval 0.1.5__py3-none-any.whl

flexeval/dependency_graph.py

@@ -0,0 +1,234 @@
+"""Determines how configured metrics depend on each other."""
+
+import json
+from typing import Any
+
+import networkx as nx
+
+from flexeval.helpers import generate_hash
+from flexeval.schema import eval_schema
+
+
+def create_metrics_graph(metrics: eval_schema.Metrics) -> list[Any]:
+    """Input is the metrics dictionary with keys 'function' and 'rubric', each of which maps to a list
+    Output is list of string representations of the nodes in the graph, in topological order
+
+    Each entry and dependency will get an ID so they are easy to match later
+    """
+
+    # Create a directed graph
+    G = nx.DiGraph()
+    metric_graph_dict = {}
+
+    # make an intermediate datastructure that adds IDs to all listed evaluations
+    user_metrics_with_ids = {}
+    for evaluation_type in ["function", "rubric"]:
+        user_metrics_with_ids[evaluation_type] = []
+        # add a hash to every metric in the list
+        item_list: list[eval_schema.MetricItem] = getattr(metrics, evaluation_type)
+        if item_list is not None:
+            for item in item_list:
+                metric_with_id = {"id": generate_hash()}
+                for k, v in item.model_dump().items():
+                    metric_with_id[k] = v
+                user_metrics_with_ids[evaluation_type].append(metric_with_id)
+
+    # now that all potential parents have IDs, find parents for each child
+    for evaluation_type in ["function", "rubric"]:
+        for metric_dict in user_metrics_with_ids[evaluation_type]:
+            parent_metrics, depends_on_with_parent_ids = get_parent_metrics(
+                all_metrics=user_metrics_with_ids, child=metric_dict
+            )
+            metric_dict["depends_on"] = depends_on_with_parent_ids
+
+            child_metric_str, evaluation_name = get_metric_info(metric_dict)
+
+            # Now construct the graph
+            # Add an edge, which implicitly adds nodes where necessary
+            if len(parent_metrics) > 0:
+                for parent_metric_dict in parent_metrics:
+                    parent_metric_str, _ = get_metric_info(parent_metric_dict)
+                    G.add_edge(parent_metric_str, child_metric_str)
+                # make 'canonical' representation of child
+                metric_graph_dict[child_metric_str] = {
+                    "evaluation_name": evaluation_name,  # function or rubric name
+                    "evaluation_type": evaluation_type,
+                }
+                for k, v in metric_dict.items():
+                    if k not in [
+                        "function_name",
+                        "rubric_name",
+                        "type",
+                        "name",
+                    ]:
+                        metric_graph_dict[child_metric_str][k] = v
+
+                # # copy over details of parent metric that aren't already present
+                # for k, v in parent_metric_dict.items():
+                #     if k not in metric_graph_dict[child_metric]:
+                #         metric_graph_dict[child_metric][k] = v
+            else:
+                # if there is no parent, just add a node by itself
+                G.add_node(child_metric_str)
+                metric_graph_dict[child_metric_str] = {
+                    "evaluation_name": evaluation_name,  # function or rubric name
+                    "evaluation_type": evaluation_type,
+                }
+                for k, v in metric_dict.items():
+                    if k not in [
+                        "function_name",
+                        "rubric_name",
+                        "type",
+                        "name",
+                    ]:
+                        metric_graph_dict[child_metric_str][k] = v
+
+    # Make string representation with all nodes for error printing in assertion
+    graph_string = "Metric Dependencies:"
+    for edge in G.edges():
+        graph_string += f"\n{'' if edge[1] == 'root' else edge[1]} -> {edge[0]}"
+    if not nx.is_directed_acyclic_graph(G):
+        raise ValueError(
+            "The set of metric dependencies must be acyclic! You have cyclical dependencies. {graph_string}"
+        )
+
+    # Set up sequence of evaluations
+    # Perform topological sort
+    # This is the order in which metrics will be evaluated
+    # and the conditions under which they will be evaluated
+    topological_order = list(nx.topological_sort(G))
+
+    metric_graph = [metric_graph_dict[node] for node in topological_order]
+    return metric_graph
+
+
+def get_metric_info(single_metric: dict) -> tuple[str, str]:
+    """Input will be a single metric dictionary
+    Output will be
+    - string representation of metric using json.dumps
+    - evaluation_name - function_name or rubric_name
+    """
+    return json.dumps(single_metric), single_metric.get("name")
+
+
+def get_parent_metrics(all_metrics: dict, child: dict) -> tuple[list, list]:
+    """metrics_graph_ordered_list will be a list of metrics in order in which they should be run
+
+    This function takes the eval represented by "child" and finds ALL evals in "all_metrics"
+    that quality as the child's immediate parent
+
+    An eval can qualify as a parent by having a matching name, type, context_only
+    At this point, we won't have enough information to decide whether the child should be run
+    (since the child might have additional requirements on the output of the parent)
+    but this is enough to tell us that the child should be run AFTER the parent.
+    """
+
+    # if we use defaults in "depends_on", we might ends up with non-matches accidentally
+    # for a dependency, multiple keys might be listed
+    # We should find at least one parent that matches ALL of those key/value pairs, otherwise raise an error
+    parents = []
+    depends_on_with_id_added = []
+    for requirement in child.get("depends_on", []):
+        candidate_parents = []
+        allowed_types = ["function", "rubric"]
+        # if requirement has the type narrowed down, then narrow it down here too
+        if "type" in requirement and requirement["type"] is not None:
+            allowed_types = [requirement["type"]]
+        for candidate_type in allowed_types:
+            for candidate in all_metrics.get(candidate_type, []):
+                # assume the candidate is a match unless demonstrated otherwise
+                matches = True
+
+                # if it's not the right type, don't match it
+                if "type" in requirement and candidate_type not in allowed_types:
+                    matches = False
+
+                # if the conditionals are listed in the depends_on entry but don't match...
+                # Only check conditionals that are explicitly specified (not None) in the requirement
+                conditionals = ["metric_level", "context_only", "name", "kwargs"]
+                for conditional in conditionals:
+                    if (
+                        conditional in requirement
+                        and requirement.get(conditional) is not None
+                        and requirement.get(conditional) != candidate.get(conditional)
+                    ):
+                        matches = False
+                        break
+
+                if matches:
+                    candidate_parents.append(candidate)
+                    requirement["parent_id"] = candidate["id"]
+                    depends_on_with_id_added.append(requirement)
+        if len(candidate_parents) == 0:
+            raise ValueError(
+                f"We were unable to locate any match for the `depends_on` entry `{json.dumps(requirement, indent=4)}` in the metric `{json.dumps(child, indent=4)}`. The full set of parent candidates is `{json.dumps(all_metrics, indent=4)}`."
+            )
+        if len(candidate_parents) > 1:
+            raise ValueError(
+                f"We located more than one match for the `depends_on` entry `{json.dumps(requirement, indent=4)}` in the metric `{json.dumps(child, indent=4)}`. The matches were `{json.dumps(candidate_parents, indent=4)}`. Please add another criterion to disambiguate."
+            )
+        parents += candidate_parents
+
+    return parents, depends_on_with_id_added
+
+
+def apply_defaults(schema, data, path=None):
+    # Initialize path as an empty list if None. This will store the navigation path in the schema.
+
+    if path is None:
+        path = []
+
+    if data is None:
+        # If data is None and defaults are specified, apply them
+        return schema.get("default")
+
+    if isinstance(data, dict):
+        # Process dictionaries
+        if "properties" in schema:
+            # Loop over each schema property
+            for key, subschema in schema["properties"].items():
+                # Update path with current property
+                new_path = path + [key]
+                if key in data:
+                    # Recursively apply defaults, pass the path along
+                    data[key] = apply_defaults(subschema, data[key], new_path)
+                elif "default" in subschema:
+                    # Apply default if the key is not in the data
+                    data[key] = subschema["default"]
+                    # print("setting", path, key, subschema["default"])
+        elif "items" in schema:
+            if "properties" in schema["items"]:
+                # Loop over each schema property
+                for key, subschema in schema["items"]["properties"].items():
+                    # Update path with current property
+                    new_path = path + [key]
+                    if key in data:
+                        # Recursively apply defaults, pass the path along
+                        data[key] = apply_defaults(subschema, data[key], new_path)
+                    elif "default" in subschema:
+                        # Apply default if the key is not in the data
+                        data[key] = subschema["default"]
+
+        if path == ["metrics", "function"]:
+            data["type"] = "function"
+        if path == ["metrics", "rubric"]:
+            data["type"] = "rubric"
+
+        return data
+
+    if isinstance(data, list) and "items" in schema:
+        # Process lists by applying defaults to each item
+        item_schema = schema["items"]
+        # Apply defaults to each item in the list, passing along the path
+        return [apply_defaults(item_schema, item, path) for item in data]
+
+    return data
+
+
+# for verify installation
+# if function_name is defined, rubric
+# make sure "function" and "rubric" default to empty lists
+# TODO - don't set defaults in "depends_on" to make matching more flexible
+# evaluation_name: my_rubric
+# evaluation_type: rubric
+# metric_name: <

flexeval/eval_schema.json

@@ -0,0 +1,256 @@
+{
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "type": "object",
+    "properties": {
+    "data": {
+      "type": "array",
+      "items": {
+        "type": "string"
+      },
+      "description": "List of absolute or relative paths to data files. Each file must be in *.jsonl format, with one conversation per line.",
+      "default":[]
+    },
+    "do_completion": {
+      "type": "boolean",
+      "description": "Flag to determine if completions should be done for each conversation. Set to 'true' if you are testing a new API and want to evaluate the API responses. Set to 'false' (default) if you are evaluating past conversations and do not need to generate new completions.",
+      "default": false
+    },
+    "name": {
+      "type": "string",
+      "description": "Name of the test suite. Used as metadata only. Does not need to match the key of the entry in the evals.yaml file.",
+      "default": ""
+    },
+    "notes": {
+      "type": "string",
+      "description": "Additional notes regarding the configuration. Used as metadata only.",
+      "default": ""
+    },
+    "config": {
+      "type": "object",
+      "properties": {
+        "max_workers": {
+          "type": "integer",
+          "description": "The maximum number of worker threads allowed when computing metrics."
+        }
+      },
+      "description": "Specific configuration settings that may override default settings. Look in `src/llm-evals/config.yaml` for other fun things to put here.",
+      "additionalProperties": true
+    },
+    "metrics": {
+      "type": "object",
+      "properties":{
+          "function":{
+              "type": "array",
+              "description": "List of function-based metrics to be evaluated.",
+              "items": {
+                  "type": "object",
+                  "properties": {
+                    "name": {
+                      "type": "string",
+                      "description": "The function to call to compute this metric."
+                    },
+                    "kwargs": {
+                      "type": "object",
+                      "description": "Keyword arguments for the function. Each key must correspond to an argument in the function as implemented in `function_metrics.py`. Extra keys will cause an error.",
+                      "additionalProperties": true,
+                      "default": {}
+                    },
+                    "depends_on": {
+                      "type": "array",
+                      "default": [],
+                      "description": "List of dependencies that must be satisfied for this metric to be computed.",
+                      "items": {
+                        "type": "object",
+                        "properties": {
+                          "name": {
+                            "type": "string",
+                            "description": "Name of the dependency function or rubric."
+                          },
+                          "type": {
+                            "type": "string",
+                            "description": "One of 'function' or 'rubric' indicating the type of the dependency.",
+                            "pattern": "^((function)|(rubric))$"
+                          },
+                          "kwargs": {
+                            "type": "object",
+                            "description": "The keyword arguments for the dependency. If provided, used to match which evaluation this dependency is for, so must match the keyword args given for some evaluation.",
+                            "additionalProperties": true
+                          },
+                          "context_only": {
+                            "type": "boolean",
+                            "description": "The context_only value for the dependency. If provided, used to match which evaluation this dependency is for."
+                          },
+                          "last_turn_only": {
+                              "type": "boolean",
+                              "description": "The last_turn_only value for the dependency. If provided, used to match which evaluation this dependency is for."
+                          },
+                          "metric_name": {
+                            "type": "string",
+                            "description": "Name of the metric dependency. This may be different than function_name if the metric function returns a key/value pair - in which case, this will match the key."
+                          },
+                          "metric_min_value": {
+                            "type": "number",
+                            "description": "Minimum value of the dependency to consider it as satisfied.",
+                            "default": -1e20
+                          },
+                          "metric_max_value": {
+                            "type": "number",
+                            "description": "Maximum value of the dependency to consider it as satisfied.",
+                            "default": 1e20
+                          }
+                        },
+                        "additionalProperties": false
+                      },
+                      "required": ["name"]
+                    },
+                    "metric_level": {
+                      "type": "string",
+                      "description": "What level of granularity (ToolCall, Message, Turn, or Thread) this rubric should be applied to",
+                      "default": "Turn"
+                    },
+                    "context_only": {
+                      "type": "boolean",
+                      "description": "If true, only the context (that is, the previous messages) will be evaluated, not the current object. Cannot be done with only thread",
+                      "default": false
+                    },
+                    "last_instance_only": {
+                      "type": "boolean",
+                      "description": "If true, the object will only be evaluated if it's the last instance (i.e., turn or message depending on metric_level) in an existing conversation, or if it's a new completion.",
+                      "default": false
+                    }
+                  },
+                  "required": ["name"]
+                }
+          },
+          "rubric":{
+              "type": "array",
+              "description": "List of rubrics to be evaluated",
+              "items": {
+                  "type": "object",
+                  "properties": {
+                      "name": {
+                          "type": "string",
+                          "description": "The rubric to use to evaluate this metric."
+                      },
+                      "kwargs": {
+                          "type": "object",
+                          "description": "Keyword arguments for the function. Each key must correspond to an argument in the function as implemented in `function_metrics.py`. Extra keys will cause an error.",
+                          "additionalProperties": true,
+                          "default":{}
+                      },
+                      "metric_level": {
+                        "type": "string",
+                        "description": "What level of granularity (ToolCall, Message, Turn, or Thread) this rubric should be applied to",
+                        "default": "Turn"
+                      },
+                      "context_only": {
+                          "type": "boolean",
+                          "description": "If true, only the context (that is, the previous messages) will be evaluated, not the current object. Cannot be done with only thread",
+                          "default": false
+                      },
+                      "last_instance_only": {
+                          "type": "boolean",
+                          "description": "If true, the object will only be evaluated if it's the last instance (i.e., turn or message depending on metric_level) in an existing conversation, or if it's a new completion.",
+                          "default": false
+                      },
+                      "depends_on": {
+                          "type": "array",
+                          "description": "List of dependencies that must be satisfied for this metric to be computed.",
+                          "default":[],
+                          "items": {
+                              "type": "object",
+                              "properties": {
+                                  "name": {
+                                    "type": "string",
+                                    "description": "Name of the dependency function or rubric."
+                                  },
+                                  "type": {
+                                    "type": "string",
+                                    "description": "One of 'function' or 'rubric' indicating the type of the dependency.",
+                                    "pattern": "^((function)|(rubric))$"
+                                  },
+                                  "kwargs": {
+                                    "type": "object",
+                                    "description": "The keyword arguments for the dependency. If provided, used to match which evaluation this dependency is for, so must match the keyword args given for some evaluation.",
+                                    "additionalProperties": true
+                                  },
+                                  "context_only": {
+                                    "type": "boolean",
+                                    "description": "The context_only value for the dependency. If provided, used to match which evaluation this dependency is for."
+                                  },
+                                  "last_turn_only": {
+                                      "type": "boolean",
+                                      "description": "The last_turn_only value for the dependency. If provided, used to match which evaluation this dependency is for."
+                                  },
+                                  "metric_name": {
+                                      "type": "string",
+                                      "description": "Name of the metric dependency. This may be different than function_name if the metric function returns a key/value pair - in which case, this will match the key."
+                                  },
+                                  "metric_min_value": {
+                                      "type": "number",
+                                      "description": "Minimum value of the dependency to consider it as satisfied.",
+                                      "default": -1e20
+                                  },
+                                  "metric_max_value": {
+                                      "type": "number",
+                                      "description": "Maximum value of the dependency to consider it as satisfied.",
+                                      "default": 1e20
+                                  }
+                              },
+                              "additionalProperties": false
+                          },
+                          "required": ["name"]
+                      }
+                  },
+                  "required": ["name"]
+                  }
+          }
+      }
+    },
+    "completion_llm": {
+      "type": "object",
+      "description":"Specification of the LLM or API used to perform new completions. Must be defined if `do_completions: true` is set.",
+      "properties": {
+        "function_name": {
+          "type": "string",
+          "description": "Completion function defined in `completion_functions.py`. Must be specified."
+        },
+        "include_system_prompt": {
+          "type": "boolean",
+          "default": false
+        },
+        "kwargs": {
+          "type": "object",
+          "description": "Additional arguments that will be passed to the completion function. Must correspond to arguments in tne named function.",
+          "default": {},
+          "additionalProperties": true
+        }
+      },
+      "required": ["function_name"],
+      "additionalProperties": false
+    },
+    "grader_llm": {
+      "type": "object",
+      "description":"Specification of the LLM or API used to grade rubrics. Must be defined if any rubric_metrics are specified.",
+      "properties": {
+        "function_name": {
+          "type": "string",
+          "description": "Function defined in `completion_functions.py`. We're not really completing a conversation, but we ARE asking an LLM to provide a response to an input - in this case, the rubric."
+        },
+        "kwargs": {
+          "type": "object",
+          "description": "Additional arguments that will be passed to the completion function. Must correspond to arguments in tne named function.",
+          "default": {},
+          "additionalProperties": true
+        }
+      },
+      "required": ["function_name"],
+      "optional": ["kwargs"],
+      "additionalProperties": false
+    }
+    
+  },
+  "required": ["data", "metrics"],
+  "additionalProperties": true
+}
+  

flexeval/function_types.py

@@ -0,0 +1,173 @@
+"""Inspection utilities that use type hints to determine the appropriate object to pass to a function metric.
+
+See :mod:`~flexeval.schema.eval_schema`.
+"""
+
+import inspect
+import logging
+import types
+import typing
+from collections.abc import Callable, Iterable
+
+from flexeval.classes import message, thread, tool_call, turn
+from flexeval.schema import eval_schema
+
+AnyFunctionObjectInput = typing.Union[
+    turn.Turn,
+    message.Message,
+    thread.Thread,
+    tool_call.ToolCall,
+]
+FLEXEVAL_TYPE_SET: set[type] = {
+    turn.Turn,
+    message.Message,
+    thread.Thread,
+    tool_call.ToolCall,
+}
+
+logger = logging.getLogger(__name__)
+
+
+def is_callable_valid_for_metric_level(
+    metric_function: Callable, metric_level: eval_schema.MetricLevel
+) -> bool:
+    valid_levels = get_valid_levels_for_callable(metric_function)
+    return metric_level in valid_levels
+
+
+def get_valid_levels_for_callable(metric_function: Callable) -> set[str]:
+    """Given a callable, determine the valid metric_level values based on the type annotation of the first parameter.
+
+    Args:
+        metric_function (Callable): A callable, probably one available via EvalRun
+
+    Returns:
+        set[str]: Valid values for MetricItem.metric_level
+    """
+    accepted_parameter_types = get_first_parameter_types(metric_function)
+    valid_levels = set()
+    for flexeval_type in FLEXEVAL_TYPE_SET:
+        if flexeval_type in accepted_parameter_types:
+            valid_levels.add(flexeval_type.__name__)
+    if str in accepted_parameter_types:
+        for level in ["Message", "Turn", "Thread"]:
+            valid_levels.add(level)
+    if list in accepted_parameter_types:
+        for level in ["Turn", "Thread"]:
+            valid_levels.add(level)
+    if dict in accepted_parameter_types:
+        valid_levels.add("ToolCall")
+    return valid_levels
+
+
+def get_first_parameter_types(metric_function: Callable) -> set[type]:
+    input_type = next(
+        iter(inspect.signature(metric_function).parameters.values())
+    ).annotation
+    if input_type is inspect._empty:
+        logger.debug(
+            f"Function '{metric_function}' has a first parameter with no type annotation."
+        )
+        return set()
+    return get_acceptable_arg_types(input_type)
+
+
+def get_acceptable_arg_types(input_type: type) -> set[type]:
+    # Note: we don't support NewType annotations yet
+    origin_type = typing.get_origin(input_type)
+    if origin_type is typing.Annotated:
+        # unpack Annotated types
+        input_type = typing.get_args(input_type)[0]
+        origin_type = typing.get_origin(input_type)
+    if origin_type in (typing.Union, types.UnionType):
+        union_arg_type_sets = [
+            get_acceptable_arg_types(type_arg)
+            for type_arg in typing.get_args(input_type)
+        ]
+        return set.union(*union_arg_type_sets)
+    else:  # not a union type
+        if origin_type is not None:
+            # e.g. input_type=list[str], origin_type=list
+            return {origin_type}
+        else:
+            # e.g. input_type=list, origin_type=list
+            if input_type is list or input_type is Iterable:
+                logger.warning(
+                    "Type hint {input_type} lacks the detail that would allow us to determine the specific objects it accepts."
+                )
+            return {input_type}
+
+
+def get_function_input(
+    metric_function: Callable,
+    metric_level: eval_schema.MetricLevel,
+    input_object: AnyFunctionObjectInput,
+    context_only: bool,
+) -> AnyFunctionObjectInput | str | dict | list:
+    """Coerce input_object to a type accepted by metric_function at this metric_level.
+
+    Args:
+        metric_function (Callable): Function to invoke with the returned input.
+        metric_level (eval_schema.MetricLevel): The metric level at which metric_function is being invoked.
+        input_object (AnyFunctionObjectInput): The input_object to be coerced, or passed as-is if accepted by metric_function.
+        context_only (bool): Determines how strings and lists are converted. See schema documentation.
+
+    Raises:
+        ValueError: If the function accepts at least one declared type, but
+        it's a type we don't support at all e.g. set or
+        it's a type we don't support at this metric_level.
+
+    Returns:
+        AnyFunctionObjectInput | str | dict | list: The coerced input for metric_function.
+    """
+    if metric_level not in eval_schema.VALID_METRIC_LEVELS:
+        raise ValueError(
+            f"metric_level '{metric_level}' not one of the valid levels: {eval_schema.VALID_METRIC_LEVELS}"
+        )
+    input_type = type(input_object)
+    accepted_parameter_types = get_first_parameter_types(metric_function)
+    if len(accepted_parameter_types) == 0:
+        logger.debug(
+            f"Metric function '{metric_function}' has a first parameter with no type hint, so we can't determine if a type transformation needs to be applied."
+        )
+        return input_object
+    if input_type in accepted_parameter_types:
+        # no transformation necessary; the function accepts the type we already have
+        return input_object
+    elif dict in accepted_parameter_types and metric_level == "ToolCall":
+        return input_object.get_dict_representation()
+    elif list in accepted_parameter_types and metric_level in ["Turn", "Thread"]:
+        if context_only:
+            return input_object.get_context()
+        else:
+            # this is on a single turn - pass in the parsed list
+            return input_object.get_content()
+    elif str in accepted_parameter_types:
+        if metric_level == "ToolCall":
+            raise ValueError(
+                "Functions that accept strings can't be used for tool calls. Accept a dict (or a flexeval.classes.tool_call.ToolCall) instead."
+            )
+        if context_only:
+            # join together all previous turns
+            return join_all_contents_to_string(input_object.get_context())
+        else:
+            # current turn only
+            return join_all_contents_to_string(input_object.get_content())
+    else:
+        # the function accepts at least one declared type, but either:
+        # - it's a type we don't support at all e.g. set
+        # - it's a type we don't support at this metric_level
+        raise ValueError(
+            f"For metric level '{metric_level}', can't coerce '{input_type.__name__}' for function '{metric_function}' to accepted parameter type(s) '{', '.join([type.__name__ for type in accepted_parameter_types])}'."
+        )
+
+
+def join_all_contents_to_string(content: list[dict] | typing.Any) -> str:
+    """
+    content is a list of dictionaries whose keys include 'content'.
+    Returns a string with all the 'content' entries concatenated together,
+    separated by newline.
+    """
+    if isinstance(content, list):
+        content = "\n".join([item.get("content", "") for item in content])
+    return content