python-flexeval 0.1.5__py3-none-any.whl

flexeval/classes/tool_call.py

@@ -0,0 +1,51 @@
+import peewee as pw
+
+from flexeval.classes.base import BaseModel
+from flexeval.classes.dataset import Dataset
+from flexeval.classes.eval_set_run import EvalSetRun
+from flexeval.classes.message import Message
+from flexeval.classes.thread import Thread
+from flexeval.classes.turn import Turn
+
+
+class ToolCall(BaseModel):
+    """Holds a single component of a single turn
+    Corresponds to one output of a node in LangGraph
+    or one Turn in jsonl
+    """
+
+    id = pw.IntegerField(primary_key=True)
+
+    evalsetrun = pw.ForeignKeyField(EvalSetRun, backref="toolcalls")
+    dataset = pw.ForeignKeyField(Dataset, backref="toolcalls")
+    thread = pw.ForeignKeyField(Thread, backref="toolcalls")
+    message = pw.ForeignKeyField(Message, backref="toolcalls")
+    turn = pw.ForeignKeyField(Turn, backref="toolcalls")
+
+    function_name = pw.TextField()
+    args = pw.TextField()
+    additional_kwargs = (
+        pw.TextField()
+    )  # holds any additional info we want to save with a tool call
+    tool_call_id = pw.TextField()
+    response_content = pw.TextField()
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.metrics_to_evaluate = []
+
+    def get_dict_representation(self) -> dict:
+        """
+        Get a dictionary representation of the content of this toolcall,
+        suitable for passing to function metrics that need a standard Python
+        data structure representation of a tool call.
+
+        Keys in returned dictionary are role, content (for the response content),
+        function_name, and args.
+        """
+        return {
+            "role": "toolcall",
+            "content": self.response_content,
+            "args": self.args,
+            "function_name": self.function_name,
+        }

flexeval/classes/turn.py

@@ -0,0 +1,206 @@
+import copy
+import json
+import logging
+
+import peewee as pw
+from playhouse.shortcuts import model_to_dict
+
+from flexeval.classes.base import BaseModel
+from flexeval.classes.dataset import Dataset
+from flexeval.classes.eval_set_run import EvalSetRun
+from flexeval.classes.thread import Thread
+from flexeval.configuration import completion_functions
+
+logger = logging.getLogger(__name__)
+
+
+class Turn(BaseModel):
+    """Holds a single turn
+    In a conversational exchange, each 'Turn' holds information
+    from 1 or more outputs from the same source or role in sequence
+    """
+
+    id = pw.IntegerField(primary_key=True)
+
+    evalsetrun = pw.ForeignKeyField(EvalSetRun, backref="turns")
+    dataset = pw.ForeignKeyField(Dataset, backref="turns")
+    thread = pw.ForeignKeyField(Thread, backref="turns")
+    index_in_thread = pw.IntegerField()
+    role = pw.TextField()
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.metrics_to_evaluate = []
+
+    def get_completion(self):
+        # only get a completion if this is the final turn - we probably don't want to branch from mid-conversation
+        if self.is_final_turn_in_input:
+            completion_config = json.loads(self.evalsetrun.completion_llm)
+            completion_fn_name = completion_config.get("function_name", None)
+            completion_function_kwargs = completion_config.get("kwargs", None)
+
+            # Check if the function name exists in the global namespace and call it
+            if hasattr(completion_functions, completion_fn_name) and hasattr(
+                completion_functions, completion_fn_name
+            ):
+                completion_function = getattr(
+                    completion_functions, completion_fn_name, None
+                )
+                completion = completion_function(
+                    conversation_history=self.get_formatted_prompt(
+                        include_system_prompt=False
+                    ),
+                    **completion_function_kwargs,
+                )
+            else:
+                logger.warning(
+                    "In completion_functions.py: No callable function named "
+                    + completion_fn_name
+                    + " found."
+                )
+                completion = None
+
+            # "completion" will be the output of an existing completion function
+            # We need to make the message object
+            # and probably also a turn object
+
+            # which means it'll have a structure like this
+            # TODO - make this a requirement of the completion functions?
+            #       - make the completion function just return content?
+            # {"choices": [{"message": {"content": "hi", "role": "assistant"}}]}
+            result = model_to_dict(self, exclude=[self.id])
+            result["evalsetrun"] = self.evalsetrun
+            result["dataset"] = self.dataset
+            result["datasetrow"] = self.datasetrow
+            result["turn_number"] = self.turn_number + 1
+            result["role"] = "assistant"
+            result["context"] = self.get_formatted_prompt(include_system_prompt=False)
+            result["is_final_turn_in_input"] = False  # b/c it's not in input
+            self.is_final_turn_in_input = False
+            result["is_completion"] = True
+            result["completion"] = completion
+            result["model"] = completion.get("model", None)
+            result["prompt_tokens"] = completion.get("usage", {}).get(
+                "prompt_tokens", None
+            ) / len(completion.get("choices", [1]))
+            result["completion_tokens"] = completion.get("usage", {}).get(
+                "completion_tokens", None
+            ) / len(
+                completion.get("choices", [1])
+            )  # TODO - use tiktoken here instead?? this will just give the average
+
+            result_list = []
+            for ix, choice in enumerate(completion["choices"]):
+                temp = copy.deepcopy(result)
+                temp["tool_used"] = choice["message"].get("tool_calls", None)
+                temp["turn"] = [choice["message"]]
+                temp["content"] = choice["message"]["content"]
+                temp["completion_number"] = ix + 1
+                result_list.append(temp)
+
+            return result_list
+        else:
+            return None
+
+    def get_context(self, include_system_prompt=False) -> list[dict[str, str]]:
+        """
+        Context is the context of the first message in the turn
+        """
+        context = ""
+        for message in self.messages:
+            context = message.context
+            break
+        context = json.loads(context)
+        if not include_system_prompt:
+            context = [
+                cur_dict for cur_dict in context if cur_dict.get("role") != "system"
+            ]
+        return context
+
+    def get_formatted_prompt(self, include_system_prompt=False):
+        formatted_prompt = []
+        if include_system_prompt:
+            if hasattr(self, "system_prompt"):
+                # TODO this is a bit hacky; it allows for an override of the system prompt by setting it on the Turn object
+                system_prompt = self.system_prompt
+            else:
+                system_prompt = self.thread.system_prompt
+            # if system prompt not available in this thread, we have nothing to include
+            if system_prompt is not None:
+                formatted_prompt.append({"role": "system", "content": system_prompt})
+
+        # context = json.loads(self.context)
+        context = self.get_context()
+
+        if len(context) > 0:
+            formatted_prompt += context  # TODO - we might just want a subset of this
+
+        formatted_prompt += self.get_content()
+        return formatted_prompt
+
+    def get_content(self, include_toolcalls=True, include_tool_messages=True):
+        """
+        Content is a list of dictionaries where each dictionary
+        contains the role and content of messages and tool calls
+        in the turn. Each tool call appears after the message it's
+        associated with. If toolcalls are not desired, pass False
+        to include_toolcalls.
+        """
+        content = []
+        for message in self.messages:
+            if include_tool_messages or message.langgraph_message_type != "ToolMessage":
+                content.append({"role": message.role, "content": message.content})
+            if include_toolcalls:
+                for toolcall in message.toolcalls:
+                    content.append(toolcall.get_dict_representation())
+
+        return content
+
+    def format_input_for_rubric(
+        self, include_system_prompt: bool = False, include_tool_messages: bool = False
+    ):
+        """This is the 'public' method that returns the info for this Turn"""
+        output_minus_completion = ""
+        if include_system_prompt:
+            output_minus_completion.append(
+                {"role": "system", "content": self.system_prompt}
+            )
+
+        for msg in self.get_context():  # input[:-1]:
+            # this outputs user: XYZ, or assistant: 123
+            if len(msg["content"]) > 0 and (
+                include_tool_messages or msg.get("langgraph_role") != "tool"
+            ):
+                output_minus_completion += f"{msg['role']}: {msg['content']}\n"
+        # Including role as prefix to account for both tool and assistant
+        completion = ""
+        for msg in self.get_content(include_tool_messages=include_tool_messages):
+            if len(msg["content"]) > 0:
+                completion += f"{msg['role']}: {msg['content']}\n"
+        # completion = f"{self.get_content()['content']}"
+        output = output_minus_completion + completion
+
+        tool_call_text = ""
+        for tc in self.toolcalls:
+            printme = True
+            # if there's a property called tc.additional_kwargs and it evalues to False...don't print
+            if hasattr(tc, "additional_kwargs"):
+                if not json.loads(tc.additional_kwargs).get("print", False):
+                    printme = False
+            if printme:
+                tool_call_text += """
+
+    Function name: {function_name}
+    Input arguments: {args}
+    Function output: {response_content}
+    """.format(
+                    function_name=tc.function_name,
+                    args=tc.args,
+                    response_content=tc.response_content,
+                )
+
+        # output - all turns
+        # output_minus_completion - all turns except the last
+        # completion - last turn
+        # tool_call_text - all tool calls
+        return output, output_minus_completion, completion, tool_call_text

flexeval/cli.py

@@ -0,0 +1,104 @@
+"""CLI commands."""
+
+import logging
+from pathlib import Path
+from typing import Annotated
+
+import typer
+
+from flexeval import db_utils, log_utils, runner
+from flexeval.io.parsers import yaml_parser
+from flexeval.metrics import access
+
+logger = logging.getLogger(__name__)
+
+
+def global_callback(
+    ctx: typer.Context,
+    log_level: Annotated[
+        log_utils.LogLevel, typer.Option(help="Log level to use.", case_sensitive=False)
+    ] = log_utils.LogLevel.INFO.value,
+):
+    """FlexEval offers a number of CLI commands for convenience."""
+    log_utils.set_up_logging(log_utils.LogLevel.get_logging_constant(log_level.value))
+
+
+app = typer.Typer(callback=global_callback)
+
+
+@app.command(no_args_is_help=True)
+def run(
+    eval_run_yaml_path: Annotated[
+        Path, typer.Argument(help="YAML file specifying the Eval Run.")
+    ],
+):
+    """Run FlexEval using the given YAML Eval Run configuration."""
+    eval_run = yaml_parser.load_eval_run_from_yaml(eval_run_yaml_path)
+    runner.run(eval_run)
+
+
+@app.command(no_args_is_help=True)
+def summarize_metrics(
+    eval_run_yaml_path: Annotated[
+        Path | None,
+        typer.Argument(
+            help="YAML file specifying the Eval Run.",
+            exists=True,
+            dir_okay=False,
+        ),
+    ] = None,
+    database_path: Annotated[
+        Path | None,
+        typer.Option(help="Database path.", exists=True, dir_okay=False),
+    ] = None,
+):
+    """Print a summary of computed metrics."""
+    if eval_run_yaml_path is not None:
+        if database_path is not None:
+            logger.warning(
+                "Ignoring database_path since eval_run_yaml_path is provided."
+            )
+        eval_run = yaml_parser.load_eval_run_from_yaml(eval_run_yaml_path)
+        database_path = eval_run.database_path
+
+    if database_path is None:
+        raise ValueError("Must provide an Eval Run or a database path.")
+    db_utils.initialize_database(database_path)
+    counts = access.count_dict_values(access.get_all_metrics())
+    print("Summary of metric value counts:")
+    for key, counter in counts.items():
+        print("  " + key)
+        for value, count in counter.most_common(5):
+            value = str(value)
+            display_limit = 50
+            if len(value) > display_limit:
+                value = value[: display_limit - 3].replace("\n", "\\n") + "..."
+            print(f"    {value}: {count}")
+
+
+@app.command(no_args_is_help=True)
+def run_eval_by_name(
+    input_data: Annotated[list[Path], typer.Option(help="Input data filepaths.")],
+    database_path: Annotated[Path, typer.Option(help="Output database path.")],
+    eval_name: str,
+    evals_path: Path,
+    config_path: Path,
+    clear_tables: bool = False,
+):
+    """Run an eval by name."""
+    runner.run_from_name_args(
+        input_data,
+        database_path,
+        eval_name,
+        config_path,
+        evals_path,
+        clear_tables=clear_tables,
+    )
+
+
+def main():
+    app()
+
+
+if __name__ == "__main__":
+    main()

flexeval/completions.py

@@ -0,0 +1,147 @@
+"""Completing conversations using LLMs."""
+
+import json
+import logging
+from collections.abc import Callable
+from concurrent.futures import Future, ThreadPoolExecutor, as_completed
+
+from flexeval import classes
+from flexeval.configuration import completion_functions
+from flexeval.schema.eval_schema import CompletionLlm
+from flexeval.schema.evalrun_schema import EvalRun
+
+logger = logging.getLogger(__name__)
+
+
+def get_completion_function(completion_llm: CompletionLlm) -> Callable:
+    """Identify a completion function given the completion LLM configuration.
+
+    Args:
+        completion_llm (CompletionLlm): The description of the function to retrieve.
+
+    Raises:
+        ValueError: If not found in :mod:`~flexeval.configuration.completion_functions` or in globals by function_name.
+
+    Returns:
+        Callable: The completion function.
+    """
+    # Check if the function name exists in the global namespace and call it
+    if hasattr(completion_functions, completion_llm.function_name):
+        completion_function = getattr(
+            completion_functions, completion_llm.function_name
+        )
+        return completion_function
+    if completion_llm.function_name in globals() and callable(
+        globals()[completion_llm.function_name]
+    ):
+        # TODO probably don't allow this by default, and also offer a way to specify other places where completion_functions can live.
+        logger.debug("Found function in globals(), which could be trouble.")
+        return globals()[completion_llm.function_name]
+    else:
+        raise ValueError(
+            "No completion function named " + completion_llm.function_name + " found."
+        )
+
+
+def get_completion(turn: classes.turn.Turn, completion_llm: CompletionLlm):
+    completion_function = get_completion_function(completion_llm)
+    conversation_history = turn.get_formatted_prompt(
+        include_system_prompt=completion_llm.include_system_prompt
+    )
+    completion = completion_function(
+        conversation_history=conversation_history,
+        **completion_llm.kwargs,
+    )
+    return completion
+
+
+def get_completions(eval_run: EvalRun, evalsetrun: classes.eval_set_run.EvalSetRun):
+    n_workers = eval_run.config.max_workers
+    if n_workers == 1:
+        for thread in evalsetrun.threads:
+            # select last turn in thread
+            if len(thread.turns) == 0:
+                continue
+            turn = (
+                thread.turns.select()
+                .order_by(classes.turn.Turn.index_in_thread.desc())
+                .first()
+            )
+            # TODO handle exceptions appropriately (#58)
+            completion = get_completion(turn, eval_run.eval.completion_llm)
+            if completion is None:
+                continue
+            save_completion(completion, turn, evalsetrun, eval_run)
+    else:
+        with ThreadPoolExecutor(max_workers=n_workers) as executor:
+            futures: dict[Future, classes.turn.Turn] = {}
+            for thread in evalsetrun.threads:
+                if len(thread.turns) == 0:
+                    continue
+                turn = (
+                    thread.turns.select()
+                    .order_by(classes.turn.Turn.index_in_thread.desc())
+                    .first()
+                )
+                future = executor.submit(
+                    get_completion, turn, eval_run.eval.completion_llm
+                )
+                futures[future] = turn
+
+            for future in as_completed(futures):
+                turn = futures[future]
+                completion = future.result()
+                if completion is None:
+                    continue
+                save_completion(completion, turn, evalsetrun, eval_run)
+
+
+def save_completion(
+    completion: dict,
+    turn: classes.turn.Turn,
+    evalsetrun: classes.eval_set_run.EvalSetRun,
+    eval_run: EvalRun,
+):
+    new_message_completions = completion["choices"]
+    if len(new_message_completions) > 1:
+        logger.warning(
+            "We don't yet support multiple completions, using just the first one."
+        )
+    new_message_completion = new_message_completions[0]["message"]
+    if turn.role == "assistant":
+        # don't create a new Turn, because this completion is a continuation of an existing assistant turn
+        new_turn = turn
+    else:
+        new_turn = classes.turn.Turn.create(
+            evalsetrun=evalsetrun,
+            dataset=turn.dataset,
+            thread=turn.thread,
+            index_in_thread=turn.index_in_thread + 1,
+            role=new_message_completion["role"],
+        )
+    prev_message = (
+        turn.messages.select()
+        .order_by(classes.message.Message.index_in_thread.desc())
+        .first()
+    )
+    new_message_context = prev_message.get_context()
+    new_message_context.append(
+        {"role": prev_message.role, "content": prev_message.content}
+    )
+    classes.message.Message.create(
+        evalsetrun=evalsetrun,
+        dataset=turn.dataset,
+        thread=turn.thread,
+        turn=new_turn,
+        index_in_thread=prev_message.index_in_thread + 1,
+        role=new_message_completion["role"],
+        content=new_message_completion["content"],
+        context=json.dumps(new_message_context),
+        system_prompt=prev_message.system_prompt,
+        is_flexeval_completion=True,
+        # TODO I have no idea what model_name is supposed to be, and the completion function name doesn't seem that useful
+        model_name=eval_run.eval.completion_llm.function_name,
+        prompt_tokens=completion.get("usage", {}).get("prompt_tokens", None),
+        completion_tokens=completion.get("usage", {}).get("completion_tokens", None),
+    )
+    # TODO also save any toolcalls