python-flexeval 0.1.5__py3-none-any.whl

flexeval/__init__.py

@@ -0,0 +1,11 @@
+"""FlexEval is a Python package for designing custom metrics, completion functions, and LLM-graded rubrics for evaluating the behavior of LLM-powered systems.
+
+This top-level import exposes the :func:`~flexeval.runner.run` method."""
+
+from flexeval import metrics
+from flexeval.runner import run
+
+__all__ = [
+    "metrics",
+    "run",
+]

flexeval/__main__.py

@@ -0,0 +1,11 @@
+"""Pass-through to the CLI."""
+
+from flexeval import cli
+
+
+def main():
+    cli.main()
+
+
+if __name__ == "__main__":
+    main()

flexeval/classes/__init__.py

@@ -0,0 +1,15 @@
+"""Peewee classes used for saving the results of a FlexEval run.
+
+See :mod:`peewee` for more information on the capabilities of these objects."""
+
+from . import dataset, eval_set_run, message, metric, thread, tool_call, turn
+
+__all__ = [
+    "dataset",
+    "eval_set_run",
+    "message",
+    "metric",
+    "thread",
+    "tool_call",
+    "turn",
+]

flexeval/classes/base.py

@@ -0,0 +1,32 @@
+import peewee as pw
+from playhouse.shortcuts import ThreadSafeDatabaseMetadata
+from playhouse.sqliteq import SqliteQueueDatabase
+
+
+def create_sqlite_database(
+    database_path: str | None = None, use_queue_db: bool = False
+) -> pw.SqliteDatabase:
+    if use_queue_db:
+        return SqliteQueueDatabase(
+            database_path,
+            use_gevent=False,
+            autostart=False,
+            results_timeout=5.0,
+            queue_max_size=64,  # Max. # of pending writes that can accumulate
+            pragmas={"journal_mode": "wal"},  # use Write-ahead Logging
+        )
+    return pw.SqliteDatabase(
+        database_path,
+        pragmas={"journal_mode": "wal"},  # use Write-ahead Logging
+    )
+
+
+database = create_sqlite_database()
+
+
+class BaseModel(pw.Model):
+    """Peewee base class for all FlexEval database models."""
+
+    class Meta:
+        model_metadata_class = ThreadSafeDatabaseMetadata
+        database = database

flexeval/classes/dataset.py

@@ -0,0 +1,82 @@
+import os.path
+
+import peewee as pw
+
+from flexeval.classes.base import BaseModel
+from flexeval.classes.eval_set_run import EvalSetRun
+
+
+class Dataset(BaseModel):
+    """Holds a dataset, e.g. a jsonl file"""
+
+    id = pw.IntegerField(primary_key=True)
+    evalsetrun = pw.ForeignKeyField(EvalSetRun, backref="datasets")
+    filename = pw.TextField()
+    datatype = pw.TextField(null=True)
+    contents = pw.TextField(null=True)  # raw contents
+
+    max_n_conversation_threads = pw.IntegerField(null=True)
+    nb_evaluations_per_thread = pw.IntegerField(null=True, default=1)
+
+    # In line with LangGraph expectations, we assume n=1 for all outputs of LLMs
+    # However, each node can append list with length 2+ to the message queue
+
+    # Thread - conversation
+    # Turn - adjacent messages from the same agent
+    # Message -
+    #   role - human or ai, user or assistant
+    #   text - empty string or non-empty
+    #   list of 0+ Tool Calls
+    #   post-processing - add a turn_id
+    #   additional_kwargs JSON
+    # ToolUse
+    #   foreign keys to "invoker" message and "function output" message
+    #   message that invoked it - foreign key
+    #   parameters of the input
+    #   result of tool call
+    # Metric
+    #   granularity type
+    #   foreign key to the object
+
+    # **each entry from LangGraph is a LIST of completions - usually with length 1
+
+    # Completion - has one bit of text content, and 0+ ToolCalls
+    # ToolCall - tool call (and response!) associated with the completion
+    #   completion_id
+    #   message_id
+    #   turn_id
+
+    def load_data(self):
+        from flexeval import (
+            data_loader,
+        )  # Local import as this needs to happen after the module is fully loaded
+
+        if self.filename.endswith(".jsonl"):
+            self.datatype = "json"
+            data_loader.load_jsonl(
+                dataset=self,
+                filename=self.filename,
+                max_n_conversation_threads=self.max_n_conversation_threads,
+                nb_evaluations_per_thread=self.nb_evaluations_per_thread,
+            )
+
+        elif is_sqlite_file(self.filename):
+            self.datatype = "sqlite"
+            data_loader.load_langgraph_sqlite(
+                dataset=self,
+                filename=self.filename,
+                max_n_conversation_threads=self.max_n_conversation_threads,
+                nb_evaluations_per_thread=self.nb_evaluations_per_thread,
+            )
+        else:
+            raise ValueError(
+                f"Unsupported format '{os.path.splitext(self.filename)[-1]}'. Each Data File must be either a jsonl or sqlite file. You provided the file: '{self.filename}'"
+            )
+
+
+def is_sqlite_file(filepath):
+    # Open the file in binary mode
+    with open(filepath, "rb") as file:
+        header = file.read(16)
+    # Check if the header matches the SQLite format header
+    return header == b"SQLite format 3\x00"

flexeval/classes/eval_runner.py

@@ -0,0 +1,158 @@
+import logging
+from datetime import datetime
+from pathlib import Path
+
+import dotenv
+from peewee import SqliteDatabase
+
+from flexeval import db_utils, dependency_graph
+from flexeval.schema import EvalRun
+
+logger = logging.getLogger(__name__)
+
+
+class EvalRunner:
+    """Class for maintaining database connection, logs, and run state
+    Does not need to write anything to database itself.
+    """
+
+    database: SqliteDatabase
+
+    def __init__(
+        self,
+        evalrun: EvalRun,
+    ):
+        self.evalrun: EvalRun = evalrun
+
+        self.initialize_logger()
+        self.add_file_logger()
+        self.load_env()
+        self.initialize_database()
+        self.load_evaluation_settings()
+
+    def initialize_logger(self, add_stream_handler: bool = False):
+        """Configure the logger for this class.
+
+        Args:
+            add_stream_handler (bool, optional): If True, will add a stream handler at the INFO level. Defaults to False.
+        """
+        self.logger = logging.getLogger("FlexEval")
+        self.logger.setLevel(logging.DEBUG)
+
+        if add_stream_handler:
+            # TODO this stream handler logic should probably be removed
+            # Create a console handler for lower level messages to output to console
+            ch = logging.StreamHandler()
+            ch.setLevel(logging.INFO)
+
+            # Create a formatter and set it for the handlers
+            formatter = logging.Formatter(
+                "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+            )
+            ch.setFormatter(formatter)
+
+            # Add the handlers to the logger
+            self.logger.addHandler(ch)
+
+    def add_file_logger(self):
+        if self.evalrun.config.logs_path is None:
+            logger.info("No log path specified, so not doing any file logging.")
+            return
+        logs_path = self.evalrun.config.logs_path
+        if logs_path.is_file():
+            raise ValueError(
+                f"Config logs_path expects a directory, but was set to existing file '{logs_path}'."
+            )
+        elif not logs_path.exists():
+            if logs_path.suffix != "":
+                logger.warning(
+                    f"Creating logs_path '{logs_path}' as a directory, despite apparent suffix '{logs_path.suffix}'."
+                )
+            logs_path.mkdir(parents=True, exist_ok=True)
+
+        # Get the current date to use in the filename
+        current_date = datetime.now().strftime("%Y-%m-%d")
+
+        # Create a file handler that logs debug and higher level messages to a date-based file
+        log_filepath = logs_path / f"{current_date}_{self.evalrun.eval.name}.log"
+        fh = logging.FileHandler(log_filepath)
+        fh.setLevel(logging.DEBUG)
+
+        # Create a formatter and set it for the handlers
+        formatter = logging.Formatter(
+            "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+        )
+        fh.setFormatter(formatter)
+        self.logger.addHandler(fh)
+        self.logger.info(f"Started logging to log file '{log_filepath}'.")
+
+    def load_env(self):
+        env_filepath = self.evalrun.config.env_filepath
+        if env_filepath is not None and str(env_filepath).strip() != "":
+            if not env_filepath.exists():
+                raise ValueError(
+                    f"Environment file not present at configured path '{env_filepath}'."
+                )
+            dotenv.load_dotenv(env_filepath, verbose=True)
+            self.logger.debug(f"Finished loading .env file from '{env_filepath}'.")
+        else:
+            self.logger.debug(
+                f"Skipping .env file loading as config env_filepath is '{env_filepath}'."
+            )
+
+    def get_database_path(self) -> Path:
+        return self.evalrun.database_path
+
+    def initialize_database(self):
+        """Initializes database and tables. If config.clear_tables, then current contents of tables are dropped."""
+        db_utils.initialize_database(
+            self.evalrun.database_path, clear_tables=self.evalrun.config.clear_tables
+        )
+
+    def load_evaluation_settings(self):
+        """This function parses our eval suite and puts it in the data structure we'll need
+        for easy use at run-time
+        """
+        # if the current eval has a 'config' entry, overwrite configuration options with its entries
+        if (
+            self.evalrun.eval.model_extra is not None
+            and len(self.evalrun.eval.model_extra) > 0
+        ):
+            model_extra = self.evalrun.eval.model_extra
+            self.logger.debug(
+                f"Extra configuration keys provided in eval: {list(model_extra.keys())}"
+            )
+            for field_name in model_extra.keys():
+                if hasattr(self.evalrun.config, field_name):
+                    old_value = getattr(self.evalrun.config, field_name)
+                    new_value = model_extra[field_name]
+                    self.logger.info(
+                        f"Updating configuration setting: '{field_name}'='{new_value}' (old='{old_value}')"
+                    )
+                    setattr(self.evalrun.config, field_name, new_value)
+                else:
+                    self.logger.warning(
+                        f"Unknown configuration field '{field_name}' was ignored."
+                    )
+
+        # TODO verify that applying defaults is done solely by pydantic and this step is no longer necessary
+        # apply defaults to the schema
+        # self.eval = apply_defaults(schema=target_schema, data=self.eval)
+
+        # convert into graph structure
+        self.metrics_graph_ordered_list = dependency_graph.create_metrics_graph(
+            self.evalrun.eval.metrics
+        )
+        # validate: completion function defined
+        if len(self.metrics_graph_ordered_list) > 0:
+            if self.evalrun.eval.grader_llm is None:
+                self.logger.warning(
+                    f"'{len(self.metrics_graph_ordered_list)}' metrics defined, but no grader LLM defined."
+                )
+
+    def shutdown_logging(self):
+        # remove logging handler so we don't get repeat logs if we call run() twice
+        handlers = self.logger.handlers[:]
+        for handler in handlers:
+            handler.close()
+            self.logger.removeHandler(handler)

flexeval/classes/eval_set_run.py

@@ -0,0 +1,32 @@
+import json
+from datetime import datetime
+
+import peewee as pw
+
+from flexeval.classes.base import BaseModel
+
+
+class EvalSetRun(BaseModel):
+    """Class for running set of evaluations"""
+
+    id = pw.IntegerField(primary_key=True)
+    name = pw.CharField(null=True)
+    notes = pw.TextField(null=True)
+    dataset_files = pw.TextField()  # JSON string
+    metrics = pw.TextField()
+    metrics_graph_ordered_list = pw.TextField()
+    do_completion = pw.BooleanField()
+    completion_llm = pw.TextField(null=True)  # JSON string
+    grader_llm = pw.TextField(null=True)  # JSON string
+    model_name = pw.TextField(null=True)  # JSON string
+    success = pw.BooleanField(null=True)
+    rubrics = pw.TextField(null=True)
+    timestamp = pw.DateTimeField(
+        default=datetime.now
+    )  # Automatically set to current date and time
+
+    def get_datasets(self) -> list[str]:
+        # TODO Turn these into DataSource instances instead, returning list[DataSource]
+        temp = json.loads(self.dataset_files)
+        assert isinstance(temp, list), "The `data` entry in evals.yaml must be a list."
+        return temp

flexeval/classes/message.py

@@ -0,0 +1,183 @@
+import copy
+import json
+import logging
+
+import peewee as pw
+from playhouse.shortcuts import model_to_dict
+
+from flexeval.classes.base import BaseModel
+from flexeval.classes.dataset import Dataset
+from flexeval.classes.eval_set_run import EvalSetRun
+from flexeval.classes.thread import Thread
+from flexeval.classes.turn import Turn
+from flexeval.configuration import completion_functions
+
+logger = logging.getLogger(__name__)
+
+
+class Message(BaseModel):
+    """Holds a single component of a single turn
+    Corresponds to one output of a node in LangGraph
+    or one Turn in jsonl
+    """
+
+    id = pw.IntegerField(primary_key=True)
+
+    evalsetrun = pw.ForeignKeyField(EvalSetRun, backref="messages")
+    dataset = pw.ForeignKeyField(Dataset, backref="messages")
+    thread = pw.ForeignKeyField(Thread, backref="messages")
+    index_in_thread = pw.IntegerField()
+    # must be null=True because we're adding it after create()
+    turn = pw.ForeignKeyField(Turn, null=True, backref="messages")
+
+    role = pw.TextField()  # user or assistant - 'tools' are counted as assistants
+    content = pw.TextField()
+    context = pw.TextField(null=True)  # Previous messages
+
+    # helpers
+    system_prompt = pw.TextField(null=True)
+    is_flexeval_completion = pw.BooleanField(null=True)
+    is_final_turn_in_input = pw.BooleanField(null=True)
+    langgraph_print = pw.TextField(null=True)
+
+    # language model stats
+    tool_callslanggraph_print = pw.TextField(null=True)
+    tool_call_ids = pw.TextField(null=True)
+    n_tool_calls = pw.IntegerField(null=True)
+    prompt_tokens = pw.IntegerField(null=True)
+    completion_tokens = pw.IntegerField(null=True)
+    model_name = pw.TextField(null=True)
+
+    # langgraph metadata
+    langgraph_ts = pw.TextField(null=True)
+    langgraph_step = pw.IntegerField(null=True)
+    langgraph_thread_id = pw.TextField(null=True)
+    langgraph_checkpoint_id = pw.TextField(null=True)
+    langgraph_parent_checkpoint_id = pw.TextField(null=True)
+    langgraph_node = pw.TextField(null=True)
+    langgraph_message_type = pw.TextField(null=True)
+    langgraph_type = pw.TextField(null=True)
+    langgraph_invocation_id = pw.TextField(null=True)
+    # putting these at the end so the database is easier to browse
+    langgraph_checkpoint = pw.TextField(null=True)
+    langgraph_metadata = pw.TextField(null=True)
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.metrics_to_evaluate = []
+
+    def get_completion(self, include_system_prompt=False):
+        # only get a completion if this is the final turn - we probably don't want to branch from mid-conversation
+        if self.is_final_turn_in_input:
+            completion_config = json.loads(self.evalsetrun.completion_llm)
+            completion_fn_name = completion_config.get("function_name", None)
+            completion_function_kwargs = completion_config.get("kwargs", None)
+
+            # Check if the function name exists in the global namespace and call it
+
+            if hasattr(completion_functions, completion_fn_name) and hasattr(
+                completion_functions, completion_fn_name
+            ):
+                completion_function = getattr(
+                    completion_functions, completion_fn_name, None
+                )
+                completion = completion_function(
+                    conversation_history=self.get_formatted_prompt(
+                        include_system_prompt=False
+                    ),
+                    **completion_function_kwargs,
+                )
+            else:
+                logger.warning(
+                    "In completion_functions.py: No callable function named "
+                    + completion_fn_name
+                    + " found."
+                )
+                completion = None
+
+            # "completion" will be the output of an existing completion function
+            # which generally means it'll have a structure like this
+            # {"choices": [{"message": {"content": "hi", "role": "assistant"}}]}
+            result = model_to_dict(self, exclude=[self.id])
+            result["evalsetrun"] = self.evalsetrun
+            result["dataset"] = self.dataset
+            result["datasetrow"] = self.datasetrow
+            result["turn_number"] = self.turn_number + 1
+            result["role"] = "assistant"
+            result["context"] = self.get_formatted_prompt(include_system_prompt=False)
+            result["is_final_turn_in_input"] = False  # b/c it's not in input
+            self.is_final_turn_in_input = False
+            result["is_completion"] = True
+            result["completion"] = completion
+            result["model"] = completion.get("model", None)
+            result["prompt_tokens"] = completion.get("usage", {}).get(
+                "prompt_tokens", None
+            ) / len(completion.get("choices", [1]))
+            result["completion_tokens"] = completion.get("usage", {}).get(
+                "completion_tokens", None
+            ) / len(
+                completion.get("choices", [1])
+            )  # TODO - use tiktoken here instead?? this will just give the average
+
+            result_list = []
+            for ix, choice in enumerate(completion["choices"]):
+                temp = copy.deepcopy(result)
+                temp["tool_used"] = choice["message"].get("tool_calls", None)
+                temp["turn"] = [choice["message"]]
+                temp["content"] = choice["message"]["content"]
+                temp["completion_number"] = ix + 1
+                result_list.append(temp)
+
+            return result_list
+        else:
+            return None
+
+    def get_formatted_prompt(self, include_system_prompt=False) -> list[dict[str, str]]:
+        formatted_prompt = []
+        if include_system_prompt:
+            formatted_prompt.append({"role": "system", "content": self.system_prompt})
+        context = json.loads(self.context)
+        if len(context) > 0:
+            formatted_prompt += context  # TODO - we might just want a subset of this
+        formatted_prompt.append({"role": self.role, "content": self.content})
+        # for t in json.loads(self.turn):
+        #     formatted_prompt.append({"role": t["role"], "content": t["content"]})
+        return formatted_prompt
+
+    def format_input_for_rubric(self):
+        input = self.get_formatted_prompt()
+        output_minus_completion = ""
+        for i in input[:-1]:
+            output_minus_completion += f"{i['role']}: {i['content']}\n"
+        completion = f"{input[-1]['content']}"
+        output = output_minus_completion + completion
+
+        tool_call_text = ""
+        for tc in self.toolcalls:
+            tool_call_text += """
+
+Function name: {function_name}
+Input arguments: {args}
+Function output: {response_content}
+""".format(
+                function_name=tc.function_name,
+                args=tc.args,
+                response_content=tc.response_content,
+            )
+
+        # output - all turns
+        # output_minus_completion - all turns except the last
+        # completion - last turn
+        # tool_call_text - all tool calls
+        return output, output_minus_completion, completion, tool_call_text
+
+    def get_content(self) -> str:
+        return self.content
+
+    def get_context(self, include_system_prompt=False) -> list[dict[str, str]]:
+        context = json.loads(self.context)
+        if not include_system_prompt:
+            context = [
+                cur_dict for cur_dict in context if cur_dict.get("role") != "system"
+            ]
+        return context

flexeval/classes/metric.py

@@ -0,0 +1,55 @@
+import peewee as pw
+
+from flexeval.classes.base import BaseModel
+from flexeval.classes.dataset import Dataset
+from flexeval.classes.eval_set_run import EvalSetRun
+from flexeval.classes.message import Message
+from flexeval.classes.thread import Thread
+from flexeval.classes.tool_call import ToolCall
+from flexeval.classes.turn import Turn
+
+
+class Metric(BaseModel):
+    """Holds a single metric/property computed based one just ONE turn"""
+
+    id = pw.IntegerField(primary_key=True)
+
+    evalsetrun = pw.ForeignKeyField(EvalSetRun, backref="metrics_list")
+    dataset = pw.ForeignKeyField(Dataset, backref="metrics_list")
+    thread = pw.ForeignKeyField(Thread, backref="metrics_list")
+    turn = pw.ForeignKeyField(
+        Turn, null=True, backref="metrics_list"
+    )  # Only defined for Turn metrics
+    message = pw.ForeignKeyField(
+        Message, null=True, backref="metrics_list"
+    )  # Only defined for Message metrics
+    toolcall = pw.ForeignKeyField(
+        ToolCall, null=True, backref="metrics_list"
+    )  # Only defined for ToolCall metrics
+
+    evaluation_name = pw.TextField()
+    evaluation_type = pw.TextField()
+    metric_name = pw.TextField()
+    # metric_type = pw.TextField() # TODO: Some parts of the code use "metric_tye" and others use "evaluation_type" - choose one for consistency
+    metric_level = pw.TextField()
+    # TODO we may want to consider adding a secondary metric_nonnumeric_value field to support non-numeric functions and rubrics
+    metric_value = pw.FloatField(
+        null=True
+    )  # necessary if rubric result is INVALID or e.g. latency doesn't apply to the very first message
+    kwargs = pw.TextField()
+    # context_only allows us to create another kind of dependency
+    # where we can quantify something about the previous conversation
+    # and then use that quantity in a downstream analysis
+    # e.g. 'would a plot be pedagogically appropriate here' is really a question about the PAST of the conversation
+    #      NOTE: but we have gotten rid of context_only for rubrics, where only {context} is used so technically here 'context_only' is False
+    # or 'was the conversation ever flagged by the moderation api' would be a question about the previous turns that might
+    #    allow to have better context for the properties of this turn
+    # context_only = pw.BooleanField(default=False)
+    source = pw.TextField()  # TODO - make another table for this? But maybe not, because this also contains filled-in rubrics
+    depends_on = pw.TextField()
+    rubric_prompt = pw.TextField(null=True)
+    rubric_completion = pw.TextField(null=True)
+    rubric_model = pw.TextField(null=True)
+    rubric_completion_tokens = pw.IntegerField(null=True)
+    rubric_prompt_tokens = pw.IntegerField(null=True)
+    rubric_score = pw.TextField(null=True)

flexeval/classes/thread.py

@@ -0,0 +1,79 @@
+import peewee as pw
+
+from flexeval.classes.base import BaseModel
+from flexeval.classes.dataset import Dataset
+from flexeval.classes.eval_set_run import EvalSetRun
+
+
+class Thread(BaseModel):
+    """Class for holding a single thread / conversation
+    This corresponds to a single row in a jsonl file
+    or a single 'thread_id' in a langgraph checkpoint database"""
+
+    id = pw.IntegerField(primary_key=True)
+    dataset = pw.ForeignKeyField(Dataset, backref="threads")
+    evalsetrun = pw.ForeignKeyField(EvalSetRun, backref="threads")
+
+    langgraph_thread_id = pw.TextField(null=True)
+    eval_run_thread_id = pw.TextField(null=True)
+    jsonl_thread_id = pw.TextField(null=True)
+
+    system_prompt = pw.TextField(null=True)
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.metrics_to_evaluate = []
+
+    # TODO - test this!
+    def format_input_for_rubric(self):
+        input = self.get_formatted_prompt()
+        output_minus_completion = ""
+        for i in input[:-1]:
+            output_minus_completion += f"{i['role']}: {i['content']}\n"
+        completion = f"{input[-1]['role']}: {input[-1]['content']}\n"
+        output = output_minus_completion + completion
+
+        tool_call_text = ""
+        for tc in self.toolcalls:
+            tool_call_text += """
+
+Function name: {function_name}
+Input arguments: {args}
+Function output: {response_content}
+""".format(
+                function_name=tc.function_name,
+                args=tc.args,
+                response_content=tc.response_content,
+            )
+
+        # output - all turns
+        # output_minus_completion - all turns except the last
+        # completion - last turn
+        # tool_call_text - all tool calls
+        return output, output_minus_completion, completion, tool_call_text
+
+    def get_formatted_prompt(self, include_system_prompt=False):
+        formatted_prompt = []
+        if include_system_prompt:
+            formatted_prompt.append({"role": "system", "content": self.system_prompt})
+
+        formatted_prompt += self.get_content()
+
+        return formatted_prompt
+
+    def get_content(self, include_toolcalls=True):
+        """
+        Content is a list of dictionaries where each dictionary
+        contains the role and content of messages and tool calls
+        in the turn. Each tool call appears after the message it's
+        associated with. If toolcalls are not desired, pass False
+        to include_toolcalls.
+        """
+        content = []
+        for message in self.messages:
+            content.append({"role": message.role, "content": message.content})
+            if include_toolcalls:
+                for toolcall in message.toolcalls:
+                    content.append(toolcall.get_dict_representation())
+
+        return content