python-flexeval 0.1.5__py3-none-any.whl

flexeval/helpers.py

@@ -0,0 +1,52 @@
+"""Generic utility functions."""
+
+import datetime
+import hashlib
+
+import networkx as nx
+
+
+def generate_hash():
+    """Create a random 8-digit id"""
+    # Create a new SHA-256 hash object
+    hash_object = hashlib.sha256()
+
+    # Update the hash object with the bytes of the string
+    hash_object.update(datetime.datetime.now().isoformat().encode())
+
+    # Get the hexadecimal digest of the hash
+    full_hash = hash_object.hexdigest()
+
+    # Return the first 8 digits of the hash
+    return full_hash[:8]
+
+
+def visualize_graph(graph: nx.DiGraph, output_path: str | None = None):
+    """Visualize graphs produced by :class:`~flexeval.compute_metrics.MetricGraphBuilder`.
+
+    Args:
+        graph (nx.DiGraph): The graph
+        output_path (str | None, optional): If not None, will save the graph as an image using :meth:`matplotlib.pyplot.Figure.savefig`.
+
+    Raises:
+        ImportError: If matplotlib is not installed.
+    """
+    try:
+        import matplotlib.pyplot as plt
+    except ImportError:
+        raise ImportError("matplotlib must be installed to use this helper function.")
+    fig, ax = plt.subplots(1, 1, figsize=(12, 5))
+    pos = nx.spring_layout(graph)
+    nx.draw(graph, ax=ax, pos=pos)
+    nx.draw_networkx_labels(
+        graph,
+        font_size=8,
+        ax=ax,
+        pos=pos,
+        labels={
+            om: f"{om.object.__class__.__name__} {om.object.id}\n{om.metric.get('id')}(l={om.metric.get('metric_level')},r={om.metric.get('kwargs', {}).get('response')})"
+            for om in graph
+        },
+    )
+    if output_path is not None:
+        fig.savefig(output_path)

flexeval/io/__init__.py

@@ -0,0 +1 @@
+"""Input/output utilities, primarily for reading and writing the :mod:`~flexeval.schema` objects."""

flexeval/io/parsers/yaml_parser.py

@@ -0,0 +1,69 @@
+import logging
+from pathlib import Path
+from typing import Type, TypeVar
+
+import pydantic
+import yaml
+
+from flexeval.schema import Config, Eval, EvalRun
+
+logger = logging.getLogger(__name__)
+T = TypeVar("T", bound=pydantic.BaseModel)
+
+
+def load_config_from_yaml(
+    filename: Path | str,
+) -> Config:
+    return load_model_from_yaml(filename, Config)
+
+
+def load_evals_from_yaml(
+    filename: Path | str,
+) -> dict[str, Eval]:
+    return load_models_from_yaml_filepath(filename, Eval)
+
+
+def load_eval_run_from_yaml(
+    filename: Path | str,
+) -> EvalRun:
+    return load_model_from_yaml(filename, EvalRun)
+
+
+def load_models_from_yaml_filepath(
+    filename: Path | str,
+    model_type: Type[T],
+) -> dict[str, T]:
+    try:
+        with open(filename) as file:
+            return load_models_from_yaml_stream(file, model_type)
+    except (OSError, ValueError) as ex:
+        raise ValueError(
+            f"Failed to load '{filename}' as a list of '{model_type.__name__}' models: {ex}"
+        )
+
+
+def load_models_from_yaml_stream(
+    stream,
+    model_type: Type[T],
+) -> dict[str, T]:
+    try:
+        contents = yaml.safe_load(stream)
+        return {key: model_type(**value) for key, value in contents.items()}
+    except (OSError, yaml.YAMLError, pydantic.ValidationError) as ex:
+        raise ValueError(
+            f"Failed to load YAML stream as a list of '{model_type.__name__}' models: {ex}"
+        )
+
+
+def load_model_from_yaml(
+    filename: Path | str,
+    model_type: Type[T],
+) -> T:
+    try:
+        with open(filename) as file:
+            contents = yaml.safe_load(file)
+            return model_type(**contents)
+    except (OSError, yaml.YAMLError, pydantic.ValidationError) as ex:
+        raise ValueError(
+            f"Failed to load '{filename}' as a '{model_type.__name__}': {ex}"
+        )

flexeval/log_utils.py

@@ -0,0 +1,34 @@
+"""Logging utilities."""
+
+import enum
+import logging
+
+
+class LogLevel(str, enum.Enum):
+    CRITICAL = "critical"
+    ERROR = "error"
+    WARNING = "warning"
+    INFO = "info"
+    DEBUG = "debug"
+
+    @classmethod
+    def get_logging_constant(cls, level_str: str) -> int:
+        if level_str == cls.CRITICAL.value:
+            return logging.CRITICAL
+        elif level_str == cls.ERROR.value:
+            return logging.ERROR
+        elif level_str == cls.WARNING.value:
+            return logging.WARNING
+        elif level_str == cls.INFO.value:
+            return logging.INFO
+        elif level_str == cls.DEBUG.value:
+            return logging.DEBUG
+
+
+def set_up_logging(log_level: int = logging.INFO):
+    # set up logging
+    logging.basicConfig(
+        level=log_level,
+        format="%(asctime)s [%(levelname)s] %(filename)s:%(lineno)d (%(funcName)s) - %(message)s",
+        datefmt="%Y-%m-%d %H:%M:%S",
+    )

flexeval/metrics/__init__.py

@@ -0,0 +1,8 @@
+"""Utility functions for accessing metrics."""
+
+from flexeval.metrics import access, save
+
+__all__ = [
+    "access",
+    "save",
+]

flexeval/metrics/access.py

@@ -0,0 +1,28 @@
+from collections import Counter
+
+from flexeval.classes import metric
+
+
+def count_dict_values(lst: list[dict]) -> dict[str, Counter]:
+    """Convenience function for counting key values.
+
+    Args:
+        lst (list[dict]): List of dictionaries.
+
+    Returns:
+        dict[str, Counter]: counter for each key that appears in the dicts in lst.
+    """
+    counts = {}
+    for d in lst:
+        for k, v in d.items():
+            if k not in counts:
+                counts[k] = Counter()
+            counts[k][v] += 1
+    return counts
+
+
+def get_all_metrics() -> list:
+    results = []
+    for m in metric.Metric.select():
+        results.append(m.__data__.copy())
+    return results

flexeval/metrics/save.py

@@ -0,0 +1,39 @@
+import json
+from typing import Iterable
+
+from flexeval.classes.metric import Metric
+
+
+def save_metrics(metrics: Iterable[Metric]):
+    for metric in metrics:
+        # TODO - speed this up somehow
+        thread = metric.get("thread")
+        if thread is None:
+            thread = metric[metric["metric_level"].lower()].thread
+        Metric.create(
+            message=metric.get("message", None),
+            turn=metric.get("turn", None),
+            toolcall=metric.get("toolcall", None),
+            evalsetrun=metric[
+                metric["metric_level"].lower()
+            ].evalsetrun,  # metric["turn"].evalsetrun,
+            dataset=metric[
+                metric["metric_level"].lower()
+            ].dataset,  # metric["turn"].dataset,
+            thread=thread,
+            evaluation_name=metric["evaluation_name"],
+            evaluation_type=metric["evaluation_type"],
+            metric_name=metric["metric_name"],
+            metric_value=metric["metric_value"],
+            metric_level=metric["metric_level"],
+            kwargs=metric["kwargs"],
+            depends_on=json.dumps(metric["depends_on"]),
+            context_only=metric.get("context_only", False),
+            source=metric["source"],
+            rubric_prompt=metric.get("rubric_prompt", None),
+            rubric_completion=metric.get("rubric_completion", None),
+            rubric_model=metric.get("rubric_model", None),
+            rubric_completion_tokens=metric.get("rubric_completion_tokens", None),
+            rubric_prompt_tokens=metric.get("rubric_prompt_tokens", None),
+            rubric_score=metric.get("rubric_score", None),
+        )

flexeval/rubric.py

@@ -0,0 +1,62 @@
+"""Rubric metric IO utilities. Should maybe be moved to :mod:`~flexeval.io`."""
+
+import importlib.resources
+import logging
+from pathlib import Path
+
+import flexeval.configuration
+from flexeval.io.parsers import yaml_parser
+from flexeval.schema import Rubric, RubricsCollection
+
+logger = logging.getLogger(__name__)
+
+
+default_rubric_collection = (
+    None  # don't access directly, use load_default_rubric_metrics
+)
+
+
+def load_rubrics_from_yaml_file(path: Path) -> dict[str, Rubric]:
+    return yaml_parser.load_models_from_yaml_filepath(path, Rubric)
+
+
+def load_rubrics_from_yaml_stream(
+    stream,
+) -> dict[str, Rubric]:
+    return yaml_parser.load_models_from_yaml_stream(stream, Rubric)
+
+
+def get_default_rubric_collection() -> RubricsCollection:
+    global default_rubric_collection
+    if default_rubric_collection is not None:
+        return default_rubric_collection
+    logger.debug("Attempting to load from default rubric_metrics.yaml.")
+    metrics_path = (
+        importlib.resources.files(flexeval.configuration) / "rubric_metrics.yaml"
+    )
+    metrics_string = metrics_path.read_text()
+    rubric_metrics = load_rubrics_from_yaml_stream(metrics_string)
+    default_rubric_collection = RubricsCollection(rubrics=rubric_metrics)
+    return default_rubric_collection
+
+
+def load_rubric_metrics(
+    rubric_collections: list[Path | RubricsCollection], add_defaults: bool = False
+) -> dict[str, Rubric]:
+    rubric_metrics = {}
+    if add_defaults:
+        rubric_collections.append(get_default_rubric_collection())
+    for rubric_collection in rubric_collections:
+        if isinstance(rubric_collection, Path):
+            rubric_dict = load_rubrics_from_yaml_file(rubric_collection)
+        else:
+            rubric_dict = rubric_collection.rubrics
+        for key, rubric in rubric_dict.items():
+            if key not in rubric_metrics:
+                rubric_metrics[key] = rubric
+                if rubric.name is None:
+                    # infer rubric name from parent key
+                    rubric.name = key
+            else:
+                logger.warning(f"Ignoring second (or later) version of rubric '{key}'.")
+    return rubric_metrics

flexeval/run_utils.py

@@ -0,0 +1,65 @@
+"""Utilities for :mod:`~flexeval.runner`."""
+
+import json
+import logging
+
+from flexeval import rubric
+from flexeval.classes.dataset import Dataset
+from flexeval.classes.eval_runner import EvalRunner
+from flexeval.classes.eval_set_run import EvalSetRun
+
+logger = logging.getLogger(__name__)
+
+
+def build_eval_set_run(runner: EvalRunner) -> EvalSetRun:
+    rubrics = rubric.load_rubric_metrics(runner.evalrun.rubric_paths)
+
+    # TODO this code uses a model_name that does not appear in the Eval schema; should look into this
+    model_name = json.dumps(None)
+    # model_name = json.dumps(
+    #        runner.eval.get("completion_llm", {}).get("model_name", None)
+    #    )
+    evalsetrun = EvalSetRun.create(
+        name=runner.evalrun.eval.name,
+        notes=runner.evalrun.eval.notes,
+        metrics=runner.evalrun.eval.metrics.model_dump_json(),
+        metrics_graph_ordered_list=json.dumps(runner.metrics_graph_ordered_list),
+        dataset_files=json.dumps(
+            [str(data_source.path) for data_source in runner.evalrun.data_sources]
+        ),
+        do_completion=runner.evalrun.eval.do_completion,
+        completion_llm=(
+            runner.evalrun.eval.completion_llm.model_dump_json()
+            if runner.evalrun.eval.completion_llm is not None
+            else json.dumps(None)
+        ),
+        model_name=model_name,
+        grader_llm=(
+            runner.evalrun.eval.grader_llm.model_dump_json()
+            if runner.evalrun.eval.grader_llm is not None
+            else json.dumps(None)
+        ),
+        # only save rubrics that will actually be used
+        rubrics=json.dumps(
+            {
+                i["evaluation_name"]: rubrics[i["evaluation_name"]].model_dump()
+                for i in runner.metrics_graph_ordered_list
+                if i["evaluation_type"] == "rubric"
+            }
+        ),
+    )
+    return evalsetrun
+
+
+def build_datasets(runner: EvalRunner, evalsetrun: EvalSetRun):
+    for filename in evalsetrun.get_datasets():
+        # these will automatically be saved as a property of evalsetrun
+        Dataset.create(
+            evalsetrun=evalsetrun,
+            filename=filename,
+            max_n_conversation_threads=runner.evalrun.config.max_n_conversation_threads,
+            nb_evaluations_per_thread=runner.evalrun.config.nb_evaluations_per_thread,
+        )
+        runner.logger.info(
+            f"Created dataset from '{filename}'. Max number of conversation threads: '{runner.evalrun.config.max_n_conversation_threads}' - Nb of evaluations per thread: '{runner.evalrun.config.nb_evaluations_per_thread}'"
+        )

flexeval/runner.py

@@ -0,0 +1,132 @@
+"""Convenience functions for running an Eval Run."""
+
+import logging
+import random as rd
+from pathlib import Path
+
+import flexeval.metrics
+from flexeval import completions, compute_metrics, run_utils
+from flexeval.classes.eval_runner import EvalRunner
+from flexeval.io.parsers import yaml_parser
+from flexeval.schema import EvalRun, FileDataSource
+
+logger = logging.getLogger(__name__)
+
+
+# Levels of abstraction -
+# Dataset
+# Thread
+# Turn
+# Message
+# ToolCall
+# Metric
+
+# Features to add:
+# - allow comparison with 'ideal' responses
+
+
+def run_from_name_args(
+    input_data: list[Path],
+    database_path: Path,
+    eval_name: str,
+    config_path: str,
+    evals_path: str,
+    **kwargs,
+):
+    data_sources = [FileDataSource(path=input_path) for input_path in input_data]
+    config = yaml_parser.load_config_from_yaml(config_path)
+    evals = yaml_parser.load_evals_from_yaml(evals_path)
+    if eval_name not in evals:
+        raise ValueError(
+            f"Eval name '{eval_name}' not in defined evals: {list(evals.keys())}"
+        )
+    selected_eval = evals[eval_name]
+    if selected_eval.name is None or selected_eval.name.strip() == "":
+        selected_eval.name = eval_name
+    for key, value in kwargs.items():
+        setattr(config, key, value)
+    eval_run = EvalRun(
+        data_sources=data_sources,
+        database_path=database_path,
+        eval=selected_eval,
+        config=config,
+    )
+    return run(eval_run)
+
+
+def run(eval_run: EvalRun) -> EvalRunner:
+    """Runs the evaluations."""
+    runner = EvalRunner(eval_run)
+
+    #######################################################
+    ############  Create Test Run  ########################
+    #######################################################
+    try:
+        runner.logger.info("Creating EvalSetRun")
+        # TODO instead of raw 'metrics', pass in graph created when setting up the runner
+
+        evalsetrun = run_utils.build_eval_set_run(runner)
+        runner.logger.info(f"Metric graph: {evalsetrun.metrics_graph_ordered_list}")
+    except Exception:
+        runner.logger.exception(
+            "An error occurred creating the EvalSetRun.", exc_info=True
+        )
+        runner.shutdown_logging()
+        raise
+
+    #######################################################
+    ############  Load and Parse Data  ####################
+    #######################################################
+
+    try:
+        runner.logger.debug("Loading data")
+
+        # set random seed
+        rd_seed = runner.evalrun.config.random_seed_conversation_sampling
+        rd.seed(rd_seed)
+        runner.logger.info(f"Set random seed to '{rd_seed}'.")
+
+        run_utils.build_datasets(runner, evalsetrun)
+    except Exception:
+        runner.logger.exception(
+            "An error occurred creating dataset metadata.", exc_info=True
+        )
+
+    try:
+        runner.logger.info("Parsing data files")
+        for dataset in evalsetrun.datasets:
+            runner.logger.debug(f"Loading data from '{dataset.filename}'.")
+            dataset.load_data()
+    except Exception:
+        runner.logger.exception("An error occurred loading data.", exc_info=True)
+
+    # Do completions, if necessary
+    try:
+        if evalsetrun.do_completion:
+            # We do this by creating new turns
+            runner.logger.info("Generating completions")
+            completions.get_completions(eval_run, evalsetrun)
+    except Exception:
+        runner.logger.exception(
+            "An error occurred generating completions.", exc_info=True
+        )
+        if eval_run.config.raise_on_completion_error:
+            runner.shutdown_logging()
+            raise
+
+    #######################################################
+    #################  Compute Metrics  ###################
+    #######################################################
+    try:
+        metrics = compute_metrics.compute_metrics(eval_run, evalsetrun)
+        runner.logger.info(f"Saving '{len(metrics)}' metrics to database.")
+        flexeval.metrics.save.save_metrics(metrics)
+    except Exception:
+        runner.logger.exception("An error occurred computing metrics.", exc_info=True)
+        if eval_run.config.raise_on_metric_error:
+            runner.shutdown_logging()
+            raise
+
+    runner.logger.info("Evaluation run complete.")
+    runner.shutdown_logging()
+    return runner

flexeval/schema/__init__.py

@@ -0,0 +1,11 @@
+"""Pydantic schema for the core configuration options used for FlexEval.
+
+The top-level schema object is :class:`~flexeval.schema.evalrun_schema.EvalRun`.
+
+See :mod:`~flexeval.classes` for the internal Peewee objects produced from the Pydantic configuration.
+"""
+
+from .config_schema import *
+from .eval_schema import *
+from .evalrun_schema import *
+from .rubric_schema import *

flexeval/schema/config_schema.py

@@ -0,0 +1,46 @@
+from pathlib import Path
+
+from pydantic import BaseModel, Field
+
+from flexeval.schema import schema_utils
+
+
+class Config(BaseModel):
+    class Config:
+        extra = "ignore"
+        # validate_assignment will ensure that fields never take an invalid value e.g. if copied from an eval
+        validate_assignment = True
+
+    logs_path: Path | None = Field(
+        None,
+        description="Log directory path.",
+    )
+    env_filepath: Path | None = Field(
+        None,
+        description="A .env file to be processed by python-dotenv before running evals with this config.",
+    )
+    env: schema_utils.OptionalDict = Field(
+        default_factory=dict, description="Any additional environment variables."
+    )
+    clear_tables: bool = Field(
+        False,
+        description="Clear any existing tables, if the output SQLite database already exists.",
+    )
+    max_workers: int = Field(
+        1,
+        description="Max worker count. Multiple threads will be used if set to > 1. This may have usage limit implications if you are calling APIs.",
+    )
+    random_seed_conversation_sampling: int = 42
+    max_n_conversation_threads: int = (
+        50  # TODO allow setting this to None, and set it to None by default
+    )
+    nb_evaluations_per_thread: int = 1
+
+    raise_on_completion_error: bool = Field(
+        False,
+        description="If False (default), metrics will be run even if one or more completions fails.",
+    )
+    raise_on_metric_error: bool = Field(
+        False,
+        description="If False (default), no exception will be thrown if a metric function raises an exception.",
+    )