eval-framework 0.2.7__py3-none-any.whl

eval_framework/__init__.py

@@ -0,0 +1,7 @@
+from importlib.metadata import version
+
+__version__ = version("eval-framework")
+
+del version
+
+__all__ = ["__version__"]

eval_framework/base_config.py

@@ -0,0 +1,36 @@
+from enum import Enum
+from pathlib import Path
+from typing import Any
+
+import yaml  # type: ignore[import-untyped]
+from pydantic import BaseModel, ConfigDict
+
+
+class BaseConfig(BaseModel):
+    model_config = ConfigDict(extra="forbid", frozen=True, protected_namespaces=())
+
+    def as_dict(self) -> dict[str, Any]:
+        def simplify_recursive(obj: Any) -> Any:
+            if isinstance(obj, dict):
+                return {key: simplify_recursive(value) for key, value in obj.items()}
+            elif isinstance(obj, list):
+                return [simplify_recursive(item) for item in obj]
+            elif isinstance(obj, Path):
+                return str(obj)
+            elif isinstance(obj, Enum):
+                return obj.value
+            else:
+                return obj
+
+        return simplify_recursive(self.model_dump())
+
+    @classmethod
+    def from_yaml(cls, yml_filename: str | Path) -> "BaseConfig":
+        with open(yml_filename) as conf_file:
+            config_dict = yaml.load(conf_file, Loader=yaml.FullLoader)
+
+        return cls(**config_dict)
+
+    def save(self, out_file: Path) -> None:
+        with open(out_file, "w", encoding="UTF-8") as f:
+            yaml.safe_dump(self.model_dump(mode="json"), f)

eval_framework/context/determined.py

@@ -0,0 +1,177 @@
+import logging
+from pathlib import Path
+from typing import Annotated, Any
+
+from determined._info import get_cluster_info
+from determined.core._context import Context
+from determined.core._context import init as determined_core_init
+from determined.core._distributed import DummyDistributedContext
+from pydantic import AfterValidator, BaseModel, ConfigDict
+
+from eval_framework.context.eval import EvalContext
+from eval_framework.context.local import _load_model
+from eval_framework.llm.base import BaseLLM
+from eval_framework.tasks.eval_config import EvalConfig
+from eval_framework.tasks.perturbation import PerturbationConfig
+from eval_framework.tasks.registry import validate_task_name
+from eval_framework.tasks.task_loader import load_extra_tasks
+
+logger = logging.getLogger(__name__)
+
+
+class TaskArgs(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+    task_name: Annotated[str, AfterValidator(validate_task_name)]
+    num_fewshot: int
+    num_samples: int | None = None
+    max_tokens: int | None = None
+    batch_size: int | None = None
+    judge_model_name: str | None = None
+    judge_model_args: dict[str, Any] = {}
+    task_subjects: list[str] | None = None
+    hf_revision: str | None = None
+    perturbation_config: PerturbationConfig | None = None
+
+
+class Hyperparameters(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+    llm_name: str
+    output_dir: Path
+    hf_upload_dir: str | None = None
+    hf_upload_repo: str | None = None
+    wandb_project: str | None = None
+    wandb_entity: str | None = None
+    wandb_run_id: str | None = None
+    wandb_upload_results: bool | None = None
+    description: str | None = None
+    task_args: TaskArgs
+    llm_args: dict[str, Any] | None = {}
+    extra_task_modules: list[str] | None = None
+    delete_output_dir_after_upload: bool | None = None
+
+
+class DeterminedContext(EvalContext):
+    def __init__(self, **kwargs: Any) -> None:
+        super().__init__(**kwargs)
+        self._core_context: Context | None = None
+
+    def __enter__(self) -> "DeterminedContext":
+        distributed_context = DummyDistributedContext()
+        self._core_context = determined_core_init(distributed=distributed_context)
+        self._core_context.start()
+        info = get_cluster_info()
+
+        if info is None:
+            raise RuntimeError("Failed to retrieve cluster info.")
+
+        # Load extra tasks if specified first
+        extra_task_modules = info.trial.hparams.get("extra_task_modules", None)
+        if extra_task_modules:
+            name = "extra_task_modules"
+            val_cli = getattr(self, name, None)
+            val_hparams = extra_task_modules
+            if val_hparams:
+                if val_cli and val_hparams and val_cli != val_hparams:
+                    logger.info(
+                        f"CLI argument {name} ({val_cli}) is being overridden by hyperparameters:"
+                        f"({val_hparams}). If it fails due to duplicate task names, remove the CLI argument and"
+                        "consolidate as a determined hyperparameter instead."
+                    )
+                load_extra_tasks(val_hparams)
+
+        self.hparams = Hyperparameters(**info.trial.hparams)
+
+        for name in [
+            "llm_name",
+            "llm_args",
+            "output_dir",
+            "hf_upload_dir",
+            "hf_upload_repo",
+            "wandb_project",
+            "wandb_entity",
+            "wandb_run_id",
+            "wandb_upload_results",
+            "description",
+            "delete_output_dir_after_upload",
+        ]:
+            val_cli = getattr(self, name, None)
+            val_hparams = getattr(self.hparams, name, None)
+            if val_cli and val_hparams and val_cli != val_hparams:
+                logger.info(f"CLI argument {name} ({val_cli}) is being overridden by hyperparameters: ({val_hparams}).")
+
+        for name in [
+            "num_samples",
+            "max_tokens",
+            "num_fewshot",
+            "task_name",
+            "task_subjects",
+            "batch_size",
+            "hf_revision",
+            "judge_model_name",
+            "judge_model_args",
+            "perturbation_config",
+        ]:
+            val_cli = getattr(self, name, None)
+            val_hparams = getattr(self.hparams.task_args, name, None)
+            if val_cli and val_hparams and val_cli != val_hparams:
+                logger.info(f"CLI argument {name} ({val_cli}) is being overridden by hyperparameters: ({val_hparams}).")
+
+        # Hyperparameters take precedence over core context
+        llm_name = self.hparams.llm_name or self.llm_name
+        judge_model_name = self.hparams.task_args.judge_model_name or self.judge_model_name
+
+        llm_class = _load_model(llm_name, models_path=self.models_path)
+        llm_judge_class: type[BaseLLM] | None = (
+            _load_model(judge_model_name, models_path=self.judge_models_path, info="judge")
+            if judge_model_name
+            else None
+        )
+
+        # for all optional hyperparameters, resort to the respective CLI argument if the hyperparameter is not set
+        self.config = EvalConfig(
+            llm_class=llm_class,
+            llm_args=self.hparams.llm_args or self.llm_args,
+            num_samples=self.hparams.task_args.num_samples or self.num_samples,
+            max_tokens=self.hparams.task_args.max_tokens or self.max_tokens,
+            num_fewshot=self.hparams.task_args.num_fewshot,
+            task_name=self.hparams.task_args.task_name,
+            task_subjects=self.hparams.task_args.task_subjects,
+            hf_revision=self.hparams.task_args.hf_revision or self.hf_revision,
+            perturbation_config=self.hparams.task_args.perturbation_config or self.perturbation_config,
+            output_dir=self.hparams.output_dir,
+            llm_judge_class=llm_judge_class,
+            judge_model_args=self.hparams.task_args.judge_model_args or self.judge_model_args,
+            hf_upload_dir=self.hparams.hf_upload_dir or self.hf_upload_dir,
+            hf_upload_repo=self.hparams.hf_upload_repo or self.hf_upload_repo,
+            wandb_project=self.hparams.wandb_project or self.wandb_project,
+            wandb_entity=self.hparams.wandb_entity or self.wandb_entity,
+            wandb_run_id=self.hparams.wandb_run_id or self.wandb_run_id,
+            wandb_upload_results=self.hparams.wandb_upload_results or self.wandb_upload_results,
+            batch_size=self.hparams.task_args.batch_size or self.batch_size,
+            description=self.hparams.description or self.description,
+            randomize_judge_order=self.randomize_judge_order,
+            delete_output_dir_after_upload=self.hparams.delete_output_dir_after_upload
+            or self.delete_output_dir_after_upload,
+        )
+
+        return self
+
+    def __exit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc_value: BaseException | None,
+        traceback: Any | None,
+    ) -> None:
+        if self._core_context is not None:
+            self._core_context.close()
+            self._core_context = None
+
+    def should_preempt(self) -> bool:
+        if self._core_context is None:
+            return False
+        return self._core_context.preempt.should_preempt()
+
+    def get_trial_id(self) -> int | None:
+        if self._core_context is None:
+            return None
+        return self._core_context.train._trial_id

eval_framework/context/eval.py

@@ -0,0 +1,121 @@
+import importlib.util
+import inspect
+import sys
+from contextlib import AbstractContextManager
+from os import PathLike
+from pathlib import Path
+from typing import Any
+
+import eval_framework
+from eval_framework.llm.base import BaseLLM
+from eval_framework.tasks.eval_config import EvalConfig
+from eval_framework.tasks.perturbation import PerturbationConfig
+
+
+def import_models(models_file: PathLike | str) -> dict[str, type[BaseLLM]]:
+    models_file = Path(models_file).resolve()
+    library_path = Path(eval_framework.__path__[0]).resolve()
+
+    # Imports from the eval_framework module need special care to avoid
+    # import issues
+    if models_file.is_relative_to(library_path):
+        relative_path = models_file.relative_to(library_path.parent)
+        module_name = ".".join(relative_path.with_suffix("").parts)
+        module = importlib.import_module(module_name)
+    else:
+        module_name = models_file.stem
+
+        spec = importlib.util.spec_from_file_location(module_name, str(models_file))
+
+        if spec is None:
+            raise ImportError(f"Could not load module '{models_file}'.")
+
+        module = importlib.util.module_from_spec(spec)
+        sys.modules[module_name] = module
+
+        if spec.loader is None:
+            raise ImportError(f"Could not load module '{models_file}'.")
+
+        spec.loader.exec_module(module)
+
+    subclasses = {}
+    for name, clazz in inspect.getmembers(module, inspect.isclass):
+        if issubclass(clazz, BaseLLM) and clazz is not BaseLLM:
+            subclasses[name] = clazz
+
+    return subclasses
+
+
+class EvalContext(AbstractContextManager):
+    def __init__(
+        self,
+        llm_name: str,
+        models_path: Path,
+        num_samples: int | None = None,
+        max_tokens: int | None = None,
+        num_fewshot: int | None = None,
+        task_name: str | None = None,
+        task_subjects: list[str] | None = None,
+        hf_revision: str | None = None,
+        output_dir: Path | None = None,
+        wandb_project: str | None = None,
+        wandb_entity: str | None = None,
+        wandb_run_id: str | None = None,
+        wandb_upload_results: bool | None = None,
+        hf_upload_dir: str | None = None,
+        hf_upload_repo: str | None = None,
+        llm_args: dict[str, Any] | None = None,
+        judge_models_path: Path | None = None,
+        judge_model_name: str | None = None,
+        judge_model_args: dict[str, Any] | None = None,
+        batch_size: int | None = None,
+        description: str | None = None,
+        perturbation_type: str | None = None,
+        perturbation_probability: float | None = None,
+        perturbation_seed: int | None = None,
+        randomize_judge_order: bool = False,
+        delete_output_dir_after_upload: bool | None = None,
+    ) -> None:
+        self.llm_name = llm_name
+        self.models_path = models_path
+        self.num_samples = num_samples
+        self.max_tokens = max_tokens
+        self.num_fewshot = num_fewshot
+        self.task_name = task_name
+        self.task_subjects = task_subjects
+        self.hf_revision = hf_revision
+        self.output_dir = output_dir
+        self.wandb_project = wandb_project
+        self.wandb_entity = wandb_entity
+        self.wandb_run_id = wandb_run_id
+        self.wandb_upload_results = wandb_upload_results
+        self.hf_upload_dir = hf_upload_dir
+        self.hf_upload_repo = hf_upload_repo
+        self.llm_args = llm_args if llm_args is not None else {}
+        self.judge_models_path = judge_models_path
+        self.judge_model_name = judge_model_name
+        self.judge_model_args = judge_model_args if judge_model_args is not None else {}
+        self.batch_size = batch_size
+        self.description = description
+        self.randomize_judge_order = randomize_judge_order
+        self.delete_output_dir_after_upload = delete_output_dir_after_upload
+
+        if perturbation_type or perturbation_probability is not None:
+            perturbation = {
+                "type": perturbation_type,
+                "probability": perturbation_probability,
+                "seed": perturbation_seed,
+            }
+            self.perturbation_config: PerturbationConfig | None = PerturbationConfig(
+                **{k: v for k, v in perturbation.items() if v is not None}
+            )
+        else:
+            self.perturbation_config = None
+
+        self.config: EvalConfig | None = None
+
+    def should_preempt(self) -> bool:
+        return False
+
+    def get_trial_id(self) -> int | None:
+        return None

eval_framework/context/local.py

@@ -0,0 +1,78 @@
+import importlib
+from os import PathLike
+from typing import Any
+
+from eval_framework.context.eval import EvalContext, import_models
+from eval_framework.llm.base import BaseLLM
+from eval_framework.tasks.eval_config import EvalConfig
+
+
+def _load_model(llm_name: str, models_path: str | PathLike | None, *, info: str = "") -> type[BaseLLM]:
+    """Load a model class either from a models file or as a fully qualified module path.
+
+    Args:
+        llm_name: The name of the model class to load, or a fully qualified module path.
+        models_path: The path to a Python file containing model class definitions
+        info: Additional info to include in error messages.
+    Returns:
+        The model class.
+    """
+    if models_path is None or "." in llm_name:
+        # The llm_name must a a fully qualified module path
+        if "." not in llm_name:
+            raise ValueError(f"LLM {info} '{llm_name}' is not a fully qualified module path.")
+        module_path, llm_class_name = llm_name.rsplit(".", 1)
+        module = importlib.import_module(module_path)
+        if not hasattr(module, llm_class_name):
+            raise ValueError(f"LLM '{llm_class_name}' not found in module '{module_path}'.")
+        return getattr(module, llm_class_name)
+    else:
+        models_dict = import_models(models_path)
+        if llm_name not in models_dict:
+            if info:
+                info = f"{info.strip()} "
+            raise ValueError(f"LLM {info} '{llm_name}' not found in {models_path}.")
+        return models_dict[llm_name]
+
+
+class LocalContext(EvalContext):
+    def __enter__(self) -> "LocalContext":
+        llm_class = _load_model(self.llm_name, models_path=self.models_path)
+        self.llm_judge_class: type[BaseLLM] | None = None
+        if self.judge_model_name is not None:
+            self.llm_judge_class = _load_model(self.judge_model_name, models_path=self.judge_models_path, info="judge")
+
+        self.config = EvalConfig(
+            llm_class=llm_class,
+            llm_args=self.llm_args,
+            num_samples=self.num_samples,
+            max_tokens=self.max_tokens,
+            num_fewshot=self.num_fewshot,
+            perturbation_config=self.perturbation_config,
+            task_name=self.task_name,
+            task_subjects=self.task_subjects,
+            hf_revision=self.hf_revision,
+            output_dir=self.output_dir,
+            hf_upload_dir=self.hf_upload_dir,
+            hf_upload_repo=self.hf_upload_repo,
+            wandb_entity=self.wandb_entity,
+            wandb_project=self.wandb_project,
+            wandb_run_id=self.wandb_run_id,
+            wandb_upload_results=self.wandb_upload_results,
+            llm_judge_class=self.llm_judge_class,
+            judge_model_args=self.judge_model_args,
+            batch_size=self.batch_size,
+            description=self.description,
+            randomize_judge_order=self.randomize_judge_order,
+            delete_output_dir_after_upload=self.delete_output_dir_after_upload,
+        )
+
+        return self
+
+    def __exit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc_value: BaseException | None,
+        traceback: Any | None,
+    ) -> None:
+        pass

eval_framework/evaluation_generator.py

@@ -0,0 +1,234 @@
+import logging
+import math
+
+import numpy as np
+import pandas as pd
+import wandb
+from tqdm import tqdm
+
+from eval_framework.metrics.base import BaseMetric
+from eval_framework.metrics.efficiency.bytes_per_sequence_position import (
+    BytesCompletion,
+    BytesLoglikelihood,
+    SequencePositionsCompletion,
+    SequencePositionsLoglikelihood,
+)
+from eval_framework.metrics.llm.base import BaseLLMJudgeMetric
+from eval_framework.result_processors.base import Result, ResultProcessor
+from eval_framework.shared.types import Completion, Loglikelihood
+from eval_framework.tasks.base import ResponseType
+from eval_framework.tasks.eval_config import EvalConfig
+from eval_framework.tasks.registry import get_task
+from eval_framework.utils.constants import RED, RESET
+from eval_framework.utils.tqdm_handler import get_disable_bar_flag, safe_tqdm_write
+
+logger = logging.getLogger(__name__)
+
+
+class EvaluationGenerator:
+    def __init__(self, config: EvalConfig, result_processor: ResultProcessor) -> None:
+        logger.info("EvaluationGenerator initialized")
+
+        self.few_shot = config.num_fewshot
+        self.config = config
+        self.num_samples = config.num_samples
+        self.max_tokens = config.max_tokens
+        self.result_processor = result_processor
+        self.save_intermediate_results = config.save_intermediate_results
+
+        task_class = get_task(config.task_name)
+        if task_class.RESPONSE_TYPE == ResponseType.COMPLETION:
+            self.metrics = task_class.METRICS + [BytesCompletion, SequencePositionsCompletion]
+        elif task_class.RESPONSE_TYPE == ResponseType.LOGLIKELIHOODS:
+            self.metrics = task_class.METRICS + [BytesLoglikelihood, SequencePositionsLoglikelihood]
+        else:
+            raise NotImplementedError
+
+        self.task_name = task_class.NAME
+
+    def _run_metric_calculators(self, responses: list[Completion | Loglikelihood]) -> list[Result]:
+        results: list[Result] = self.result_processor.load_metrics_results()
+        llm_name = self.result_processor.load_metadata()["llm_name"]
+
+        subject_result_id_existing = set()
+        for result in results:
+            subject_result_id_existing.add(f"{result.subject}_{result.id}_{result.metric_class_name}")
+
+        """
+        we have three dimensions: subject, metric, sample_id
+        we wanna average over sample_id
+        and also over all subjects by averaging over the averages
+        dict[metric, dict[subject, dict[sample_id, list[result]]]]
+        """
+        llm_judge = None
+        for metric_class in self.metrics:
+            metric: BaseMetric
+            if issubclass(metric_class, BaseLLMJudgeMetric):
+                if llm_judge is None:
+                    assert self.config.llm_judge_class is not None, "The llm_judge_class must be defined in the config."
+                    llm_judge = self.config.llm_judge_class(**self.config.judge_model_args)
+                metric = metric_class(
+                    llm_judge=llm_judge,
+                    randomize_order=self.config.randomize_judge_order,
+                )
+            else:
+                metric = metric_class()
+
+            logger.info(f"Starting calculation of {metric.NAME}")
+            safe_tqdm_write(f"INFO: Calculating {metric.NAME}")
+            for response in tqdm(responses, desc=f"Calculating {metric.NAME}", disable=get_disable_bar_flag()):
+                if f"{response.subject}_{response.id}_{metric.__class__.__name__}" in subject_result_id_existing:
+                    continue
+
+                subject = response.subject
+                metric_results = metric.calculate(response)
+                for metric_result in metric_results:
+                    if "/" in metric_result.metric_name:
+                        metric_name, key = metric_result.metric_name.split("/")
+                    else:
+                        metric_name = metric_result.metric_name
+                        key = None
+                    completion = response.completion if isinstance(response, Completion) else str(response.ground_truth)
+
+                    result = Result(
+                        id=response.id,
+                        metric_class_name=metric.__class__.__name__,
+                        metric_name=metric_name,
+                        num_fewshot=self.few_shot,
+                        key=key,
+                        subject=subject,
+                        llm_name=llm_name,
+                        task_name=self.task_name,
+                        value=metric_result.value,
+                        higher_is_better=metric_result.higher_is_better,
+                        prompt=response.prompt,
+                        response=completion,
+                        llm_judge_prompt=metric_result.llm_judge_prompt,
+                        llm_judge_response=metric_result.llm_judge_response,
+                        code_execution_trace=metric_result.code_execution_trace,
+                        error=metric_result.error,
+                    )
+                    results.append(result)
+                    if self.save_intermediate_results:
+                        self.result_processor.save_metrics_result(result)
+
+            logger.info(f"Completed calculation of {metric.NAME}")
+            safe_tqdm_write(f"INFO: Completed {metric.NAME}")
+
+        if not self.save_intermediate_results:
+            self.result_processor.save_metrics_results(results)
+        return results
+
+    def _aggregate_results(self, results: list[Result]) -> dict[str, float | None]:
+        data = pd.DataFrame([r.model_dump() for r in results])
+        if len(data) == 0:
+            return {}
+        data.fillna({"key": ""}, inplace=True)
+        metrics = sorted(data["metric_name"].unique())
+        aggregated_results: dict[str, float | None] = {}
+
+        for metric in metrics:
+            # filter for metric
+            data_subset = data[data["metric_name"] == metric][["subject", "key", "value", "error"]]
+
+            # filter and count errors
+            total_count = len(data_subset)
+
+            mask = data["error"].isnull()
+            data_subset_error_free = data_subset.loc[mask, ["subject", "key", "value"]]
+            # data_subset_error_free = data_subset[data_subset["error"].isnull()][["subject", "key", "value"]]
+
+            aggregated_results[f"ErrorFreeRatio {metric}"] = float(len(data_subset_error_free) / total_count)
+
+            # aggregate by key and subject first to have equal weights for all key / subject combinations
+            key_subject_mean = data_subset_error_free.groupby(["key", "subject"]).mean()
+            aggregated_results[f"Average {metric}"] = float(key_subject_mean[["value"]].mean()["value"])
+
+            std_err_mean_sum_of_squares = 0.0
+            std_err_mean_total_num_samples = 0.0
+            std_err_mean_num_subjects = 0
+
+            for column in ["key", "subject"]:
+                if len(data_subset[column].unique()) > 1:
+                    for name, _group in key_subject_mean.groupby([column]):
+                        mask = data_subset[column] == name[0]
+                        group = data_subset.loc[mask, ["subject", "key", "value", "error"]]
+                        # group = data_subset[data[column] == name][["subject", "key", "value", "error"]]
+                        group_total_count = len(group)
+                        group_error_free = group[group["error"].isnull()][["subject", "key", "value"]]
+                        aggregated_results[f"ErrorFreeRatio {metric} - {name[0]}"] = float(
+                            len(group_error_free) / group_total_count
+                        )
+
+                        group_key_subject_mean = group_error_free.groupby(["key", "subject"]).mean()
+                        value = float(group_key_subject_mean[["value"]].mean()["value"])
+                        aggregated_results[f"Average {metric} - {name[0]}"] = value if not math.isnan(value) else None
+
+                        if not ("SequencePositions" in metric or "Bytes" in metric):
+                            # calculate standard error for selected  metrics
+                            group_key_subject_std = group_error_free.groupby(["key", "subject"]).std()
+                            std = float(group_key_subject_std[["value"]].mean()["value"])
+                            num_samples = len(group_error_free)
+
+                            if math.isnan(std) or num_samples == 0:
+                                aggregated_results[f"StdErr {metric} - {name[0]}"] = None
+                            else:
+                                aggregated_results[f"StdErr {metric} - {name[0]}"] = std / np.sqrt(num_samples)
+                            aggregated_results[f"NumSamples {metric} - {name[0]}"] = num_samples
+
+                            std_err_mean_sum_of_squares += std**2 / num_samples
+                            std_err_mean_total_num_samples += num_samples
+                            std_err_mean_num_subjects += 1
+
+            if not ("SequencePositions" in metric or "Bytes" in metric):
+                # calculate standard error for selected  metrics
+                if std_err_mean_total_num_samples > 0:
+                    # calculate the standard error of the mean (SEM) for the aggregated results (eg. add in quadrature)
+                    # SEM = sqrt(sum(variance_i * n_i) / i)
+                    # where variance_i is the variance of each group and i is the number of groups
+                    # (the combined mean is also not weighted by the number of samples)
+                    if math.isnan(std) or std_err_mean_total_num_samples == 0:
+                        aggregated_results[f"StdErr {metric}"] = None
+                    else:
+                        aggregated_results[f"StdErr {metric}"] = np.sqrt(
+                            std_err_mean_sum_of_squares / std_err_mean_num_subjects
+                        )
+                    aggregated_results[f"NumSamples {metric}"] = std_err_mean_total_num_samples
+                else:
+                    # if there are no sub-groups to combine, calculate the SEM here directly
+                    key_subject_std = data_subset_error_free.groupby(["key", "subject"]).std()
+                    std = float(key_subject_std[["value"]].mean()["value"])
+                    num_samples = len(data_subset_error_free)
+                    if math.isnan(std) or num_samples == 0:
+                        aggregated_results[f"StdErr {metric}"] = None
+                    else:
+                        aggregated_results[f"StdErr {metric}"] = std / np.sqrt(num_samples)
+                    aggregated_results[f"NumSamples {metric}"] = num_samples
+
+        if (
+            "Average Bytes" in aggregated_results
+            and "Average SequencePositions" in aggregated_results
+            and aggregated_results["Average Bytes"]
+            and aggregated_results["Average SequencePositions"]
+        ):
+            aggregated_results["Average Bytes per Sequence Position"] = (
+                aggregated_results["Average Bytes"] / aggregated_results["Average SequencePositions"]
+            )
+
+        return aggregated_results
+
+    def run_eval(self) -> list[Result]:
+        """Runs evaluation using saved completions."""
+        logger.info("Running evaluation...")
+        responses = self.result_processor.load_responses()
+        if not responses:
+            raise ValueError("No saved completions found. Run 'run_completions' first.")
+
+        metrics_results = self._run_metric_calculators(responses)
+        aggregated_results = self._aggregate_results(metrics_results)
+
+        wandb.log(aggregated_results)
+        self.result_processor.save_aggregated_results(aggregated_results)
+        logger.info(aggregated_results)
+        logger.info(f"{RED}[ Evaluation completed and results saved! ]{RESET}")
+        return metrics_results

eval_framework/exceptions.py

@@ -0,0 +1,2 @@
+class LogicError(Exception):
+    pass

eval_framework/external/ifeval_impl/README.md

@@ -0,0 +1,5 @@
+IFEval implementation here is taken 1:1 (besides some minuscule details, like adding Swedish to `LANGUAGE_CODES`)
+from [lm-evaluation-harness](https://github.com/EleutherAI/lm-evaluation-harness/tree/main/lm_eval/tasks/ifeval) as of Dec 19 2024.
+The original repository is https://github.com/google-research/google-research/tree/master/instruction_following_eval .
+
+This code doesn't have to be unit-tested.