eval-framework 0.2.0__py3-none-any.whl

eval_framework/llm/vllm.py

@@ -0,0 +1,438 @@
+import logging
+from abc import ABC, abstractmethod
+from collections.abc import Callable, Sequence
+from dataclasses import dataclass
+from functools import partial
+from typing import Any, Literal, Protocol, cast, override
+
+import torch
+from vllm import LLM, SamplingParams
+from vllm.inputs.data import TokensPrompt
+from vllm.outputs import RequestOutput
+from vllm.transformers_utils.tokenizer import get_tokenizer
+
+from eval_framework.llm.base import BaseLLM
+from eval_framework.shared.types import (
+    ConcatCompression,
+    Error,
+    PromptTooLongException,
+    RawCompletion,
+    RawLoglikelihood,
+)
+from eval_framework.tasks.base import Sample
+from eval_framework.tasks.utils import raise_errors
+from eval_framework.utils.constants import RED, RESET
+from template_formatting.formatter import BaseFormatter, HFFormatter, Message
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class TokenizedContainer:
+    """
+    Container object to store tokens and formatted prompt
+    """
+
+    tokens: list[int]
+    text: str
+
+
+class VLLMTokenizerAPI[prompt_type: (list[Message], str)](ABC):
+    """
+    Protocol for tokenizer interface that defines required methods.
+    Needed for type checking because of the vllm tokenizer.
+    """
+
+    @abstractmethod
+    def encode_formatted_struct(self, struct: prompt_type) -> TokenizedContainer:
+        """Encode prompt to token IDs."""
+        pass
+
+    @abstractmethod
+    def encode_plain_text(self, text: str) -> TokenizedContainer:
+        pass
+
+    @property
+    def chat_template(self) -> str | None:
+        return None
+
+
+class HFTokenizerProtocol(Protocol):
+    def encode(self, text: str, add_special_tokens: bool = False) -> list[int]:
+        """Encode text to token IDs."""
+        ...
+
+    def decode(self, tokens: list[int]) -> str:
+        """Decode token IDs to text."""
+        ...
+
+    @property
+    def chat_template(self) -> str | None:
+        """Chat template for the tokenizer."""
+        ...
+
+
+class VLLMTokenizer(VLLMTokenizerAPI[str]):
+    def __init__(self, target_mdl: str) -> None:
+        self.tokenizer = cast(HFTokenizerProtocol, get_tokenizer(target_mdl))
+
+    def _encode_text(self, text: str) -> TokenizedContainer:
+        tokens = self.tokenizer.encode(text, add_special_tokens=False)
+        return TokenizedContainer(tokens=tokens, text=text)
+
+    def encode_formatted_struct(self, struct: str) -> TokenizedContainer:
+        return self._encode_text(text=struct)
+
+    def encode_plain_text(self, text: str) -> TokenizedContainer:
+        return self._encode_text(text=text)
+
+    def decode(self, tokens: list[int]) -> str:
+        return self.tokenizer.decode(tokens)
+
+    @override
+    @property
+    def chat_template(self) -> str | None:
+        return self.tokenizer.chat_template
+
+
+class VLLMModel(BaseLLM):
+    LLM_NAME: str
+    DEFAULT_FORMATTER: Callable[[], BaseFormatter] | None = None
+    SEQ_LENGTH: int | None = None
+
+    def __init__(
+        self,
+        formatter: BaseFormatter | None = None,
+        max_model_len: int | None = None,
+        tensor_parallel_size: int = 1,
+        gpu_memory_utilization: float = 0.9,
+        batch_size: int = 1,
+        checkpoint_path: str | None = None,
+        checkpoint_name: str | None = None,
+        sampling_params: SamplingParams | dict[str, Any] | None = None,
+        **kwargs: Any,
+    ) -> None:
+        # Store the max_model_len for later use
+        self._max_model_len = max_model_len
+        self.checkpoint_name = checkpoint_name
+        self.checkpoint_path = checkpoint_path
+
+        model_args = {
+            "model": self.checkpoint_path or self.LLM_NAME,
+            "max_model_len": max_model_len or self.SEQ_LENGTH,
+            "max_num_seqs": batch_size,
+            "tensor_parallel_size": tensor_parallel_size,
+            "gpu_memory_utilization": gpu_memory_utilization,
+            **kwargs,
+        }
+
+        device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu")
+
+        self.batch_size = batch_size
+        self._tokenizer: None | VLLMTokenizerAPI = None
+
+        self.model = LLM(**model_args, device=device)
+
+        self.sampling_params: SamplingParams = self._process_sampling_params(sampling_params)
+
+        logger.info(f"{RED}[ Model initialized --------------------- {RESET}{self.LLM_NAME} {RED}]{RESET}")
+        self._set_formatter(formatter)
+
+    def _process_sampling_params(self, sampling_params: SamplingParams | dict[str, Any] | None) -> SamplingParams:
+        processed_sampling_params: SamplingParams | None = None
+        if isinstance(sampling_params, dict):
+            processed_sampling_params = SamplingParams(**sampling_params)
+            logger.info(f"Converted sampling_params dict to SamplingParams: {processed_sampling_params}")
+        elif sampling_params is not None:
+            processed_sampling_params = sampling_params
+        else:
+            processed_sampling_params = self.model.get_default_sampling_params()
+
+        return processed_sampling_params
+
+    def _set_formatter(self, formatter: BaseFormatter | None = None) -> None:
+        if formatter is not None:
+            self._formatter = formatter
+        elif self.DEFAULT_FORMATTER is not None:
+            self._formatter = self.DEFAULT_FORMATTER()
+        elif self.tokenizer.chat_template is not None:
+            self._formatter = HFFormatter(self.LLM_NAME)
+        else:
+            raise ValueError("No formatter specified and no default formatter available.")
+
+        logger.info(
+            f"{RED}[ Using default formatter --------------------- {RESET}{self._formatter.__class__.__name__} {RED}]{RESET}"  # noqa: E501
+        )
+
+    @property
+    def tokenizer(self) -> VLLMTokenizerAPI:
+        if self._tokenizer is None:
+            self._tokenizer = VLLMTokenizer(target_mdl=self.LLM_NAME)
+        return self._tokenizer
+
+    def count_tokens(self, text: str, /) -> int:
+        return len(self.tokenizer.encode_plain_text(text).tokens)
+
+    @property
+    def formatter_output_mode(self) -> Literal["string", "list"]:
+        return "string"
+
+    @property
+    def name(self) -> str:
+        if self.checkpoint_name:
+            return f"{self.__class__.__name__}_checkpoint_{self.checkpoint_name}"
+        return self.__class__.__name__
+
+    def build_redis_key_from_prompt_objs(
+        self, prompt_objs: list[TokenizedContainer], sampling_params: SamplingParams
+    ) -> Any:
+        """
+        Build a redis key from a list of prompt objects and sampling parameters.
+        TokenizedContainers are not serializable so we just pass the tokens and sampling params.
+        """
+        return ([obj.tokens for obj in prompt_objs], sampling_params)
+
+    def generate_from_messages(
+        self,
+        messages: list[Sequence[Message]],
+        stop_sequences: list[str] | None = None,
+        max_tokens: int | None = None,
+        temperature: float | None = None,
+    ) -> list[RawCompletion]:
+        raw_completions: list[RawCompletion | None] = [None] * len(messages)
+        prompt_objs = []
+        prompt_info = []
+
+        sampling_params = self._resolve_sampling_params(self.sampling_params, max_tokens, stop_sequences, temperature)
+
+        for i, single_messages in enumerate(messages):
+            output_mode = self.formatter_output_mode
+            prompt: str | list[Message] = self._formatter.format(single_messages, output_mode=output_mode)
+            prompt_obj: TokenizedContainer = self.tokenizer.encode_formatted_struct(prompt)
+            prompt_token_count = len(prompt_obj.tokens)
+
+            max_tokens_to_generate = self.max_seq_length - prompt_token_count
+
+            if max_tokens is not None:
+                max_tokens_to_generate = min(max_tokens_to_generate, max_tokens)
+
+            if max_tokens_to_generate < 1:
+                if raise_errors():
+                    raise PromptTooLongException("Prompt exceeded context size.")
+
+                raw_completions[i] = RawCompletion(
+                    prompt=prompt_obj.text,
+                    prompt_sequence_positions=prompt_token_count,
+                    completion="",
+                    completion_sequence_positions=0,
+                    raw_completion_error=Error(
+                        error_class=PromptTooLongException.__name__,
+                        message="Prompt exceeded context size.",
+                        traceback="",
+                    ),
+                )
+                continue
+
+            prompt_objs.append(prompt_obj)
+            prompt_info.append((i, single_messages))
+
+        if prompt_objs:
+            model_outputs = self._model_generate(prompt_objs=prompt_objs, sampling_params=sampling_params)
+
+            for (original_index, single_messages), prompt_obj, output in zip(prompt_info, prompt_objs, model_outputs):
+                raw_completions[original_index] = RawCompletion(
+                    prompt=prompt_obj.text,
+                    prompt_sequence_positions=len(output.prompt_token_ids) if output.prompt_token_ids else 0,
+                    concat_compression=ConcatCompression.calculate(
+                        single_messages, count_tokens=self.count_tokens, completion=output.outputs[0].text
+                    ),
+                    completion=output.outputs[0].text,
+                    completion_sequence_positions=len(output.outputs[0].token_ids)
+                    if output.outputs[0].token_ids
+                    else 0,
+                    raw_completion_error=None,
+                )
+
+        # Ensure all positions are filled (should never be None at this point)
+        return cast(list[RawCompletion], raw_completions)
+
+    @staticmethod
+    def _resolve_sampling_params(
+        sampling_params: SamplingParams,
+        max_tokens: int | None,
+        stop_sequences: list[str] | None,
+        temperature: float | None,
+    ) -> SamplingParams:
+        sampling_params.max_tokens = max_tokens
+        sampling_params.stop = stop_sequences
+        if temperature is not None:
+            logger.warning(
+                f"Overriding sampling params temperature {sampling_params.temperature} with custom value {temperature}"
+            )
+            sampling_params.temperature = temperature
+        else:
+            logger.info(
+                f"Using sampling params temperature value: {sampling_params.temperature} "
+                f"as no custom temperature value was provided"
+            )
+        return sampling_params
+
+    def _model_generate(
+        self,
+        prompt_objs: list[TokenizedContainer],
+        sampling_params: SamplingParams,
+    ) -> list[RequestOutput]:
+        vllm_token_prompt = [TokensPrompt(prompt_token_ids=prompt_obj.tokens) for prompt_obj in prompt_objs]
+        outputs = self.model.generate(vllm_token_prompt, sampling_params)
+
+        return outputs
+
+    def logprobs(self, samples: list[Sample]) -> list[RawLoglikelihood]:
+        """Batched version of logprobs for improved performance."""
+        results: list[RawLoglikelihood | None] = [None] * len(samples)
+
+        # Collect all prompt-choice combinations
+        batch_data = []
+        sample_choice_indices = []  # Maps batch index back to (sample_index, choice)
+
+        for sample_idx, sample in enumerate(samples):
+            output_mode = self.formatter_output_mode
+            prompt: str | list[Message] = self._formatter.format(sample.messages, output_mode=output_mode)
+            prompt_obj: TokenizedContainer = self.tokenizer.encode_formatted_struct(prompt)
+
+            choices_log_probs: dict[str, float] = {}
+            choices_log_probs_sequence_positions: dict[str, int] = {}
+            error: Error | None = None
+            valid_choices = []
+
+            for choice in sample.possible_completions or []:
+                choice_obj: TokenizedContainer = self.tokenizer.encode_plain_text(choice)
+                total_tokens_count = len(prompt_obj.tokens + choice_obj.tokens)
+
+                if total_tokens_count > self.max_seq_length:
+                    if raise_errors():
+                        raise PromptTooLongException("Prompt exceeded context size.")
+                    choices_log_probs = {}
+                    choices_log_probs_sequence_positions = {}
+                    error = Error(
+                        error_class=PromptTooLongException.__name__,
+                        message="Prompt and choice exceeded context size.",
+                        traceback="",
+                    )
+                    break
+                else:
+                    batch_data.append((prompt_obj, choice_obj))
+                    sample_choice_indices.append((sample_idx, choice))
+                    valid_choices.append(choice)
+                    choices_log_probs_sequence_positions[choice] = len(choice_obj.tokens)
+
+            # If we had an error, store the result immediately
+            if error is not None:
+                results[sample_idx] = RawLoglikelihood(
+                    prompt=prompt_obj.text,
+                    prompt_sequence_positions=len(prompt_obj.tokens),
+                    loglikelihoods=choices_log_probs,
+                    loglikelihoods_sequence_positions=choices_log_probs_sequence_positions,
+                    raw_loglikelihood_error=error,
+                )
+            else:
+                results[sample_idx] = RawLoglikelihood(
+                    prompt=prompt_obj.text,
+                    prompt_sequence_positions=len(prompt_obj.tokens),
+                    loglikelihoods=choices_log_probs,
+                    loglikelihoods_sequence_positions=choices_log_probs_sequence_positions,
+                    raw_loglikelihood_error=None,
+                    concat_compression=ConcatCompression.calculate(
+                        sample.messages, count_tokens=self.count_tokens, choices=valid_choices
+                    ),
+                )
+
+        # Process batch if we have valid data
+        if batch_data:
+            batch_logprobs = self._model_log_probs(batch_data)
+
+            # Distribute results back to samples
+            for batch_idx, logprob in enumerate(batch_logprobs):
+                sample_idx, choice = sample_choice_indices[batch_idx]
+                result = results[sample_idx]
+                if result is not None:
+                    result.loglikelihoods[choice] = logprob
+
+        return cast(list[RawLoglikelihood], results)
+
+    def _model_log_probs(self, batch_data: list[tuple[TokenizedContainer, TokenizedContainer]]) -> list[float]:
+        """Batched version of _model_log_probs for processing multiple prompt-choice pairs at once."""
+        sampling_params = SamplingParams(
+            max_tokens=1,
+            temperature=0.0,
+            prompt_logprobs=1,
+            detokenize=False,
+        )
+
+        vllm_token_prompts = [
+            TokensPrompt(prompt_token_ids=prompt_obj.tokens + choice_obj.tokens)
+            for prompt_obj, choice_obj in batch_data
+        ]
+
+        try:
+            outputs = self.model.generate(vllm_token_prompts, sampling_params)
+        except Exception as e:
+            raise e
+
+        results = []
+        for i, (prompt_obj, choice_obj) in enumerate(batch_data):
+            output = outputs[i]
+            assert output.prompt_logprobs is not None
+
+            choice_logprobs = output.prompt_logprobs[-len(choice_obj.tokens) :]
+            total_logprob = 0.0
+
+            # VLLM guarantees the actual token's logprob is included in the output
+            for j, token_id in enumerate(choice_obj.tokens):
+                logprob_obj = choice_logprobs[j]
+                assert logprob_obj is not None, f"logprob_obj is None: {logprob_obj}"
+                logprob_value = getattr(logprob_obj[token_id], "logprob")
+                assert logprob_value is not None, f"logprob_value is None: {logprob_value}"
+                total_logprob += logprob_value
+
+            results.append(total_logprob)
+
+        return results
+
+    @property
+    def max_seq_length(self) -> int:
+        """
+        Returns the maximum sequence length for this model.
+        Priority order:
+        1. max_model_len parameter passed to __init__
+        2. SEQ_LENGTH class attribute
+        3. Model's actual max_model_len from config
+        4. Default fallback of 2048
+        """
+        if self._max_model_len is not None:
+            return self._max_model_len
+
+        if self.SEQ_LENGTH is not None:
+            return self.SEQ_LENGTH
+
+        if hasattr(self.model, "llm_engine") and hasattr(self.model.llm_engine, "model_config"):
+            return self.model.llm_engine.model_config.max_model_len
+
+        return 2048
+
+    @property
+    def seq_length(self) -> int | None:
+        """
+        Kept for backward compatibility.
+        """
+        return self.max_seq_length
+
+
+class Qwen3_0_6B_VLLM(VLLMModel):
+    LLM_NAME = "Qwen/Qwen3-0.6B"
+    DEFAULT_FORMATTER = partial(HFFormatter, LLM_NAME, chat_template_kwargs={"enable_thinking": True})
+
+
+class Qwen3_0_6B_VLLM_No_Thinking(VLLMModel):
+    LLM_NAME = "Qwen/Qwen3-0.6B"
+    DEFAULT_FORMATTER = partial(HFFormatter, LLM_NAME, chat_template_kwargs={"enable_thinking": False})

eval_framework/logger.py

@@ -0,0 +1,3 @@
+import logging
+
+logger = logging.getLogger("eval_framework")

eval_framework/main.py

@@ -0,0 +1,187 @@
+import json
+import logging
+from collections.abc import Callable
+from pathlib import Path
+from typing import Any, Literal
+
+import wandb
+
+from eval_framework.evaluation_generator import EvaluationGenerator, Result
+from eval_framework.llm.base import BaseLLM
+from eval_framework.response_generator import ResponseGenerator
+from eval_framework.result_processors.hf_processor import HFProcessor
+from eval_framework.result_processors.result_processor import ResultsFileProcessor, generate_output_dir
+from eval_framework.tasks.eval_config import EvalConfig
+from eval_framework.utils.constants import RED, RESET
+from eval_framework.utils.logging import setup_logging
+
+logger = logging.getLogger(__name__)
+
+
+def main(
+    llm: BaseLLM,
+    config: EvalConfig,
+    should_preempt_callable: Callable[[], bool] | None = None,
+    trial_id: int | None = None,
+) -> list[Result]:
+    """Runs the entire evaluation process: responses generation and evaluation."""
+    # Set up centralized logging early
+    output_dir = generate_output_dir(llm.name, config)
+    print(f"Output directory for evaluation: {output_dir}")
+    setup_logging(output_dir=output_dir, log_level=logging.INFO, log_filename="evaluation.log")
+
+    logger.info(f"{RED}[ Running full evaluation process ------- ]{RESET}")
+    logger.info(f"Evaluating {llm.name} on {config.task_name}")
+    logger.info(f"Configuration: num_fewshot={config.num_fewshot}, num_samples={config.num_samples}")
+    logger.info(f"Output directory: {output_dir}")
+
+    if not should_preempt_callable:
+        should_preempt_callable = lambda: False  # noqa: E731
+    preemption_data = None
+
+    if trial_id:
+        preemption_data = _read_preemption_data(config, trial_id)
+
+    if preemption_data is None:
+        output_dir = generate_output_dir(llm.name, config)
+        # config.wandb_run_id  defaults to none, if no run_id is provided then it starts a new one
+        wandb_run_id = config.wandb_run_id
+    else:
+        logger.info("Found preempted run restarting ...")
+        output_dir = preemption_data["output_dir"]
+        wandb_run_id = preemption_data.get("wandb_run_id", None)
+
+    logger.info(f"Output directory: {output_dir}")
+    assert output_dir is not None
+
+    file_processor = ResultsFileProcessor(output_dir)
+    response_generator = ResponseGenerator(llm, config, file_processor)
+    # take care of init after preemption handling. If we have a run
+    # id from preemption, then we resume the original wandb run
+    with wandb.init(
+        entity=config.wandb_entity,
+        project=config.wandb_project,
+        group=llm.name,
+        job_type=config.task_name,
+        id=wandb_run_id,
+        config=response_generator._get_metadata(),
+        resume="allow",
+        mode=_wandb_mode(config.wandb_project),
+    ) as run:
+        _, preempted = response_generator.generate(should_preempt_callable)
+        if preempted:
+            logger.info("Response generation was preempted")
+            assert trial_id is not None
+            run.mark_preempting()
+            _save_preemption_data(config, trial_id, output_dir, wandb_run_id=run.id)
+            wandb.finish(exit_code=1)
+            return []
+        # update config from response generator with get metadata
+        if trial_id is not None:
+            _delete_preemption_file(config, trial_id)
+
+        evaluator = EvaluationGenerator(config, file_processor)
+        results = evaluator.run_eval()
+
+        if config.hf_upload_dir:
+            hf_processor = HFProcessor(config, output_dir)
+            status, hf_url = hf_processor.upload_responses_to_HF()
+            if not status:
+                status_message = "*** Warning: Result upload to HF failed ***"
+            else:
+                status_message = "Successfully uploaded results to HuggingFace"
+                if hf_url and run:
+                    try:
+                        run.notes = f"Results uploaded to HuggingFace: [{hf_url}]({hf_url})"
+                    except Exception as e:
+                        logger.warning(f"Failed to update wandb notes with HF URL: {e}")
+        else:
+            status_message = f"{RED}[ Results not persisted in a HuggingFace repo ------- ]{RESET}"
+
+        logger.info(status_message)
+
+    return results
+
+
+def _read_preemption_data(config: EvalConfig, trial_id: int) -> dict[str, Any] | None:
+    preemption_file = config.output_dir / f"preemption_trial_{trial_id}.json"
+    if not preemption_file.is_file():
+        return None
+    with open(preemption_file, "rb") as f:
+        preemption_data = json.load(f)
+        preemption_data["output_dir"] = Path(preemption_data["output_dir"])
+        preemption_data["wandb_run_id"] = preemption_data.get("wandb_run_id", "")
+        logger.info(f"Loaded preemption data from {preemption_file}")
+        return preemption_data
+
+
+def _save_preemption_data(config: EvalConfig, trial_id: int, output_dir: Path, wandb_run_id: str = "") -> None:
+    preemption_file = config.output_dir / f"preemption_trial_{trial_id}.json"
+    with open(preemption_file, "w") as f:
+        json.dump({"output_dir": str(output_dir), "wandb_run_id": wandb_run_id}, f)
+
+
+def _delete_preemption_file(config: EvalConfig, trial_id: int) -> None:
+    preemption_file = config.output_dir / f"preemption_trial_{trial_id}.json"
+    if preemption_file.is_file():
+        preemption_file.unlink()
+        logger.info(f"Deleted preemption file: {preemption_file}")
+    else:
+        logger.info(f"No preemption file found to delete: {preemption_file}")
+    logger.info(f"Saved preemption data to {preemption_file}")
+
+
+def _configure_logging(output_dir: Path) -> None:
+    """Configure logging to save logs to a file in the output directory."""
+
+    # Ensure the output directory exists
+    output_dir.mkdir(parents=True, exist_ok=True)
+
+    # Set up log file path
+    log_file = output_dir / "evaluation.log"
+
+    # Create file handler
+    file_handler = logging.FileHandler(log_file, mode="w", encoding="utf-8")
+    file_handler.setLevel(logging.INFO)
+
+    # Create formatter
+    formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s", datefmt="%Y-%m-%d %H:%M:%S")
+    file_handler.setFormatter(formatter)
+
+    # Get the root logger and add the file handler
+    root_logger = logging.getLogger()
+
+    # Remove existing file handlers to avoid duplicates
+    for handler in root_logger.handlers[:]:
+        if isinstance(handler, logging.FileHandler):
+            root_logger.removeHandler(handler)
+
+    root_logger.addHandler(file_handler)
+
+    # Set logging level if not already set
+    if root_logger.level == logging.NOTSET:
+        root_logger.setLevel(logging.INFO)
+
+
+def _wandb_mode(project: str | None) -> Literal["online", "disabled"] | None:
+    """
+    Checks to see if a WandB API key is found. If not, wandb starts in offline mode.
+    """
+    if project is None:
+        logger.warning("No WandB project specified, disabling logging.")
+        return "disabled"
+    else:
+        try:
+            api_key = wandb.api.api_key
+            if api_key is None:
+                logger.warning(
+                    """No wandb API key found. Disabling Wandb logging.
+                    If you have a WandB account set the environment variable 'WANDB_API_KEY'"""
+                )
+                return "disabled"
+            else:
+                logger.info("wandb login detected. Using online mode.")
+        except Exception as e:
+            logger.warning(f"wandb login check failed: {e}. Disabling Wandb logging.")
+            return "disabled"
+    return "online"

eval_framework/metrics/base.py

@@ -0,0 +1,40 @@
+from abc import ABC, abstractmethod
+from typing import Any
+
+from pydantic import BaseModel, ConfigDict
+
+from eval_framework.shared.types import Error
+
+
+class MetricResult(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+    metric_name: str
+    value: float | None
+    higher_is_better: bool
+    llm_judge_prompt: str | None = None
+    llm_judge_response: str | None = None
+    code_execution_trace: str | None = None
+    error: Error | None = None
+
+
+class classproperty:
+    def __init__(self, method: Any) -> None:
+        self.method = method
+
+    def __get__(self, instance: Any, cls: Any) -> Any:
+        return self.method(cls)
+
+
+class BaseMetric[Response](ABC):
+    NAME: str
+    KEYS: list[str] | None = None
+
+    @classproperty
+    def NAMES(cls) -> list[str]:
+        if cls.KEYS is None:
+            return [cls.NAME]
+        return [f"{cls.NAME}/{k}" for k in cls.KEYS]
+
+    @abstractmethod
+    def calculate(self, response: Response) -> list[MetricResult]:
+        raise NotImplementedError

eval_framework/metrics/completion/__init__.py

@@ -0,0 +1 @@
+from .accuracy_completion import AccuracyCompletion

eval_framework/metrics/completion/accuracy_completion.py

@@ -0,0 +1,16 @@
+from eval_framework.metrics.base import BaseMetric, MetricResult
+from eval_framework.shared.types import Completion
+
+
+class AccuracyCompletion(BaseMetric[Completion]):
+    NAME = "Accuracy Completion"
+
+    def calculate(self, response: Completion) -> list[MetricResult]:
+        if response.error is not None:
+            return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=response.error)]
+
+        ground_truths = response.ground_truth_list
+        is_correct = any(response.completion == gt for gt in ground_truths)
+        return [
+            MetricResult(metric_name=self.NAME, value=float(is_correct), higher_is_better=True, error=response.error)
+        ]