eval-framework 0.2.12__tar.gz → 0.2.14__tar.gz

{eval_framework-0.2.12 → eval_framework-0.2.14}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: eval-framework
-Version: 0.2.12
+Version: 0.2.14
 Summary: Evalulation Framework
 Author: Aleph Alpha Research
 License:                                  Apache License
@@ -377,7 +377,7 @@ Subset of core capabilities benchmarks coverd by `eval-framework`:
 
 | **Reasoning** | **Knowledge** | **Math** | **Coding** | **Structured outputs** | **Long Context** |
 |---------------|---------------|----------|------------|------------------------|------------------|
-| COPA | ARC | AIME | BigCodeBench | IFEval | InfiniteBench |
+| COPA, BalancedCOPA | ARC | AIME | BigCodeBench | IFEval | InfiniteBench |
 | Hellaswag | MMLU | GSM8K | HumanEval | StructEval | QUALITY |
 | Winogrande | Openbook QA| MATH-500 | MBPP | | ZeroSCROLLS |
 

{eval_framework-0.2.12 → eval_framework-0.2.14}/README.md

@@ -106,7 +106,7 @@ Subset of core capabilities benchmarks coverd by `eval-framework`:
 
 | **Reasoning** | **Knowledge** | **Math** | **Coding** | **Structured outputs** | **Long Context** |
 |---------------|---------------|----------|------------|------------------------|------------------|
-| COPA | ARC | AIME | BigCodeBench | IFEval | InfiniteBench |
+| COPA, BalancedCOPA | ARC | AIME | BigCodeBench | IFEval | InfiniteBench |
 | Hellaswag | MMLU | GSM8K | HumanEval | StructEval | QUALITY |
 | Winogrande | Openbook QA| MATH-500 | MBPP | | ZeroSCROLLS |
 

{eval_framework-0.2.12 → eval_framework-0.2.14}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "eval-framework"
-version = "0.2.12"
+version = "0.2.14"
 description = "Evalulation Framework"
 readme = "README.md"
 license = { file = "LICENSE" }
@@ -105,6 +105,7 @@ dev = [
   "types-requests>=2.32.0.20250328,<3",
   "plotly>=5.24.1,<6",
   "ruff>=0.12.8",
+  "scipy>=1.14.0,<2",  # for tests comparing our Hungarian implementation to scipy
 ]
 flash-attn = [
   "flash-attn>=2.7.2.post1,<2.8",
@@ -178,6 +179,7 @@ addopts = "-p 'no:legacypath' --doctest-modules"
 markers = [
     "gpu: needs a GPU runner, otherwise test can not be run",
     "cpu_slow: runs for a long time (on CPU)",
+    "slow_download: smoke tests that download large datasets (>15s); excluded from CI, run manually with -m slow_download",
     "external_api: needs external services for execution",
     "vllm: tests that specifically require vLLM functionality",
     "formatter_hash: formatter consistency tests using hash comparisons",

{eval_framework-0.2.12 → eval_framework-0.2.14}/src/eval_framework/evaluation_generator.py

@@ -134,16 +134,27 @@ class EvaluationGenerator:
             # filter and count errors
             total_count = len(data_subset)
 
-            mask = data["error"].isnull()
+            mask = data_subset["error"].isnull()
             data_subset_error_free = data_subset.loc[mask, ["subject", "key", "value"]]
-            # data_subset_error_free = data_subset[data_subset["error"].isnull()][["subject", "key", "value"]]
 
-            aggregated_results[f"ErrorFreeRatio {metric}"] = float(len(data_subset_error_free) / total_count)
+            error_free_ratio = float(len(data_subset_error_free) / total_count)
+            aggregated_results[f"ErrorFreeRatio {metric}"] = error_free_ratio
 
             # aggregate by key and subject first to have equal weights for all key / subject combinations
             key_subject_mean = data_subset_error_free.groupby(["key", "subject"]).mean()
             aggregated_results[f"Average {metric}"] = float(key_subject_mean[["value"]].mean()["value"])
 
+            if error_free_ratio < 1.0:
+                # Treat error samples (with value=None) as 0 for the "including errors" average
+                data_subset_with_errors = data_subset[["key", "subject", "value", "error"]].copy()
+                # Only fill value with 0 where there's an error (not for all None values)
+                error_mask = data_subset_with_errors["error"].notna()
+                data_subset_with_errors.loc[error_mask, "value"] = data_subset_with_errors.loc[
+                    error_mask, "value"
+                ].fillna(0.0)
+                key_subject_mean_with_errors = data_subset_with_errors.groupby(["key", "subject"])["value"].mean()
+                aggregated_results[f"Average {metric} (including Errors)"] = float(key_subject_mean_with_errors.mean())
+
             std_err_mean_sum_of_squares = 0.0
             std_err_mean_total_num_samples = 0.0
             std_err_mean_num_subjects = 0
@@ -156,14 +167,29 @@ class EvaluationGenerator:
                         # group = data_subset[data[column] == name][["subject", "key", "value", "error"]]
                         group_total_count = len(group)
                         group_error_free = group[group["error"].isnull()][["subject", "key", "value"]]
-                        aggregated_results[f"ErrorFreeRatio {metric} - {name[0]}"] = float(
-                            len(group_error_free) / group_total_count
-                        )
+                        group_error_free_ratio = float(len(group_error_free) / group_total_count)
+                        aggregated_results[f"ErrorFreeRatio {metric} - {name[0]}"] = group_error_free_ratio
 
                         group_key_subject_mean = group_error_free.groupby(["key", "subject"]).mean()
                         value = float(group_key_subject_mean[["value"]].mean()["value"])
                         aggregated_results[f"Average {metric} - {name[0]}"] = value if not math.isnan(value) else None
 
+                        if group_error_free_ratio < 1.0:
+                            # Treat error samples (with value=None) as 0 for the "including errors" average
+                            group_with_errors = group[["key", "subject", "value", "error"]].copy()
+                            # Only fill value with 0 where there's an error (not for all None values)
+                            error_mask = group_with_errors["error"].notna()
+                            group_with_errors.loc[error_mask, "value"] = group_with_errors.loc[
+                                error_mask, "value"
+                            ].fillna(0.0)
+                            group_key_subject_mean_with_errors = group_with_errors.groupby(["key", "subject"])[
+                                "value"
+                            ].mean()
+                            value_with_errors = float(group_key_subject_mean_with_errors.mean())
+                            aggregated_results[f"Average {metric} (including Errors) - {name[0]}"] = (
+                                value_with_errors if not math.isnan(value_with_errors) else None
+                            )
+
                         if not ("SequencePositions" in metric or "Bytes" in metric):
                             # calculate standard error for selected  metrics
                             group_key_subject_std = group_error_free.groupby(["key", "subject"]).std()

eval_framework-0.2.14/src/eval_framework/external/drop_process_results.py

@@ -0,0 +1,250 @@
+"""DROP F1 and exact match evaluation.
+
+Logic adapted from AllenNLP DROP evaluation:
+https://github.com/allenai/allennlp-reading-comprehension/blob/master/allennlp_rc/eval/drop_eval.py
+"""
+
+import re
+import string
+
+import numpy as np
+
+_ARTICLES = re.compile(r"\b(a|an|the)\b", re.UNICODE)
+
+
+def _linear_sum_assignment(cost_matrix: np.ndarray) -> tuple[np.ndarray, np.ndarray]:
+    """Solve the linear sum assignment problem (minimize cost) using the Hungarian algorithm.
+
+    Pure NumPy implementation to avoid scipy dependency. Returns (row_ind, col_ind) with row_ind sorted,
+    matching scipy.optimize.linear_sum_assignment API for rectangular matrices.
+    """
+    cost = np.atleast_2d(np.asarray(cost_matrix, dtype=np.float64))
+    n_rows, n_cols = cost.shape
+    n = max(n_rows, n_cols)
+    # Pad to square with zeros so unassigned rows/cols contribute zero cost
+    C = np.zeros((n, n))
+    C[:n_rows, :n_cols] = cost
+
+    # Row and column reductions
+    u = np.min(C, axis=1)
+    u[u == np.inf] = 0
+    C = C - u[:, np.newaxis]
+    v = np.min(C, axis=0)
+    v[v == np.inf] = 0
+    C = C - v[np.newaxis, :]
+
+    # Starred zeros: assignment (1 = starred). Start with no stars.
+    star = np.zeros((n, n), dtype=np.intp)
+    row_covered = np.zeros(n, dtype=bool)
+    col_covered = np.zeros(n, dtype=bool)
+    prime = np.zeros((n, n), dtype=np.intp)
+
+    def find_zero() -> tuple[int, int] | None:
+        for i in range(n):
+            if row_covered[i]:
+                continue
+            for j in range(n):
+                if not col_covered[j] and C[i, j] == 0:
+                    return (i, j)
+        return None
+
+    def star_in_row(i: int) -> int | None:
+        for j in range(n):
+            if star[i, j]:
+                return j
+        return None
+
+    def star_in_col(j: int) -> int | None:
+        for i in range(n):
+            if star[i, j]:
+                return i
+        return None
+
+    def prime_in_row(i: int) -> int | None:
+        for j in range(n):
+            if prime[i, j]:
+                return j
+        return None
+
+    while True:
+        # Cover columns containing a starred zero
+        col_covered[:] = False
+        for j in range(n):
+            for i in range(n):
+                if star[i, j]:
+                    col_covered[j] = True
+                    break
+        if np.all(col_covered):
+            break
+        prime[:] = 0
+        row_covered[:] = False
+        while True:
+            z = find_zero()
+            if z is None:
+                # No uncovered zero: find minimum uncovered value and adjust
+                min_val = np.inf
+                for i in range(n):
+                    if not row_covered[i]:
+                        for j in range(n):
+                            if not col_covered[j] and C[i, j] < min_val:
+                                min_val = C[i, j]
+                if min_val == np.inf or min_val <= 0:
+                    min_val = 1e-10
+                for i in range(n):
+                    if row_covered[i]:
+                        C[i, :] += min_val
+                for j in range(n):
+                    if not col_covered[j]:
+                        C[:, j] -= min_val
+                continue
+            i, j = z
+            prime[i, j] = 1
+            cj = star_in_row(i)
+            if cj is None:
+                # Augmenting path: unstar starred, star primed along path
+                path = [(i, j)]
+                while True:
+                    ji = star_in_col(path[-1][1])
+                    if ji is None:
+                        break
+                    path.append((ji, path[-1][1]))
+                    pj = prime_in_row(ji)
+                    if pj is None:
+                        break
+                    path.append((ji, pj))
+                for pi, pj in path:
+                    star[pi, pj] = 1 - star[pi, pj]
+                prime[:] = 0
+                row_covered[:] = False
+                col_covered[:] = False
+                break
+            row_covered[i] = True
+            col_covered[cj] = False
+
+    # Extract assignment: (row_ind, col_ind) for starred zeros, row_ind sorted
+    row_ind = np.array([i for i in range(n) for j in range(n) if star[i, j]], dtype=np.intp)
+    col_ind = np.array([j for i in range(n) for j in range(n) if star[i, j]], dtype=np.intp)
+    # Keep only assignments within original matrix
+    mask = (row_ind < n_rows) & (col_ind < n_cols)
+    row_ind = row_ind[mask]
+    col_ind = col_ind[mask]
+    # Sort by row index (scipy API)
+    perm = np.argsort(row_ind)
+    return row_ind[perm], col_ind[perm]
+
+
+def process_results(doc: dict, results: list) -> dict[str, float]:
+    """Compute DROP exact_match and F1 between predictions and gold answers.
+
+    doc["answers"] is a list of gold answers (each a tuple or list of strings).
+    results is a list of predicted answers (one per prediction; for one completion use [pred]).
+    """
+    preds, golds = results, doc["answers"]
+    max_em = 0.0
+    max_f1 = 0.0
+    for gold_answer in golds:
+        exact_match, f1_score = get_metrics(preds, gold_answer)
+        if gold_answer and (gold_answer[0].strip() if isinstance(gold_answer[0], str) else True):
+            max_em = max(max_em, exact_match)
+            max_f1 = max(max_f1, f1_score)
+    return {"exact_match": max_em, "f1": max_f1}
+
+
+def get_metrics(predicted: list | str, gold: tuple | list) -> tuple[float, float]:
+    """Return (exact_match, f1) for one gold answer. predicted and gold can be string or list of strings."""
+    predicted_bags = _answer_to_bags(predicted)
+    gold_bags = _answer_to_bags(gold)
+
+    if set(predicted_bags[0]) == set(gold_bags[0]) and len(predicted_bags[0]) == len(gold_bags[0]):
+        exact_match = 1.0
+    else:
+        exact_match = 0.0
+
+    f1_per_bag = _align_bags(predicted_bags[1], gold_bags[1])
+    f1 = float(np.mean(f1_per_bag))
+    f1 = round(f1, 2)
+    return exact_match, f1
+
+
+def _answer_to_bags(answer: list | tuple | str) -> tuple[list[str], list[set]]:
+    if isinstance(answer, list | tuple):
+        raw_spans = list(answer)
+    else:
+        raw_spans = [answer]
+    normalized_spans = []
+    token_bags = []
+    for raw_span in raw_spans:
+        normalized_span = _normalize(str(raw_span))
+        normalized_spans.append(normalized_span)
+        token_bags.append(set(normalized_span.split()))
+    return normalized_spans, token_bags
+
+
+def _align_bags(predicted: list[set], gold: list[set]) -> np.ndarray:
+    scores = np.zeros([len(gold), len(predicted)])
+    for gold_index, gold_item in enumerate(gold):
+        for pred_index, pred_item in enumerate(predicted):
+            if _match_numbers_if_present(gold_item, pred_item):
+                scores[gold_index, pred_index] = _compute_f1(pred_item, gold_item)
+    row_ind, col_ind = _linear_sum_assignment(-scores)
+
+    max_scores = np.zeros([max(len(gold), len(predicted))])
+    for row, column in zip(row_ind, col_ind):
+        max_scores[row] = max(max_scores[row], scores[row, column])
+    return max_scores
+
+
+def _compute_f1(predicted_bag: set, gold_bag: set) -> float:
+    intersection = len(gold_bag.intersection(predicted_bag))
+    if not predicted_bag:
+        precision = 1.0
+    else:
+        precision = intersection / float(len(predicted_bag))
+    if not gold_bag:
+        recall = 1.0
+    else:
+        recall = intersection / float(len(gold_bag))
+    return (2 * precision * recall) / (precision + recall) if (precision or recall) else 0.0
+
+
+def _match_numbers_if_present(gold_bag: set, predicted_bag: set) -> bool:
+    gold_numbers = {w for w in gold_bag if _is_number(w)}
+    predicted_numbers = {w for w in predicted_bag if _is_number(w)}
+    return (not gold_numbers) or bool(gold_numbers.intersection(predicted_numbers))
+
+
+def _is_number(text: str) -> bool:
+    try:
+        float(text)
+        return True
+    except ValueError:
+        return False
+
+
+def _remove_articles(text: str) -> str:
+    return _ARTICLES.sub(" ", text)
+
+
+def _white_space_fix(text: str) -> str:
+    return " ".join(text.split())
+
+
+def _remove_punc(text: str) -> str:
+    exclude = set(string.punctuation)
+    return "".join(ch for ch in text if ch not in exclude) if not _is_number(text) else text
+
+
+def _fix_number(text: str) -> str:
+    return str(float(text)) if _is_number(text) else text
+
+
+def _tokenize(text: str) -> list[str]:
+    return re.split(" |-", text)
+
+
+def _normalize(answer: str) -> str:
+    tokens = [
+        _white_space_fix(_remove_articles(_fix_number(_remove_punc(token.lower())))) for token in _tokenize(answer)
+    ]
+    tokens = [t for t in tokens if t.strip()]
+    return " ".join(tokens).strip()

{eval_framework-0.2.12 → eval_framework-0.2.14}/src/eval_framework/llm/aleph_alpha.py

@@ -200,11 +200,16 @@ class AlephAlphaAPIModel(BaseLLM):
         stop_sequences: list[str] | None = None,
         max_tokens: int | None = None,
         temperature: float | None = None,
+        top_p: float | None = None,
     ) -> list[RawCompletion]:
         effective_temperature = temperature if temperature is not None else self._temperature
+        if effective_temperature is not None and not (0 <= effective_temperature <= 2):
+            raise ValueError(f"temperature must be between 0 and 2, got {effective_temperature}")
+        effective_top_p = top_p if top_p is not None else self._top_p
+        if effective_top_p is not None and not (0 < effective_top_p <= 1):
+            raise ValueError(f"top_p must be between 0 and 1 (exclusive), got {effective_top_p}")
 
         requests: list[CompletionRequest] = []
-
         # Adjust max tokens based on bytes_per_token_scalar so that non-standard models generate full responses
         scaled_max_tokens = math.ceil(max_tokens * self.bytes_per_token_scalar) if max_tokens is not None else None
 
@@ -215,7 +220,7 @@ class AlephAlphaAPIModel(BaseLLM):
                     maximum_tokens=scaled_max_tokens,
                     stop_sequences=stop_sequences,
                     temperature=effective_temperature,
-                    top_p=self._top_p,
+                    top_p=effective_top_p,
                 )
             )
 

{eval_framework-0.2.12 → eval_framework-0.2.14}/src/eval_framework/llm/base.py

@@ -24,6 +24,7 @@ class BaseLLM(ABC):
         stop_sequences: list[str] | None = None,
         max_tokens: int | None = None,
         temperature: float | None = None,
+        top_p: float | None = None,
     ) -> list[RawCompletion]:
         """
         stop_sequences and max_tokens are injected by the task if exist. They should be overwritten or
@@ -47,6 +48,7 @@ class BaseLLM(ABC):
         stop_sequences: list[str] | None = None,
         max_tokens: int | None = None,
         temperature: float | None = None,
+        top_p: float | None = None,
     ) -> list[RawCompletion]:
         """
         stop_sequences and max_tokens are injected by the task if exist. They should be overwritten or
@@ -79,6 +81,7 @@ class BaseLLM(ABC):
         stop_sequences: list[str] | None = None,
         max_tokens: int | None = None,
         temperature: float | None = None,
+        top_p: float | None = None,
     ) -> list[RawCompletion]:
         """Generates a model response for each sample.
 
@@ -86,10 +89,10 @@ class BaseLLM(ABC):
         otherwise falls back to 'generate_from_messages'.
         """
         try:
-            return self.generate_from_samples(samples, stop_sequences, max_tokens, temperature)
+            return self.generate_from_samples(samples, stop_sequences, max_tokens, temperature, top_p)
         except NotImplementedError:
             messages: list[Sequence[Message]] = [sample.messages for sample in samples]
-            return self.generate_from_messages(messages, stop_sequences, max_tokens, temperature)
+            return self.generate_from_messages(messages, stop_sequences, max_tokens, temperature, top_p)
 
     def post_process_completion(self, completion: str, sample: Sample) -> str:
         """

{eval_framework-0.2.12 → eval_framework-0.2.14}/src/eval_framework/llm/huggingface.py

@@ -10,7 +10,14 @@ from typing import Any
 
 import torch
 from tokenizers import Tokenizer
-from transformers import AutoModelForCausalLM, AutoTokenizer, StoppingCriteria, StoppingCriteriaList
+from transformers import (
+    AutoModelForCausalLM,
+    AutoTokenizer,
+    StoppingCriteria,
+    StoppingCriteriaList,
+)
+from transformers.models.gpt2 import GPT2Tokenizer
+from transformers.tokenization_utils import PreTrainedTokenizerBase
 
 from eval_framework.llm.base import BaseLLM
 from eval_framework.shared.types import (
@@ -83,9 +90,13 @@ class BaseHFLLM(BaseLLM):
     SEQ_LENGTH: int | None = None
     BYTES_PER_TOKEN: float = 4.0  # rule of thumb according to https://platform.openai.com/tokenizer
 
+    def _load_tokenizer(self) -> PreTrainedTokenizerBase:
+        """Load the tokenizer. Override in subclasses to use a specific tokenizer class."""
+        return AutoTokenizer.from_pretrained(self.LLM_NAME)
+
     def __init__(self, formatter: BaseFormatter | None = None, bytes_per_token: float | None = None) -> None:
         self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
-        self.tokenizer = AutoTokenizer.from_pretrained(self.LLM_NAME)
+        self.tokenizer = self._load_tokenizer()
         self.model = AutoModelForCausalLM.from_pretrained(self.LLM_NAME, device_map="auto")
         logger.info(f"{RED}[ Model initialized --------------------- {RESET}{self.LLM_NAME} {RED}]{RESET}")
         self._set_formatter(formatter)
@@ -135,6 +146,7 @@ class BaseHFLLM(BaseLLM):
         stop_sequences: list[str] | None = None,
         max_tokens: int | None = None,
         temperature: float | None = None,
+        top_p: float | None = None,
     ) -> list[RawCompletion]:
         if temperature is None:
             effective_temperature = 0.0  # Current default, TODO: refactor to use model's default
@@ -143,7 +155,8 @@ class BaseHFLLM(BaseLLM):
             )
         else:
             effective_temperature = temperature
-
+        if top_p is not None:
+            logger.warning("Huggingface LLM does not support top_p. Ignoring top_p value.")
         raw_completions = []
         for single_messages in messages:
             # format
@@ -403,14 +416,24 @@ class Pythia410m(HFLLM):
 
 
 class SmolLM135M(HFLLM):
+    """SmolLM-135M uses a GPT2-style tokenizer; AutoTokenizer can incorrectly select LlamaTokenizer."""
+
     LLM_NAME = "HuggingFaceTB/SmolLM-135M"
     DEFAULT_FORMATTER = ConcatFormatter
 
+    def _load_tokenizer(self) -> PreTrainedTokenizerBase:
+        return GPT2Tokenizer.from_pretrained(self.LLM_NAME)
+
 
 class Smollm135MInstruct(HFLLM):
+    """SmolLM-135M-Instruct uses a GPT2-style tokenizer; AutoTokenizer can incorrectly select LlamaTokenizer."""
+
     LLM_NAME = "HuggingFaceTB/SmolLM-135M-Instruct"
     DEFAULT_FORMATTER = partial(HFFormatter, LLM_NAME)
 
+    def _load_tokenizer(self) -> PreTrainedTokenizerBase:
+        return GPT2Tokenizer.from_pretrained(self.LLM_NAME)
+
 
 class Qwen3_0_6B(HFLLM):
     LLM_NAME = "Qwen/Qwen3-0.6B"

{eval_framework-0.2.12 → eval_framework-0.2.14}/src/eval_framework/llm/openai.py

@@ -34,6 +34,7 @@ class OpenAIModel(BaseLLM):
         model_name: str | None = None,
         formatter: BaseFormatter | None = None,
         temperature: float | None = None,
+        top_p: float | None = None,
         api_key: str | None = os.getenv("OPENAI_API_KEY", ""),
         organization: str | None = None,
         base_url: str | None = None,
@@ -46,6 +47,7 @@ class OpenAIModel(BaseLLM):
             model_name: OpenAI model name (e.g., "gpt-4o", "gpt-3.5-turbo"). If None, uses LLM_NAME class attribute.
             formatter: Optional message formatter.
             temperature: Sampling temperature used when not passed to generate methods (from 0.0 to 2.0).
+            top_p: Nucleus sampling probability mass (from 0.0 to 1.0). If None, the API default is used.
             api_key: OpenAI API key (defaults to OPENAI_API_KEY env variable).
             organization: Optional OpenAI organization ID.
             base_url: Optional API base URL for Azure or alternate endpoints.
@@ -59,6 +61,10 @@ class OpenAIModel(BaseLLM):
         self._temperature = temperature if temperature is not None else 0.0
         assert 0.0 <= self._temperature <= 2.0, "Temperature must be between 0.0 and 2.0"
 
+        if top_p is not None:
+            assert 0.0 <= top_p <= 1.0, "top_p must be between 0.0 and 1.0"
+        self._top_p = top_p
+
         self._client = OpenAI(
             api_key=api_key,
             organization=organization,
@@ -97,6 +103,7 @@ class OpenAIModel(BaseLLM):
         stop_sequences: list[str] | None = None,
         max_tokens: int | None = None,
         temperature: float | None = None,
+        top_p: float | None = None,
     ) -> list[RawCompletion]:
         """
         Generate completions for a list of message sequences concurrently.
@@ -108,6 +115,7 @@ class OpenAIModel(BaseLLM):
             stop_sequences: Optional list of stop sequences.
             max_tokens: Optional maximum number of tokens to generate.
             temperature: Sampling temperature.
+            top_p: Nucleus sampling probability mass (0.0 to 1.0). Overrides instance default if provided.
 
         Returns:
             List of RawCompletion objects containing prompts and completions.
@@ -116,6 +124,10 @@ class OpenAIModel(BaseLLM):
         effective_temperature = temperature if temperature is not None else self._temperature
         assert 0.0 <= effective_temperature <= 2.0, "Temperature must be between 0.0 and 2.0"
 
+        effective_top_p = top_p if top_p is not None else self._top_p
+        if effective_top_p is not None:
+            assert 0.0 <= effective_top_p <= 1.0, "top_p must be between 0.0 and 1.0"
+
         def _process_one(single_messages: Sequence[Message]) -> RawCompletion:
             # Adjust max tokens based on bytes_per_token_scalar so that non-standard models generate full responses
             scaled_max_tokens = math.ceil(max_tokens * self.bytes_per_token_scalar) if max_tokens is not None else None
@@ -129,6 +141,7 @@ class OpenAIModel(BaseLLM):
                     model=self._model_name,
                     prompt=prompt,
                     temperature=effective_temperature,
+                    top_p=effective_top_p,
                     max_tokens=scaled_max_tokens,
                     stop=stop_sequences,
                 )
@@ -158,6 +171,7 @@ class OpenAIModel(BaseLLM):
                     model=self._model_name,
                     messages=chat_messages,
                     temperature=effective_temperature,
+                    top_p=effective_top_p,
                     max_tokens=scaled_max_tokens,
                     stop=stop_sequences,
                 )
@@ -300,6 +314,7 @@ class OpenAIEmbeddingModel(BaseLLM):
         stop_sequences: list[str] | None = None,
         max_tokens: int | None = None,
         temperature: float | None = None,
+        top_p: float | None = None,
     ) -> list[RawCompletion]:
         raise NotImplementedError(
             "Embedding model does not support generate_from_messages. Use generate_embeddings instead."

{eval_framework-0.2.12 → eval_framework-0.2.14}/src/eval_framework/llm/vllm.py

@@ -134,11 +134,12 @@ class BaseVLLMModel(BaseLLM):
             **kwargs,
         }
 
-        device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu")
-
         self.batch_size = batch_size
 
-        self.model = LLM(**model_args, device=device)
+        if "VLLM_TARGET_DEVICE" not in os.environ and not torch.cuda.is_available():
+            os.environ["VLLM_TARGET_DEVICE"] = "cpu"
+
+        self.model = LLM(**model_args)
 
         self._tokenizer: None | VLLMTokenizerAPI = None
         _ = self.tokenizer  # make sure tokenizer is initialized
@@ -225,6 +226,7 @@ class BaseVLLMModel(BaseLLM):
         stop_sequences: list[str] | None = None,
         max_tokens: int | None = None,
         temperature: float | None = None,
+        top_p: float | None = None,
     ) -> list[RawCompletion]:
         raw_completions: list[RawCompletion | None] = [None] * len(messages)
         prompt_objs = []
@@ -234,7 +236,7 @@ class BaseVLLMModel(BaseLLM):
         scaled_max_tokens = math.ceil(max_tokens * self.bytes_per_token_scalar) if max_tokens is not None else None
 
         sampling_params = self._resolve_sampling_params(
-            self.sampling_params, scaled_max_tokens, stop_sequences, temperature
+            self.sampling_params, scaled_max_tokens, stop_sequences, temperature, top_p
         )
 
         for i, single_messages in enumerate(messages):
@@ -294,6 +296,7 @@ class BaseVLLMModel(BaseLLM):
         max_tokens: int | None,
         stop_sequences: list[str] | None,
         temperature: float | None,
+        top_p: float | None = None,
     ) -> SamplingParams:
         sampling_params.max_tokens = max_tokens
         sampling_params.stop = stop_sequences
@@ -307,6 +310,13 @@ class BaseVLLMModel(BaseLLM):
                 f"Using sampling params temperature value: {sampling_params.temperature} "
                 f"as no custom temperature value was provided"
             )
+        if top_p is not None:
+            logger.warning(f"Overriding sampling params top_p {sampling_params.top_p} with custom value {top_p}")
+            sampling_params.top_p = top_p
+        else:
+            logger.info(
+                f"Using sampling params top_p value: {sampling_params.top_p} as no custom top_p value was provided"
+            )
         return sampling_params
 
     def _model_generate(