eval-framework 0.2.9__tar.gz → 0.2.11__tar.gz

{eval_framework-0.2.9 → eval_framework-0.2.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: eval-framework
-Version: 0.2.9
+Version: 0.2.11
 Summary: Evalulation Framework
 Author: Aleph Alpha Research
 License:                                  Apache License
@@ -238,7 +238,7 @@ Requires-Dist: numpy>=1.26.4
 Requires-Dist: antlr4-python3-runtime==4.11.0
 Requires-Dist: accelerate ; extra == 'accelerate'
 Requires-Dist: eval-framework[determined,api,openai,transformers,accelerate,vllm,comet,optional,mistral] ; extra == 'all'
-Requires-Dist: aleph-alpha-client>=10,<11 ; extra == 'api'
+Requires-Dist: aleph-alpha-client>=11.5.1 ; extra == 'api'
 Requires-Dist: unbabel-comet>=2.2.6,<3 ; extra == 'comet'
 Requires-Dist: determined>=0.38,<0.39 ; extra == 'determined'
 Requires-Dist: tensorboard==2.19.0 ; extra == 'determined'

{eval_framework-0.2.9 → eval_framework-0.2.11}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "eval-framework"
-version = "0.2.9"
+version = "0.2.11"
 description = "Evalulation Framework"
 readme = "README.md"
 license = { file = "LICENSE" }
@@ -53,7 +53,7 @@ determined = [
   "determined>=0.38,<0.39",
   "tensorboard==2.19.0"
 ]
-api = ["aleph-alpha-client>=10,<11"]
+api = ["aleph-alpha-client>=11.5.1"]
 openai = [
   "openai>=1.62,<2.8",
   "tiktoken>=0.9,<0.10",

{eval_framework-0.2.9 → eval_framework-0.2.11}/src/eval_framework/llm/aleph_alpha.py

@@ -3,16 +3,12 @@ import json
 import logging
 import math
 import os
-import random
 import re
-import time
 import traceback
 from collections.abc import Callable, Sequence
 
-import aiohttp
 from aleph_alpha_client import (
     AsyncClient,
-    BusyError,
     Client,
     CompletionRequest,
     CompletionResponse,
@@ -53,7 +49,6 @@ class AlephAlphaAPIModel(BaseLLM):
         max_retries: int = 100,
         max_async_concurrent_requests: int = 32,
         request_timeout_seconds: int = 30 * 60 + 5,
-        queue_full_timeout_seconds: int = 30 * 60 + 5,
         bytes_per_token: float | None = None,
         token: str = os.getenv("AA_TOKEN", "dummy"),
         base_url: str = os.getenv("AA_INFERENCE_ENDPOINT", "dummy_endpoint"),
@@ -70,7 +65,6 @@ class AlephAlphaAPIModel(BaseLLM):
         self.max_async_concurrent_requests = max_async_concurrent_requests
         self.max_retries = max_retries
         self.request_timeout_seconds = request_timeout_seconds
-        self.queue_full_timeout_seconds = queue_full_timeout_seconds
         self.token = token
         self.base_url = base_url
         self._validate_model_availability(base_url, token)
@@ -101,56 +95,6 @@ class AlephAlphaAPIModel(BaseLLM):
         except Exception as e:
             raise RuntimeError(f"Model '{self._llm_name}' is not available: {e}")
 
-    async def _request_with_backoff(
-        self, client: AsyncClient, request: CompletionRequest, id: int
-    ) -> CompletionResponse:
-        """
-        Query Aleph-Alpha API with complete. Retry with back-off until it responds.
-        """
-        num_attempts = 0
-        start_time: float | None = None
-
-        while True:
-            try:
-                return await client.complete(request, model=self._llm_name)
-
-            except (TimeoutError, BusyError, RuntimeError, aiohttp.ClientError) as e:
-                status_code: str = safe_json_loads(e.args[1]).get("code", "") if len(e.args) >= 2 else ""
-                str_e = str(e)
-                if status_code == "QUEUE_FULL":
-                    # Worker not available or missed a heartbeat (inference longer than scheduler's
-                    # API_MODEL_AVAILABLE_TIMEOUT_DURATION_MILLIS) or the scheduler is overloaded.
-                    if start_time is None:
-                        start_time = time.time()
-                    elapsed = time.time() - start_time
-                    if elapsed <= self.queue_full_timeout_seconds:
-                        logger.info(
-                            f"Request {id}: {status_code or str_e[:256]} - retrying: attempt"
-                            f" {num_attempts}/{self.max_retries}, elapsed {elapsed:.1f} sec"
-                        )
-                        # don't count as retry (request returns immediately, so just wait a bit not to DoS the server)
-                        await asyncio.sleep(random.randint(5, 30))
-                        continue
-
-                elif (
-                    status_code == "TIMEOUT_TASK"
-                    or isinstance(e, TimeoutError)
-                    or "502 Bad Gateway" in str_e
-                    or "504 Gateway Time-out" in str_e
-                    or isinstance(e, aiohttp.ClientError)
-                ):
-                    # client timeout, either because task too long in a queue or inference too long
-                    # (scheduler's API_CLIENT_TIMEOUT_DURATION_MILLIS). Retrying for the "inference too long"
-                    # case makes no sense but we unfortunately don't know which case has happened.
-                    num_attempts += 1
-                    start_time = None
-                    if num_attempts < self.max_retries:
-                        logger.info(f"Request {id}: TIMEOUT_TASK - retrying: attempt {num_attempts}/{self.max_retries}")
-                        await asyncio.sleep(random.randint(5, 30))
-                        continue
-
-                raise e
-
     def _error_from_exception(self, e: Exception) -> Error:
         """Convert an exception to an Error object."""
         if len(e.args) >= 2:
@@ -171,39 +115,36 @@ class AlephAlphaAPIModel(BaseLLM):
     async def _process_request_with_client(
         self,
         client: AsyncClient,
-        semaphore: asyncio.Semaphore,
         request: CompletionRequest,
         id: int,
     ) -> tuple[CompletionRequest, CompletionResponse | Error]:
         """Process a single request, returning the request and either a response or error."""
-        async with semaphore:
-            try:
-                response = await self._request_with_backoff(client=client, request=request, id=id)
-                logger.info(f"Request {id}: Success")
-                return (request, response)
-            except Exception as e:
-                if raise_errors():
-                    raise e
-                logger.info(f"Request {id}: Failure: {str(e)[:256]}")
-                return (request, self._error_from_exception(e))
+        try:
+            response = await client.complete(request, model=self._llm_name)
+            logger.info(f"Request {id}: Success")
+            return (request, response)
+        except Exception as e:
+            if raise_errors():
+                raise e
+            logger.info(f"Request {id}: Failure: {str(e)[:256]}")
+            return (request, self._error_from_exception(e))
 
     async def _process_requests(
         self,
         requests: list[CompletionRequest],
     ) -> list[tuple[CompletionRequest, CompletionResponse | Error]]:
         """Process multiple requests concurrently, returning request/response pairs."""
-        semaphore = asyncio.Semaphore(self.max_async_concurrent_requests)
         async with AsyncClient(
             host=self.base_url,
             nice=True,
             request_timeout_seconds=self.request_timeout_seconds,
             token=self.token,
-            total_retries=0,  # we have a custom retry policy in _request_with_backoff()
+            total_retries=self.max_retries,
+            limit=self.max_async_concurrent_requests,
         ) as client:
             tasks = (
                 self._process_request_with_client(
                     client,
-                    semaphore,
                     request,
                     i,
                 )

{eval_framework-0.2.9 → eval_framework-0.2.11}/src/eval_framework/llm/base.py

@@ -1,5 +1,6 @@
 from abc import ABC, abstractmethod
-from collections.abc import Sequence
+from collections.abc import Generator, Sequence
+from contextlib import contextmanager
 from pathlib import Path
 from typing import Any
 
@@ -112,21 +113,22 @@ class BaseLLM(ABC):
         """
         pass
 
+    @contextmanager
     def _get_final_checkpoint(
         self, checkpoint_path: str | Path | None = None, model_name: str | None = None, artifact_name: str | None = None
-    ) -> tuple[str | Path | None, str | None]:
+    ) -> Generator[tuple[str | Path | None, str | None], None, None]:
         if (num_provided := sum(x is not None for x in [checkpoint_path, model_name, artifact_name])) == 0:
             if not getattr(self, "LLM_NAME", ""):
                 raise ValueError("Either LLM_NAME, checkpoint_path, model_name, or artifact_name must be provided.")
-            return None, None  # no argument given, so will use the LLM_NAME of the class
+            yield None, None  # no argument given, so will use the LLM_NAME of the class
         elif num_provided > 1:
             raise ValueError("At most one of `checkpoint_path`, `model_name`, or `artifact_name` must be provided.")
 
         elif checkpoint_path is not None:
-            return checkpoint_path, str(checkpoint_path)
+            yield checkpoint_path, str(checkpoint_path)
 
         elif model_name is not None:
-            return model_name, model_name
+            yield model_name, model_name
 
         else:
             from eval_framework.utils.file_ops import WandbFs
@@ -139,7 +141,7 @@ class BaseLLM(ABC):
                 file_root = wandb_fs.find_hf_checkpoint_root_from_path_list()
                 if file_root is None:
                     raise ValueError(f"Could not find HuggingFace checkpoint in artifact {artifact_base}:{version}")
-                return file_root, artifact_name
+                yield file_root, artifact_name
 
     def _get_final_formatter(
         self,

{eval_framework-0.2.9 → eval_framework-0.2.11}/src/eval_framework/llm/huggingface.py

@@ -322,22 +322,21 @@ class HFLLM(BaseHFLLM):
         bytes_per_token: float | None = None,
         **kwargs: Any,
     ) -> None:
-        final_path, possible_name = self._get_final_checkpoint(checkpoint_path, model_name, artifact_name)
+        with self._get_final_checkpoint(checkpoint_path, model_name, artifact_name) as (final_path, possible_name):
+            self.checkpoint_name = checkpoint_name
+            if self.checkpoint_name is None and possible_name is not None:
+                self.checkpoint_name = possible_name.replace("/", "_").replace(":", "_").strip("_")  # sanitize pathname
 
-        self.checkpoint_name = checkpoint_name
-        if self.checkpoint_name is None and possible_name is not None:
-            self.checkpoint_name = possible_name.replace("/", "_").replace(":", "_").strip("_")  # sanitize pathname
+            if final_path:
+                self.LLM_NAME = str(final_path)
 
-        if final_path:
-            self.LLM_NAME = str(final_path)
+            final_formatter = self._get_final_formatter(formatter, formatter_name, formatter_kwargs)
 
-        final_formatter = self._get_final_formatter(formatter, formatter_name, formatter_kwargs)
-
-        super().__init__(
-            formatter=final_formatter,
-            bytes_per_token=bytes_per_token,
-            **kwargs,
-        )
+            super().__init__(
+                formatter=final_formatter,
+                bytes_per_token=bytes_per_token,
+                **kwargs,
+            )
 
     @property
     def name(self) -> str:

{eval_framework-0.2.9 → eval_framework-0.2.11}/src/eval_framework/llm/vllm.py

@@ -137,10 +137,12 @@ class BaseVLLMModel(BaseLLM):
         device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu")
 
         self.batch_size = batch_size
-        self._tokenizer: None | VLLMTokenizerAPI = None
 
         self.model = LLM(**model_args, device=device)
 
+        self._tokenizer: None | VLLMTokenizerAPI = None
+        _ = self.tokenizer  # make sure tokenizer is initialized
+
         self.sampling_params: SamplingParams = self._process_sampling_params(sampling_params)
 
         logger.info(
@@ -481,28 +483,27 @@ class VLLMModel(BaseVLLMModel):
         sampling_params: SamplingParams | dict[str, Any] | None = None,
         **kwargs: Any,
     ) -> None:
-        final_path, possible_name = self._get_final_checkpoint(checkpoint_path, model_name, artifact_name)
-
-        if final_path:
-            self.LLM_NAME = str(final_path)
-
-        final_name = checkpoint_name
-        if final_name is None and possible_name is not None:
-            final_name = possible_name.replace("/", "_").replace(":", "_").strip("_")  # sanitize pathname
-
-        final_formatter = self._get_final_formatter(formatter, formatter_name, formatter_kwargs)
-
-        super().__init__(
-            formatter=final_formatter,
-            checkpoint_path=final_path,
-            checkpoint_name=final_name,
-            max_model_len=max_model_len,
-            tensor_parallel_size=tensor_parallel_size,
-            gpu_memory_utilization=gpu_memory_utilization,
-            batch_size=batch_size,
-            sampling_params=sampling_params,
-            **kwargs,
-        )
+        with self._get_final_checkpoint(checkpoint_path, model_name, artifact_name) as (final_path, possible_name):
+            if final_path:
+                self.LLM_NAME = str(final_path)
+
+            final_name = checkpoint_name
+            if final_name is None and possible_name is not None:
+                final_name = possible_name.replace("/", "_").replace(":", "_").strip("_")  # sanitize pathname
+
+            final_formatter = self._get_final_formatter(formatter, formatter_name, formatter_kwargs)
+
+            super().__init__(
+                formatter=final_formatter,
+                checkpoint_path=final_path,
+                checkpoint_name=final_name,
+                max_model_len=max_model_len,
+                tensor_parallel_size=tensor_parallel_size,
+                gpu_memory_utilization=gpu_memory_utilization,
+                batch_size=batch_size,
+                sampling_params=sampling_params,
+                **kwargs,
+            )
 
 
 class VLLMRegistryModel(VLLMModel):  # deprecated

{eval_framework-0.2.9 → eval_framework-0.2.11}/src/eval_framework/tasks/benchmarks/arc.py

@@ -15,7 +15,7 @@ class ARC(BaseTask[str]):
     """ARC dataset: https://huggingface.co/datasets/allenai/ai2_arc"""
 
     NAME = "ARC"
-    DATASET_PATH = "ai2_arc"
+    DATASET_PATH = "allenai/ai2_arc"
     SAMPLE_SPLIT = "test"
     FEWSHOT_SPLIT = "train"
     RESPONSE_TYPE = ResponseType.LOGLIKELIHOODS

{eval_framework-0.2.9 → eval_framework-0.2.11}/src/eval_framework/tasks/benchmarks/casehold.py

@@ -9,8 +9,10 @@ from eval_framework.tasks.base import NO_SUBJECT, RANDOM_SEED, BaseTask, Languag
 
 
 class CASEHOLD(BaseTask[str]):
+    """CASEHOLD dataset: https://huggingface.co/datasets/coastalcph/lex_glue"""
+
     NAME = "CaseHold"
-    DATASET_PATH = "lex_glue"
+    DATASET_PATH = "coastalcph/lex_glue"
     SAMPLE_SPLIT = "test"
     FEWSHOT_SPLIT = "train"
     RESPONSE_TYPE = ResponseType.LOGLIKELIHOODS

{eval_framework-0.2.9 → eval_framework-0.2.11}/src/eval_framework/tasks/benchmarks/gsm8k.py

@@ -91,7 +91,7 @@ class GSM8KEvalHarness(BaseTask[str]):
     """
 
     NAME = "GSM8KEvalHarness"
-    DATASET_PATH = "gsm8k"
+    DATASET_PATH = "openai/gsm8k"
     SAMPLE_SPLIT = "test"
     FEWSHOT_SPLIT = "train"
     RESPONSE_TYPE = ResponseType.COMPLETION

{eval_framework-0.2.9 → eval_framework-0.2.11}/src/eval_framework/tasks/benchmarks/math_reasoning.py

@@ -539,7 +539,7 @@ class GSM8KReasoning(MATHReasoning):
     """
 
     NAME = "GSM8KReasoning"
-    DATASET_PATH = "gsm8k"
+    DATASET_PATH = "openai/gsm8k"
     SAMPLE_SPLIT = "test"
     FEWSHOT_SPLIT = "train"
     RESPONSE_TYPE = ResponseType.COMPLETION

{eval_framework-0.2.9 → eval_framework-0.2.11}/src/eval_framework/tasks/benchmarks/truthfulqa.py

@@ -38,7 +38,7 @@ class TRUTHFULQA(BaseTask[str]):
     """TRUTHFULQA dataset: https://huggingface.co/datasets/truthfulqa/truthful_qa"""
 
     NAME = "TruthfulQA"
-    DATASET_PATH = "truthful_qa"
+    DATASET_PATH = "truthfulqa/truthful_qa"
     SAMPLE_SPLIT = "validation"
     FEWSHOT_SPLIT = ""
     RESPONSE_TYPE = ResponseType.LOGLIKELIHOODS

{eval_framework-0.2.9 → eval_framework-0.2.11}/src/eval_framework/tasks/benchmarks/winogrande.py

@@ -13,10 +13,10 @@ ANSWER_STR_TO_NUM = {"1": 0, "2": 1}
 
 
 class WINOGRANDE(BaseTask[str]):
-    """WINOGRANDE dataset: https://huggingface.co/datasets/winogrande"""
+    """WINOGRANDE dataset: https://huggingface.co/datasets/allenai/winogrande"""
 
     NAME = "Winogrande"
-    DATASET_PATH = "winogrande"
+    DATASET_PATH = "allenai/winogrande"
     SAMPLE_SPLIT = "validation"
     FEWSHOT_SPLIT = "train"
     RESPONSE_TYPE = ResponseType.LOGLIKELIHOODS