evalscope 0.7.2__py3-none-any.whl → 0.8.1__py3-none-any.whl

tests/swift/test_run_swift_vlm_eval.py

@@ -1,13 +1,12 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
+import json
 import os
+import requests
 import shutil
 import subprocess
 import time
 import unittest
 
-import json
-import requests
-
 from evalscope.backend.vlm_eval_kit import VLMEvalKitBackendManager
 from evalscope.run import run_task
 from evalscope.summarizer import Summarizer

tests/swift/test_run_swift_vlm_jugde_eval.py

@@ -1,13 +1,12 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
+import json
 import os
+import requests
 import shutil
 import subprocess
 import time
 import unittest
 
-import json
-import requests
-
 from evalscope.backend.vlm_eval_kit import VLMEvalKitBackendManager
 from evalscope.run import run_task
 from evalscope.summarizer import Summarizer

tests/vlm/test_vlmeval.py

@@ -40,8 +40,9 @@ class TestVLMEval(unittest.TestCase):
                 }],  # model name for VLMEval config
                 'nproc': 1,
                 'reuse': True,
-                'work_dir': 'outputs'
-            }
+            },
+            'work_dir': 'outputs',
+            'use_cache': 'outputs/20241216_142838'
         }
 
         logger.info(f'>> Start to run task: {task_cfg}')

evalscope/backend/rag_eval/ragas/metrics/__init__.py

@@ -1,2 +0,0 @@
-from .multi_modal_faithfulness import multimodal_faithness, MultiModalFaithfulness
-from .multi_modal_relevance import multimodal_relevance, MultiModalRelevance

evalscope/backend/rag_eval/ragas/metrics/multi_modal_faithfulness.py

@@ -1,91 +0,0 @@
-import typing as t
-import numpy as np
-from dataclasses import dataclass, field
-from ragas.dataset_schema import SingleTurnSample
-from ragas.metrics.base import MetricWithLLM, SingleTurnMetric, MetricType
-from pydantic import BaseModel, Field
-from evalscope.backend.rag_eval.ragas.prompts.multi_modal_prompt import ImageTextPrompt
-
-
-class FaithfulnessInput(BaseModel):
-    response: str = Field(description="response from AI")
-    retrieved_contexts: list[str] = Field(description="contexts retrieved from the LLM")
-
-    def to_string_list(self):
-        return [
-            "inputs:",
-            self.response,
-            "retrieved_contexts: ",
-        ] + self.retrieved_contexts
-
-
-class FaithfulnessOutput(BaseModel):
-    faithful: bool = Field(description="boolean indicating if request was faithful")
-
-
-class MultiModalFaithfulnessPrompt(
-    ImageTextPrompt[FaithfulnessInput, FaithfulnessOutput]
-):
-    # refer: https://github.com/run-llama/llama_index/blob/main/llama-index-core/llama_index/core/evaluation/multi_modal/faithfulness.py
-    instruction = "Please tell if a given piece of information is supported by the visual as well as textual context information. You need to answer with either True or False. Answer True if any of the image(s) and textual context supports the information"
-    input_model = FaithfulnessInput
-    output_model = FaithfulnessOutput
-    examples = [
-        (
-            FaithfulnessInput(
-                response="Apple pie is generally double-crusted.",
-                retrieved_contexts=[
-                    "An apple pie is a fruit pie in which the principal filling ingredient is apples.",
-                    "Apple pie is often served with whipped cream, ice cream ('apple pie à la mode'), custard or cheddar cheese.",
-                    "It is generally double-crusted, with pastry both above and below the filling; the upper crust may be solid or latticed (woven of crosswise strips).",
-                ],
-            ),
-            FaithfulnessOutput(faithful=True),
-        ),
-        (
-            FaithfulnessInput(
-                response="Apple pies tastes bad.",
-                retrieved_contexts=[
-                    "An apple pie is a fruit pie in which the principal filling ingredient is apples.",
-                    "Apple pie is often served with whipped cream, ice cream ('apple pie à la mode'), custard or cheddar cheese.",
-                    "It is generally double-crusted, with pastry both above and below the filling; the upper crust may be solid or latticed (woven of crosswise strips).",
-                ],
-            ),
-            FaithfulnessOutput(faithful=False),
-        ),
-    ]
-
-
-@dataclass
-class MultiModalFaithfulness(MetricWithLLM, SingleTurnMetric):
-    name: str = "faithful_rate"  # type: ignore
-    _required_columns: t.Dict[MetricType, t.Set[str]] = field(
-        default_factory=lambda: {
-            MetricType.SINGLE_TURN: {
-                "response",
-                "retrieved_contexts",
-            }
-        }
-    )
-    faithfulness_prompt: ImageTextPrompt = MultiModalFaithfulnessPrompt()
-
-    async def _ascore(self, row: t.Dict, callbacks) -> float:
-        prompt_input = FaithfulnessInput(
-            response=row["response"], retrieved_contexts=row["retrieved_contexts"]
-        )
-        assert self.llm is not None, "LLM is not set"
-        prompt_response = await self.faithfulness_prompt.generate(
-            data=prompt_input, llm=self.llm, callbacks=callbacks
-        )
-        if prompt_response is None:
-            return np.nan
-        return float(prompt_response.faithful)
-
-    async def _single_turn_ascore(
-        self, sample: SingleTurnSample, callbacks
-    ) -> float:
-        row = sample.to_dict()
-        return await self._ascore(row, callbacks)
-
-
-multimodal_faithness = MultiModalFaithfulness()

evalscope/backend/rag_eval/ragas/metrics/multi_modal_relevance.py

@@ -1,99 +0,0 @@
-import typing as t
-from dataclasses import dataclass, field
-import numpy as np
-from ragas.dataset_schema import SingleTurnSample
-from ragas.metrics.base import MetricWithLLM, SingleTurnMetric, MetricType
-from pydantic import BaseModel, Field
-from evalscope.backend.rag_eval.ragas.prompts.multi_modal_prompt import ImageTextPrompt
-
-
-class RelevanceInput(BaseModel):
-    user_input: str = Field(description="user input")
-    response: str = Field(description="response from AI")
-    retrieved_contexts: list[str] = Field(description="contexts retrieved from the LLM")
-
-    def to_string_list(self):
-        return [
-            f"Question: {self.user_input}",
-            f"Response: {self.response}",
-            "retrieved_contexts: ",
-        ] + self.retrieved_contexts
-
-
-class RelevanceOutput(BaseModel):
-    relevance: bool = Field(description="boolean indicating if request was relevance")
-
-
-class MultiModalRelevancePrompt(ImageTextPrompt[RelevanceInput, RelevanceOutput]):
-    # refer https://github.com/run-llama/llama_index/blob/main/llama-index-core/llama_index/core/evaluation/multi_modal/relevancy.py
-    instruction = """
-Your task is to evaluate if the response for the query is in line with the images and textual context information provided.
-You have two options to answer. Either True / False.
-Answer - True, if the response for the query is in line with context information otherwise False.
-"""
-    input_model = RelevanceInput
-    output_model = RelevanceOutput
-    examples = [
-        (
-            RelevanceInput(
-                user_input="What is the primary ingredient in a traditional Margherita pizza?",
-                response="The primary ingredients in a Margherita pizza are tomatoes, mozzarella cheese, and fresh basil.",
-                retrieved_contexts=[
-                    "A traditional Margherita pizza consists of a thin crust.",
-                    "The main toppings include tomatoes, mozzarella cheese, fresh basil, salt, and olive oil.",
-                    "It is one of the simplest and most classic types of pizza.",
-                ],
-            ),
-            RelevanceOutput(relevance=True),
-        ),
-        (
-            RelevanceInput(
-                user_input="Who won the Best Actor award at the Oscars in 2021?",
-                response="The Best Actor award in 2021 was won by Leonardo DiCaprio.",
-                retrieved_contexts=[
-                    "The 93rd Academy Awards were held in 2021.",
-                    "Anthony Hopkins won the Best Actor award for his role in 'The Father'.",
-                    "The event was unique due to COVID-19 restrictions.",
-                ],
-            ),
-            RelevanceOutput(relevance=False),
-        ),
-    ]
-
-
-@dataclass
-class MultiModalRelevance(MetricWithLLM, SingleTurnMetric):
-    name: str = "relevance_rate"  # type: ignore
-    _required_columns: t.Dict[MetricType, t.Set[str]] = field(
-        default_factory=lambda: {
-            MetricType.SINGLE_TURN: {
-                "user_input",
-                "response",
-                "retrieved_contexts",
-            }
-        }
-    )
-    relevance_prompt: ImageTextPrompt = MultiModalRelevancePrompt()
-
-    async def _ascore(self, row: t.Dict, callbacks) -> float:
-        prompt_input = RelevanceInput(
-            user_input=row["user_input"],
-            response=row["response"],
-            retrieved_contexts=row["retrieved_contexts"],
-        )
-        assert self.llm is not None, "LLM is not set"
-        prompt_response = await self.relevance_prompt.generate(
-            data=prompt_input, llm=self.llm, callbacks=callbacks
-        )
-        if prompt_response is None:
-            return np.nan
-        return float(prompt_response.relevance)
-
-    async def _single_turn_ascore(
-        self, sample: SingleTurnSample, callbacks
-    ) -> float:
-        row = sample.to_dict()
-        return await self._ascore(row, callbacks)
-
-
-multimodal_relevance = MultiModalRelevance()

evalscope/cache.py

@@ -1,98 +0,0 @@
-# Copyright (c) Alibaba, Inc. and its affiliates.
-
-import os
-from typing import Union
-
-import cachetools
-from cachetools import Cache as CachetoolsCache
-from pympler import asizeof
-from datetime import datetime, timedelta
-import pickle
-
-from evalscope.constants import DEFAULT_ROOT_CACHE_DIR
-from evalscope.utils.logger import get_logger
-
-logger = get_logger()
-
-
-DEFAULT_CACHE_MAXSIZE = 1 * 1024 * 1024 * 1024  # 1 GB
-DEFAULT_CACHE_EXPIRE = 60 * 60 * 24             # 1 day (seconds)
-DEFAULT_MEM_CACHE_PATH = os.environ.get('MEM_CACHE_PATH',
-                                        os.path.join(os.path.expanduser(DEFAULT_ROOT_CACHE_DIR),
-                                                     'mem_cache', 'global_cache.pkl'))
-
-
-class Cache:
-
-    # TODO:   by xingjun.wxj@alibaba-inc.com
-    #  1. atomic operation for saving cache
-    #  2. consider the distributed env
-
-    @classmethod
-    def lru_cache(cls, maxsize: int = DEFAULT_CACHE_MAXSIZE):
-        return cachetools.LRUCache(maxsize=maxsize, getsizeof=asizeof.asizeof)
-
-    @classmethod
-    def ttl_cache(cls, max_size: float = DEFAULT_CACHE_MAXSIZE, expire: float = DEFAULT_CACHE_EXPIRE):
-        return cachetools.TTLCache(maxsize=max_size,
-                                   ttl=timedelta(seconds=expire),
-                                   timer=datetime.now,
-                                   getsizeof=asizeof.asizeof)
-
-    @classmethod
-    def load(cls, path: str) -> Union[CachetoolsCache, None]:
-        """
-        Load cache from disk. Pickle is used for serialization.
-
-        Args:
-            path: The local path to load the cache.
-
-        Returns:
-            The cache instance loaded from disk. Should be cachetools.Cache or None.
-        """
-        if os.path.exists(path):
-            logger.info(f'** Loading cache from {path} ...')
-            with open(path, 'rb') as f:
-                return pickle.load(f)
-        else:
-            return None
-
-    @classmethod
-    def save(cls, cache: CachetoolsCache, path: str = DEFAULT_MEM_CACHE_PATH):
-        """
-        Dump memory cache to disk. Pickle is used for serialization.
-
-        Args:
-            cache: The cache instance to be saved.
-            path: The local path to save the cache.
-
-        Returns: None
-        """
-        os.makedirs(os.path.dirname(path), exist_ok=True)
-        with open(path, 'wb') as f:
-            pickle.dump(cache, f)
-        logger.info(f'** Cache saved to {path} !')
-
-
-def init_mem_cache(method: str = 'ttl', cache_file_path: str = DEFAULT_MEM_CACHE_PATH) -> CachetoolsCache:
-    """
-    Initialize memory cache.
-
-    Args:
-        method (str): 'ttl' or 'lru', see https://cachetools.readthedocs.io/en/latest/ for details.
-        cache_file_path (str): The local cache path. Should be a pickle file.
-
-    Returns:
-        The cache instance. Should be cachetools.Cache.
-    """
-    logger.info(f'** Initializing memory cache with method `{method}` ... \n')
-    mem_cache = Cache.load(path=cache_file_path)
-    if mem_cache is None:
-        if method == 'ttl':
-            mem_cache = Cache.ttl_cache()
-        elif method == 'lru':
-            mem_cache = Cache.lru_cache()
-        else:
-            raise ValueError(f'Unknown cache method {method}. Please choose from `ttl` or `lru`.')
-
-    return mem_cache