evalscope 0.6.0rc0__py3-none-any.whl → 0.6.1__py3-none-any.whl

evalscope/backend/opencompass/tasks/eval_datasets.py

@@ -51,12 +51,12 @@ with read_base():
     from opencompass.configs.datasets.triviaqa.triviaqa_gen_2121ce import triviaqa_datasets
     from opencompass.configs.datasets.cmb.cmb_gen_dfb5c4 import cmb_datasets
     from opencompass.configs.datasets.cmmlu.cmmlu_gen_c13365 import cmmlu_datasets
-    from opencompass.configs.datasets.bbh.bbh_gen_5b92b0 import bbh_datasets
 
     # Note: to be supported
     # from opencompass.configs.datasets.flores.flores_gen_806ede import flores_datasets
     # from opencompass.configs.datasets.TheoremQA.TheoremQA_5shot_gen_6f0af8 import TheoremQA_datasets
     # from opencompass.configs.datasets.commonsenseqa.commonsenseqa_gen_c946f2 import commonsenseqa_datasets
+    # from opencompass.configs.datasets.bbh.bbh_gen_5b92b0 import bbh_datasets
 
 
 datasets = []

evalscope/backend/rag_eval/ragas/tasks/testset_generation.py

@@ -1,15 +1,15 @@
-import os
 import asyncio
+import os
+
 import pandas as pd
-from tqdm import tqdm
-from ragas.llms import LangchainLLMWrapper
 from ragas.embeddings import LangchainEmbeddingsWrapper
-from .translate_prompt import translate_prompts
-from evalscope.utils.logger import get_logger
-from evalscope.backend.rag_eval.ragas.arguments import TestsetGenerationArguments
-from evalscope.backend.rag_eval import EmbeddingModel, LLM, ChatOpenAI
+from ragas.llms import LangchainLLMWrapper
+from tqdm import tqdm
 
-os.environ['DO_NOT_TRACK'] = 'true'
+from evalscope.backend.rag_eval import LLM, ChatOpenAI, EmbeddingModel
+from evalscope.backend.rag_eval.ragas.arguments import TestsetGenerationArguments
+from evalscope.utils.logger import get_logger
+from .translate_prompt import translate_prompts
 
 logger = get_logger()
 
@@ -17,116 +17,110 @@ logger = get_logger()
 def get_transform(llm, embedding, language):
     """
     Creates and returns a default set of transforms for processing a knowledge graph.
-
-    This function defines a series of transformation steps to be applied to a
-    knowledge graph, including extracting summaries, keyphrases, titles,
-    headlines, and embeddings, as well as building similarity relationships
-    between nodes.
-
-    The transforms are applied in the following order:
-    1. Parallel extraction of summaries and headlines
-    2. Embedding of summaries for document nodes
-    3. Splitting of headlines
-    4. Parallel extraction of embeddings, keyphrases, and titles
-    5. Building cosine similarity relationships between nodes
-    6. Building cosine similarity relationships between summaries
-
-    Returns
-    -------
-    Transforms
-        A list of transformation steps to be applied to the knowledge graph.
-
     """
     from ragas.testset.transforms.engine import Parallel
     from ragas.testset.transforms.extractors import (
         EmbeddingExtractor,
         HeadlinesExtractor,
-        KeyphrasesExtractor,
         SummaryExtractor,
-        TitleExtractor,
     )
-    from ragas.testset.transforms.relationship_builders.cosine import (
+    from ragas.testset.transforms.extractors.llm_based import NERExtractor, ThemesExtractor
+    from ragas.testset.transforms.relationship_builders import (
         CosineSimilarityBuilder,
-        SummaryCosineSimilarityBuilder,
+        OverlapScoreBuilder,
     )
     from ragas.testset.transforms.splitters import HeadlineSplitter
+    from ragas.testset.transforms.filters import CustomNodeFilter
     from ragas.testset.graph import NodeType
+    from ragas.utils import num_tokens_from_string
+
+    def summary_filter(node):
+        return (node.type == NodeType.DOCUMENT and num_tokens_from_string(node.properties['page_content']) > 500)
 
-    # define the transforms
-    summary_extractor = SummaryExtractor(llm=llm)
-    keyphrase_extractor = KeyphrasesExtractor(llm=llm)
-    title_extractor = TitleExtractor(llm=llm)
+    summary_extractor = SummaryExtractor(llm=llm, filter_nodes=lambda node: summary_filter(node))
+    ner_extractor = NERExtractor(llm=llm, filter_nodes=lambda node: node.type == NodeType.CHUNK)
+    theme_extractor = ThemesExtractor(llm=llm)
     headline_extractor = HeadlinesExtractor(llm=llm)
 
     asyncio.run(
         translate_prompts(
             prompts=[
                 summary_extractor,
-                keyphrase_extractor,
-                title_extractor,
+                theme_extractor,
+                ner_extractor,
                 headline_extractor,
             ],
             target_lang=language,
             llm=llm,
             adapt_instruction=True,
-        )
-    )
+        ))
+
+    splitter = HeadlineSplitter(min_tokens=500)
 
-    embedding_extractor = EmbeddingExtractor(embedding_model=embedding)
-    headline_splitter = HeadlineSplitter()
-    cosine_sim_builder = CosineSimilarityBuilder(threshold=0.8)
-    summary_embedder = EmbeddingExtractor(
-        name='summary_embedder',
-        filter_nodes=lambda node: True if node.type == NodeType.DOCUMENT else False,
+    summary_emb_extractor = EmbeddingExtractor(
+        embedding_model=embedding,
         property_name='summary_embedding',
         embed_property_name='summary',
-        embedding_model=embedding,
+        filter_nodes=lambda node: summary_filter(node),
     )
-    summary_cosine_sim_builder = SummaryCosineSimilarityBuilder(threshold=0.6)
 
-    # specify the transforms and their order to be applied
+    cosine_sim_builder = CosineSimilarityBuilder(
+        property_name='summary_embedding',
+        new_property_name='summary_similarity',
+        threshold=0.7,
+        filter_nodes=lambda node: summary_filter(node),
+    )
+
+    ner_overlap_sim = OverlapScoreBuilder(threshold=0.01, filter_nodes=lambda node: node.type == NodeType.CHUNK)
+
+    node_filter = CustomNodeFilter(llm=llm, filter_nodes=lambda node: node.type == NodeType.CHUNK)
+
     transforms = [
-        Parallel(summary_extractor, headline_extractor),
-        summary_embedder,
-        headline_splitter,
-        Parallel(embedding_extractor, keyphrase_extractor, title_extractor),
-        cosine_sim_builder,
-        summary_cosine_sim_builder,
+        headline_extractor,
+        splitter,
+        summary_extractor,
+        node_filter,
+        Parallel(summary_emb_extractor, theme_extractor, ner_extractor),
+        Parallel(cosine_sim_builder, ner_overlap_sim),
     ]
+
     return transforms
 
 
 def get_distribution(llm, distribution, language):
-    from ragas.testset.synthesizers.abstract_query import (
-        AbstractQuerySynthesizer,
-        ComparativeAbstractQuerySynthesizer,
+    from ragas.testset.synthesizers.multi_hop import (
+        MultiHopAbstractQuerySynthesizer,
+        MultiHopSpecificQuerySynthesizer,
     )
-    from ragas.testset.synthesizers.specific_query import SpecificQuerySynthesizer
+    from ragas.testset.synthesizers.single_hop.specific import (
+        SingleHopSpecificQuerySynthesizer, )
 
-    abstract = AbstractQuerySynthesizer(llm=llm)
-    comparative = ComparativeAbstractQuerySynthesizer(llm=llm)
-    specific = SpecificQuerySynthesizer(llm=llm)
+    single_hop = SingleHopSpecificQuerySynthesizer(llm=llm)
+    multi_hop_abs = MultiHopAbstractQuerySynthesizer(llm=llm)
+    multi_hop_spec = MultiHopSpecificQuerySynthesizer(llm=llm)
 
     asyncio.run(
         translate_prompts(
             prompts=[
-                abstract,
-                comparative,
-                specific,
+                single_hop,
+                multi_hop_abs,
+                multi_hop_spec,
             ],
             target_lang=language,
             llm=llm,
             adapt_instruction=True,
-        )
-    )
-    return [
-        (abstract, distribution['simple']),
-        (comparative, distribution['multi_context']),
-        (specific, distribution['reasoning']),
-    ]
+        ))
+
+    mapping = {
+        'simple': single_hop,
+        'multi_context': multi_hop_abs,
+        'reasoning': multi_hop_spec,
+    }
+
+    return [(mapping[key], distribution[key]) for key in mapping if key in distribution]
 
 
-def get_knowledge_graph(documents, transforms, local_file):
+def get_knowledge_graph(documents, transforms, local_file, run_config):
     from ragas.testset.graph import KnowledgeGraph, Node, NodeType
     from ragas.testset.transforms import apply_transforms
 
@@ -148,7 +142,7 @@ def get_knowledge_graph(documents, transforms, local_file):
     kg = KnowledgeGraph(nodes=nodes)
 
     # apply transforms and update the knowledge graph
-    apply_transforms(kg, transforms)
+    apply_transforms(kg, transforms, run_config=run_config)
 
     # save the knowledge graph
     output_path = os.path.dirname(local_file)
@@ -158,6 +152,39 @@ def get_knowledge_graph(documents, transforms, local_file):
     return kg
 
 
+def get_persona(llm, kg, language):
+    from evalscope.backend.rag_eval.ragas.prompts.persona_prompt import PersonaGenerationPromptZH
+    from ragas.testset.persona import generate_personas_from_kg, PersonaGenerationPrompt
+    from ragas.testset.graph import Node
+
+    def filter(node: Node) -> bool:
+        if (node.type.name == 'DOCUMENT' and node.properties.get('summary_embedding') is not None):
+            return True
+        else:
+            return False
+
+    if language == 'chinese':
+        persona_prompt = PersonaGenerationPromptZH()
+    else:
+        persona_prompt = PersonaGenerationPrompt()
+    # NOTE: can't translate this yet
+    # asyncio.run(
+    #     translate_prompts(
+    #         prompts=[persona_prompt],
+    #         target_lang=language,
+    #         llm=llm,
+    #         adapt_instruction=True,
+    #     ))
+
+    return generate_personas_from_kg(
+        llm=llm,
+        kg=kg,
+        num_personas=3,
+        persona_generation_prompt=persona_prompt,
+        filter_fn=filter,
+    )
+
+
 def load_data(file_path):
     from langchain_community.document_loaders import UnstructuredFileLoader
 
@@ -178,32 +205,31 @@ def generate_testset(args: TestsetGenerationArguments) -> None:
     generator_llm = LLM.load(**args.generator_llm)
     embeddings = EmbeddingModel.load(**args.embeddings)
 
+    wrapped_llm = LangchainLLMWrapper(generator_llm)
+    wrapped_embeddings = LangchainEmbeddingsWrapper(embeddings)
+
     # Change resulting question type distribution
-    distributions = get_distribution(
-        LangchainLLMWrapper(generator_llm), args.distribution, args.language
-    )
+    distributions = get_distribution(wrapped_llm, args.distribution, args.language)
 
+    run_config = RunConfig(timeout=600, max_retries=3, max_wait=120, max_workers=1, log_tenacity=True)
     # get transforms
     transforms = get_transform(
-        LangchainLLMWrapper(generator_llm),
-        LangchainEmbeddingsWrapper(embeddings),
+        wrapped_llm,
+        wrapped_embeddings,
         args.language,
     )
 
     # get knowledge graph
-    knowledge_graph = get_knowledge_graph(documents, transforms, args.knowledge_graph)
+    knowledge_graph = get_knowledge_graph(documents, transforms, args.knowledge_graph, run_config)
 
-    generator = TestsetGenerator.from_langchain(
-        generator_llm, embeddings, knowledge_graph
-    )
+    persona_list = get_persona(llm=wrapped_llm, kg=knowledge_graph, language=args.language)
+
+    generator = TestsetGenerator(llm=wrapped_llm, knowledge_graph=knowledge_graph, persona_list=persona_list)
 
-    runconfig = RunConfig(
-        timeout=600, max_retries=3, max_wait=120, max_workers=1, log_tenacity=True
-    )
     testset = generator.generate(
         testset_size=args.test_size,
         query_distribution=distributions,
-        run_config=runconfig,
+        run_config=run_config,
         with_debugging_logs=True,
         raise_exceptions=True,
     )
@@ -212,9 +238,7 @@ def generate_testset(args: TestsetGenerationArguments) -> None:
     testset_df = testset.to_pandas()
     output_path = os.path.dirname(args.output_file)
     os.makedirs(output_path, exist_ok=True)
-    testset_df.to_json(
-        args.output_file, indent=4, index=False, orient='records', force_ascii=False
-    )
+    testset_df.to_json(args.output_file, indent=4, index=False, orient='records', force_ascii=False)
 
     # get answer
     testset_with_answer = get_answer(testset_df, generator_llm, args.language)
@@ -243,21 +267,17 @@ Answer:
         contexts = '\n'.join(row['reference_contexts'])
 
         # Combine question and contexts as input for the LLM
-        input_text = template.format(
-            language=language, question=question, contexts=contexts
-        )
+        input_text = template.format(language=language, question=question, contexts=contexts)
 
         # Generate the answer using the generator LLM
         answer = generator_llm.invoke(input_text)
         if isinstance(generator_llm, ChatOpenAI):
             answer = answer.content
-        items.append(
-            {
-                'user_input': question,
-                'retrieved_contexts': row['reference_contexts'],
-                'response': answer,
-                'reference': row['reference'],
-            }
-        )
+        items.append({
+            'user_input': question,
+            'retrieved_contexts': row['reference_contexts'],
+            'response': answer,
+            'reference': row['reference'],
+        })
 
     return pd.DataFrame.from_dict(items)

evalscope/backend/rag_eval/utils/clip.py

@@ -0,0 +1,149 @@
+import os
+import torch
+import torch.nn.functional as F
+from typing import List
+from PIL import Image
+from evalscope.backend.rag_eval.utils.tools import download_model, PIL_to_base64
+from transformers import AutoModel, AutoProcessor
+from langchain_core.embeddings import Embeddings
+
+
+class VisionModel:
+    @staticmethod
+    def load(**kw):
+        api_base = kw.get("api_base", None)
+        if api_base:
+
+            return VLMAPI(
+                model_name=kw.get("model_name", ""),
+                openai_api_base=api_base,
+                openai_api_key=kw.get("api_key", "EMPTY"),
+                prompt=kw.get("prompt", None),
+            )
+        else:
+            return CLIPModel(**kw)
+
+
+class VLMAPI:
+    def __init__(self, model_name, openai_api_base, openai_api_key, prompt=None):
+        from langchain_openai import ChatOpenAI
+        from langchain_core.prompts import ChatPromptTemplate
+
+        self.model_name = model_name
+        self.model = ChatOpenAI(
+            model_name=model_name,
+            openai_api_base=openai_api_base,
+            openai_api_key=openai_api_key,
+        )
+        self.default_prompt = "Please describe this image in general. Directly provide the description, do not include prefix like 'This image depicts'"
+        self.prompt = ChatPromptTemplate.from_messages(
+            [
+                ("system", prompt if prompt else self.default_prompt),
+                (
+                    "user",
+                    [
+                        {
+                            "type": "image_url",
+                            "image_url": {"url": "data:image/jpeg;base64,{image_data}"},
+                        }
+                    ],
+                ),
+            ]
+        )
+        self.chain = self.prompt | self.model
+        self.transform = PIL_to_base64
+
+    def encode_image(self, images):
+        captions = []
+        for image in images:
+            response = self.chain.invoke({"image_data": image})
+            captions.append(response.content)
+        return captions
+
+
+class CLIPModel(Embeddings):
+    def __init__(
+        self,
+        model_name: str,
+        revision: str = "master",
+        hub="modelscope",
+        device="cpu",
+    ):
+        self.device = device
+        self.model_name = model_name
+        self.revision = revision
+
+        # Download the model if it doesn't exist locally
+        if not os.path.exists(model_name) and hub == "modelscope":
+            model_name = download_model(self.model_name, self.revision)
+
+        # Load the model and processor
+        self.model = AutoModel.from_pretrained(model_name).to(self.device)
+        self.processor = AutoProcessor.from_pretrained(model_name)
+        self.transform = self.processor.image_processor
+        self.tokenizer = self.processor.tokenizer
+
+    def encode_text(self, batch_texts: List[str] | List[List[str]]):
+        if isinstance(batch_texts[0], list):
+            batch_texts = [
+                text for _, texts in enumerate(batch_texts) for text in texts
+            ]
+        # Ensure that the input texts are within the token limit
+        max_length = self.tokenizer.model_max_length
+        if not max_length or max_length > 0xFFFFFF:
+            max_length = 512
+        encoded_inputs = self.tokenizer(
+            text=batch_texts,
+            max_length=max_length,
+            padding=True,
+            truncation=True,
+            return_tensors="pt",
+        )
+
+        inputs = {k: v.to(self.device) for k, v in encoded_inputs.items()}
+
+        with torch.no_grad():
+            text_features = self.model.get_text_features(**inputs)
+        text_features = F.normalize(text_features, p=2, dim=-1)
+        return text_features
+
+    def encode_image(self, image):
+        batch_images = torch.stack([d["pixel_values"][0] for d in image])
+        batch_images = batch_images.to(self.device)
+        with torch.no_grad():
+            image_features = self.model.get_image_features(batch_images)
+        image_features = F.normalize(image_features, p=2, dim=-1)
+        return image_features
+
+    def embed_documents(self, texts):
+        text_features = self.encode_text(texts)
+        return text_features.cpu().numpy().tolist()
+
+    def embed_query(self, text):
+        text_features = self.encode_text([text])
+        return text_features.cpu().numpy().tolist()[0]
+
+    def embed_image(self, uris: List[str]):
+        # read image and transform
+        images = [Image.open(image_path) for image_path in uris]
+        transformed_images = [
+            self.transform(
+                image,
+                return_tensors="pt",
+            )
+            for image in images
+        ]
+        image_features = self.encode_image(transformed_images)
+        return image_features.cpu().numpy().tolist()
+
+
+if __name__ == "__main__":
+    model = CLIPModel("AI-ModelScope/chinese-clip-vit-large-patch14-336px")
+    model.embed_image(
+        [
+            "custom_eval/multimodal/images/AMNH.jpg",
+            "custom_eval/multimodal/images/AMNH.jpg",
+        ]
+    )
+    model.encode_text(["我喜欢吃饭" * 1000])
+    print("done")

evalscope/backend/rag_eval/utils/embedding.py

@@ -0,0 +1,183 @@
+import os
+import torch
+from typing import List, Optional, Union, Dict
+from sentence_transformers import models
+from sentence_transformers.SentenceTransformer import SentenceTransformer
+from sentence_transformers.cross_encoder import CrossEncoder
+from torch import Tensor
+from evalscope.backend.rag_eval.utils.tools import download_model
+from evalscope.utils.logger import get_logger
+from langchain_core.embeddings import Embeddings
+
+logger = get_logger()
+
+
+class BaseModel(Embeddings):
+    def __init__(
+        self,
+        model_name_or_path: str,
+        max_seq_length: int = 512,
+        prompt: str = '',
+        revision: Optional[str] = None,
+        **kwargs,
+    ):
+        self.model_name_or_path = model_name_or_path
+        self.max_seq_length = max_seq_length
+        self.model_kwargs = kwargs.pop('model_kwargs', {})
+        self.model_kwargs['trust_remote_code'] = True
+
+        self.config_kwargs = kwargs.pop('config_kwargs', {})
+        self.config_kwargs['trust_remote_code'] = True
+
+        self.encode_kwargs = kwargs.pop('encode_kwargs', {})
+        self.encode_kwargs['convert_to_tensor'] = True
+
+        self.prompt = prompt
+        self.revision = revision
+
+    @property
+    def mteb_model_meta(self):
+        """Model metadata for MTEB (Multilingual Task Embeddings Benchmark)"""
+        from mteb import ModelMeta
+
+        return ModelMeta(
+            name=os.path.basename(self.model_name_or_path),
+            revision=self.revision,
+            languages=None,
+            release_date=None,
+        )
+
+    def embed_documents(self, texts: List[str]) -> List[List[float]]:
+        """Embed search docs. Compact langchain.
+
+        Args:
+            texts: List of text to embed.
+
+        Returns:
+            List of embeddings.
+        """
+        return self.encode_corpus(texts).tolist()
+
+    def embed_query(self, text: str) -> List[float]:
+        """Embed query text. Compact langchain.
+
+        Args:
+            text: Text to embed.
+
+        Returns:
+            Embedding.
+        """
+        return self.encode_queries(text).tolist()
+
+    def encode(self, texts: Union[str, List[str]], **kwargs) -> List[List[float]]:
+        """Embed text."""
+        raise NotImplementedError
+
+    def encode_queries(self, queries: List[str], **kwargs) -> list[torch.Tensor]:
+        """Embed query text. Compact mteb."""
+        raise NotImplementedError
+
+    def encode_corpus(self, corpus: List[str] | List[Dict[str, str]], **kwargs) -> list[torch.Tensor]:
+        """Embed search docs . Compact mteb."""
+        raise NotImplementedError
+
+
+class SentenceTransformerModel(BaseModel):
+    def __init__(
+        self, model_name_or_path: str, pooling_mode: Optional[str] = None, **kwargs
+    ):
+        super().__init__(model_name_or_path, **kwargs)
+
+        if not pooling_mode:
+            self.model = SentenceTransformer(
+                self.model_name_or_path,
+                config_kwargs=self.config_kwargs,
+                model_kwargs=self.model_kwargs,
+            )
+        else:
+            word_embedding_model = models.Transformer(
+                self.model_name_or_path,
+                config_args=self.config_kwargs,
+                model_args=self.model_kwargs,
+            )
+            pooling_model = models.Pooling(
+                word_embedding_model.get_word_embedding_dimension(),
+                pooling_mode=pooling_mode,
+            )
+            self.model = SentenceTransformer(
+                modules=[word_embedding_model, pooling_model],
+            )
+
+        self.model.max_seq_length = self.max_seq_length
+
+    def encode(self, texts: Union[str, List[str]], prompt=None, **kwargs) -> List[torch.Tensor]:
+        kwargs.pop('prompt_name', '')  # remove prompt name, use prompt
+        self.encode_kwargs.update(kwargs)
+
+        embeddings = self.model.encode(texts, prompt=prompt, **self.encode_kwargs)
+        assert isinstance(embeddings, Tensor)
+        return embeddings.cpu().detach()
+
+    def encode_queries(self, queries, **kwargs):
+        return self.encode(queries, prompt=self.prompt)
+
+    def encode_corpus(self, corpus, **kwargs):
+        if isinstance(corpus[0], dict):
+            input_texts = ['{} {}'.format(doc.get('title', ''), doc['text']).strip() for doc in corpus]
+        else:
+            input_texts = corpus
+        return self.encode(input_texts)
+
+
+class CrossEncoderModel(BaseModel):
+    def __init__(self, model_name_or_path: str, **kwargs):
+        super().__init__(model_name_or_path, **kwargs)
+        self.model = CrossEncoder(
+            self.model_name_or_path,
+            trust_remote_code=True,
+            max_length=self.max_seq_length,
+        )
+
+    def predict(self, sentences: List[List[str]], **kwargs) -> List[List[float]]:
+        self.encode_kwargs.update(kwargs)
+
+        if len(sentences[0]) == 3:  # Note: For mteb retrieval task
+            processed_sentences = []
+            for query, docs, instruction in sentences:
+                if isinstance(docs, dict):
+                    docs = docs['text']
+                processed_sentences.append((self.prompt + query, docs))
+            sentences = processed_sentences
+        embeddings = self.model.predict(sentences, **self.encode_kwargs)
+        assert isinstance(embeddings, Tensor)
+        return embeddings
+
+
+class EmbeddingModel:
+    """Custom embeddings"""
+
+    @staticmethod
+    def load(
+        model_name_or_path: str = '',
+        is_cross_encoder: bool = False,
+        hub: str = 'modelscope',
+        revision: Optional[str] = 'master',
+        **kwargs,
+    ):
+        # If model path does not exist and hub is 'modelscope', download the model
+        if not os.path.exists(model_name_or_path) and hub == 'modelscope':
+            model_name_or_path = download_model(model_name_or_path, revision)
+
+        # Return different model instances based on whether it is a cross-encoder and pooling mode
+        if is_cross_encoder:
+            return CrossEncoderModel(
+                model_name_or_path,
+                revision=revision,
+                **kwargs,
+            )
+        else:
+            return SentenceTransformerModel(
+                model_name_or_path,
+                revision=revision,
+                **kwargs,
+            )

evalscope/backend/rag_eval/utils/llm.py

@@ -0,0 +1,72 @@
+import os
+from typing import Any, Dict, Iterator, List, Mapping, Optional
+from modelscope.utils.hf_util import GenerationConfig
+from langchain_core.callbacks.manager import CallbackManagerForLLMRun
+from langchain_core.language_models.llms import LLM as BaseLLM
+from evalscope.models.model_adapter import ChatGenerationModelAdapter
+from langchain_openai import ChatOpenAI
+
+
+class LLM:
+    @staticmethod
+    def load(**kw):
+        api_base = kw.get('api_base', None)
+        if api_base:
+            return ChatOpenAI(
+                model_name=kw.get('model_name', ''),
+                openai_api_base=api_base,
+                openai_api_key=kw.get('api_key', 'EMPTY'),
+            )
+        else:
+            return LocalLLM(**kw)
+
+
+class LocalLLM(BaseLLM):
+    """A custom LLM that loads a model from a given path and performs inference."""
+
+    model_name_or_path: str
+    model_revision: str = 'master'
+    template_type: str = 'default'
+    model_name: Optional[str]
+    model: Optional[ChatGenerationModelAdapter]
+    generation_config: Optional[Dict]
+
+    def __init__(self, **kw):
+        super().__init__(**kw)
+        self.model_name = os.path.basename(self.model_name_or_path)
+        self.model = ChatGenerationModelAdapter(
+            model_id=self.model_name_or_path,
+            model_revision=self.model_revision,
+            template_type=self.template_type,
+            generation_config=GenerationConfig(**self.generation_config) if self.generation_config else None,
+        )
+
+    def _call(
+        self,
+        prompt: str,
+        stop: Optional[List[str]] = None,
+        run_manager: Optional[CallbackManagerForLLMRun] = None,
+        **kwargs: Any,
+    ) -> str:
+        """Run the LLM on the given input."""
+        infer_cfg = {'stop': stop}
+
+        response = self.model._model_generate(prompt, infer_cfg)
+        return response
+
+    @property
+    def _identifying_params(self) -> Dict[str, Any]:
+        """Return a dictionary of identifying parameters."""
+        return {
+            # The model name allows users to specify custom token counting
+            # rules in LLM monitoring applications (e.g., in LangSmith users
+            # can provide per token pricing for their model and monitor
+            # costs for the given LLM.)
+            'model_name': self.model_name,
+            'revision': self.model_revision,
+        }
+
+    @property
+    def _llm_type(self) -> str:
+        """Get the type of language model used by this chat model. Used for logging purposes only."""
+        return self.model_name

evalscope/backend/rag_eval/utils/tools.py

@@ -0,0 +1,63 @@
+import io
+import os
+import base64
+from modelscope import snapshot_download
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+
+def PIL_to_bytes(image_format, **kwargs):
+    OPTIONS = {
+        "webp": dict(format="webp", lossless=True),
+        "png": dict(format="png"),
+        "jpg": dict(format="jpeg"),
+    }
+
+    def transform(image):
+        bytestream = io.BytesIO()
+        image.save(bytestream, **OPTIONS[image_format])
+        return bytestream.getvalue()
+
+    return transform
+
+
+def PIL_to_base64(image, **kwargs):
+    bytestream = io.BytesIO()
+    image.save(bytestream, format="jpeg")
+    return base64.b64encode(bytestream.getvalue()).decode("utf-8")
+
+
+def path_to_bytes(filepath):
+    with open(filepath, "rb") as fp:
+        return fp.read()
+
+
+def path_to_base64(filepath):
+    file_content = path_to_bytes(filepath)
+    return base64.b64encode(file_content).decode("utf-8")
+
+
+def ensure_dir(file_path):
+    os.makedirs(os.path.dirname(file_path), exist_ok=True)
+
+
+def save_to_jsonl(df, file_path):
+    ensure_dir(file_path)
+    df.to_json(file_path, orient="records", lines=True, force_ascii=False)
+
+
+def save_to_tsv(df, file_path):
+    ensure_dir(file_path)
+    df.to_csv(file_path, sep="\t", index=False)
+
+
+def download_model(model_id: str, revision: str):
+    """
+    default base dir: '~/.cache/modelscope/hub/model_id'
+    """
+    logger.info(f"Loading model {model_id} from modelscope")
+
+    model_path = snapshot_download(model_id=model_id, revision=revision)
+
+    return model_path

evalscope/metrics/bundled_rouge_score/rouge_scorer.py

@@ -51,7 +51,7 @@ try:
     punkt_tab_url = 'https://modelscope-open.oss-cn-hangzhou.aliyuncs.com/open_data/nltk_data/punkt_tab.zip'
 
     if not os.path.exists(punkt_path):
-        os.system(f'wget -P {nltk_dir} {punkt_tab_url}')
+        os.system(f'wget --timeout=10 --tries=3 -P {nltk_dir} {punkt_tab_url}')
         os.system(f'unzip {punkt_path} -d {nltk_dir}')
     else:
         logger.info(f'{punkt_path} already exists, skipping download')

evalscope/version.py

@@ -1,4 +1,4 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 
-__version__ = "0.6.0rc0"
-__release_datetime__ = "2099-01-01 00:00:00"
+__version__ = "0.6.1"
+__release_datetime__ = "2024-11-22 15:00:00"

{evalscope-0.6.0rc0.dist-info → evalscope-0.6.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: evalscope
-Version: 0.6.0rc0
+Version: 0.6.1
 Summary: EvalScope: Lightweight LLMs Evaluation Framework
 Home-page: https://github.com/modelscope/evalscope
 Author: ModelScope team
@@ -19,22 +19,22 @@ Requires-Dist: torch
 Requires-Dist: absl-py
 Requires-Dist: accelerate
 Requires-Dist: cachetools
-Requires-Dist: datasets (<=3.0.1,>=3.0.0)
+Requires-Dist: datasets<=3.0.1,>=3.0.0
 Requires-Dist: editdistance
 Requires-Dist: jsonlines
 Requires-Dist: matplotlib
 Requires-Dist: modelscope[framework]
-Requires-Dist: nltk (>=3.9)
+Requires-Dist: nltk>=3.9
 Requires-Dist: openai
 Requires-Dist: pandas
 Requires-Dist: plotly
-Requires-Dist: pyarrow (<=17.0.0)
+Requires-Dist: pyarrow<=17.0.0
 Requires-Dist: pympler
 Requires-Dist: pyyaml
 Requires-Dist: regex
 Requires-Dist: requests
 Requires-Dist: requests-toolbelt
-Requires-Dist: rouge-score (>=0.1.0)
+Requires-Dist: rouge-score>=0.1.0
 Requires-Dist: sacrebleu
 Requires-Dist: scikit-learn
 Requires-Dist: seaborn
@@ -43,82 +43,82 @@ Requires-Dist: simple-ddl-parser
 Requires-Dist: tabulate
 Requires-Dist: tiktoken
 Requires-Dist: tqdm
-Requires-Dist: transformers (>=4.33)
+Requires-Dist: transformers>=4.33
 Requires-Dist: transformers-stream-generator
 Requires-Dist: jieba
 Requires-Dist: rouge-chinese
 Provides-Extra: all
-Requires-Dist: torch ; extra == 'all'
-Requires-Dist: absl-py ; extra == 'all'
-Requires-Dist: accelerate ; extra == 'all'
-Requires-Dist: cachetools ; extra == 'all'
-Requires-Dist: datasets (<=3.0.1,>=3.0.0) ; extra == 'all'
-Requires-Dist: editdistance ; extra == 'all'
-Requires-Dist: jsonlines ; extra == 'all'
-Requires-Dist: matplotlib ; extra == 'all'
-Requires-Dist: modelscope[framework] ; extra == 'all'
-Requires-Dist: nltk (>=3.9) ; extra == 'all'
-Requires-Dist: openai ; extra == 'all'
-Requires-Dist: pandas ; extra == 'all'
-Requires-Dist: plotly ; extra == 'all'
-Requires-Dist: pyarrow (<=17.0.0) ; extra == 'all'
-Requires-Dist: pympler ; extra == 'all'
-Requires-Dist: pyyaml ; extra == 'all'
-Requires-Dist: regex ; extra == 'all'
-Requires-Dist: requests ; extra == 'all'
-Requires-Dist: requests-toolbelt ; extra == 'all'
-Requires-Dist: rouge-score (>=0.1.0) ; extra == 'all'
-Requires-Dist: sacrebleu ; extra == 'all'
-Requires-Dist: scikit-learn ; extra == 'all'
-Requires-Dist: seaborn ; extra == 'all'
-Requires-Dist: sentencepiece ; extra == 'all'
-Requires-Dist: simple-ddl-parser ; extra == 'all'
-Requires-Dist: tabulate ; extra == 'all'
-Requires-Dist: tiktoken ; extra == 'all'
-Requires-Dist: tqdm ; extra == 'all'
-Requires-Dist: transformers (>=4.33) ; extra == 'all'
-Requires-Dist: transformers-stream-generator ; extra == 'all'
-Requires-Dist: jieba ; extra == 'all'
-Requires-Dist: rouge-chinese ; extra == 'all'
-Requires-Dist: ms-opencompass (>=0.1.3) ; extra == 'all'
-Requires-Dist: ms-vlmeval (>=0.0.5) ; extra == 'all'
-Requires-Dist: mteb (==1.19.4) ; extra == 'all'
-Requires-Dist: ragas (==0.2.3) ; extra == 'all'
-Requires-Dist: webdataset (>0.2.0) ; extra == 'all'
+Requires-Dist: torch; extra == "all"
+Requires-Dist: absl-py; extra == "all"
+Requires-Dist: accelerate; extra == "all"
+Requires-Dist: cachetools; extra == "all"
+Requires-Dist: datasets<=3.0.1,>=3.0.0; extra == "all"
+Requires-Dist: editdistance; extra == "all"
+Requires-Dist: jsonlines; extra == "all"
+Requires-Dist: matplotlib; extra == "all"
+Requires-Dist: modelscope[framework]; extra == "all"
+Requires-Dist: nltk>=3.9; extra == "all"
+Requires-Dist: openai; extra == "all"
+Requires-Dist: pandas; extra == "all"
+Requires-Dist: plotly; extra == "all"
+Requires-Dist: pyarrow<=17.0.0; extra == "all"
+Requires-Dist: pympler; extra == "all"
+Requires-Dist: pyyaml; extra == "all"
+Requires-Dist: regex; extra == "all"
+Requires-Dist: requests; extra == "all"
+Requires-Dist: requests-toolbelt; extra == "all"
+Requires-Dist: rouge-score>=0.1.0; extra == "all"
+Requires-Dist: sacrebleu; extra == "all"
+Requires-Dist: scikit-learn; extra == "all"
+Requires-Dist: seaborn; extra == "all"
+Requires-Dist: sentencepiece; extra == "all"
+Requires-Dist: simple-ddl-parser; extra == "all"
+Requires-Dist: tabulate; extra == "all"
+Requires-Dist: tiktoken; extra == "all"
+Requires-Dist: tqdm; extra == "all"
+Requires-Dist: transformers>=4.33; extra == "all"
+Requires-Dist: transformers-stream-generator; extra == "all"
+Requires-Dist: jieba; extra == "all"
+Requires-Dist: rouge-chinese; extra == "all"
+Requires-Dist: ms-opencompass>=0.1.3; extra == "all"
+Requires-Dist: ms-vlmeval>=0.0.5; extra == "all"
+Requires-Dist: mteb==1.19.4; extra == "all"
+Requires-Dist: ragas==0.2.5; extra == "all"
+Requires-Dist: webdataset>0.2.0; extra == "all"
 Provides-Extra: inner
-Requires-Dist: absl-py ; extra == 'inner'
-Requires-Dist: accelerate ; extra == 'inner'
-Requires-Dist: alibaba-itag-sdk ; extra == 'inner'
-Requires-Dist: dashscope ; extra == 'inner'
-Requires-Dist: editdistance ; extra == 'inner'
-Requires-Dist: jsonlines ; extra == 'inner'
-Requires-Dist: nltk ; extra == 'inner'
-Requires-Dist: openai ; extra == 'inner'
-Requires-Dist: pandas (==1.5.3) ; extra == 'inner'
-Requires-Dist: plotly ; extra == 'inner'
-Requires-Dist: pyarrow ; extra == 'inner'
-Requires-Dist: pyodps ; extra == 'inner'
-Requires-Dist: pyyaml ; extra == 'inner'
-Requires-Dist: regex ; extra == 'inner'
-Requires-Dist: requests (==2.28.1) ; extra == 'inner'
-Requires-Dist: requests-toolbelt (==0.10.1) ; extra == 'inner'
-Requires-Dist: rouge-score ; extra == 'inner'
-Requires-Dist: sacrebleu ; extra == 'inner'
-Requires-Dist: scikit-learn ; extra == 'inner'
-Requires-Dist: seaborn ; extra == 'inner'
-Requires-Dist: simple-ddl-parser ; extra == 'inner'
-Requires-Dist: streamlit ; extra == 'inner'
-Requires-Dist: tqdm ; extra == 'inner'
-Requires-Dist: transformers (<4.43,>=4.33) ; extra == 'inner'
-Requires-Dist: transformers-stream-generator ; extra == 'inner'
+Requires-Dist: absl-py; extra == "inner"
+Requires-Dist: accelerate; extra == "inner"
+Requires-Dist: alibaba-itag-sdk; extra == "inner"
+Requires-Dist: dashscope; extra == "inner"
+Requires-Dist: editdistance; extra == "inner"
+Requires-Dist: jsonlines; extra == "inner"
+Requires-Dist: nltk; extra == "inner"
+Requires-Dist: openai; extra == "inner"
+Requires-Dist: pandas==1.5.3; extra == "inner"
+Requires-Dist: plotly; extra == "inner"
+Requires-Dist: pyarrow; extra == "inner"
+Requires-Dist: pyodps; extra == "inner"
+Requires-Dist: pyyaml; extra == "inner"
+Requires-Dist: regex; extra == "inner"
+Requires-Dist: requests==2.28.1; extra == "inner"
+Requires-Dist: requests-toolbelt==0.10.1; extra == "inner"
+Requires-Dist: rouge-score; extra == "inner"
+Requires-Dist: sacrebleu; extra == "inner"
+Requires-Dist: scikit-learn; extra == "inner"
+Requires-Dist: seaborn; extra == "inner"
+Requires-Dist: simple-ddl-parser; extra == "inner"
+Requires-Dist: streamlit; extra == "inner"
+Requires-Dist: tqdm; extra == "inner"
+Requires-Dist: transformers<4.43,>=4.33; extra == "inner"
+Requires-Dist: transformers-stream-generator; extra == "inner"
 Provides-Extra: opencompass
-Requires-Dist: ms-opencompass (>=0.1.3) ; extra == 'opencompass'
+Requires-Dist: ms-opencompass>=0.1.3; extra == "opencompass"
 Provides-Extra: rag
-Requires-Dist: mteb (==1.19.4) ; extra == 'rag'
-Requires-Dist: ragas (==0.2.3) ; extra == 'rag'
-Requires-Dist: webdataset (>0.2.0) ; extra == 'rag'
+Requires-Dist: mteb==1.19.4; extra == "rag"
+Requires-Dist: ragas==0.2.5; extra == "rag"
+Requires-Dist: webdataset>0.2.0; extra == "rag"
 Provides-Extra: vlmeval
-Requires-Dist: ms-vlmeval (>=0.0.5) ; extra == 'vlmeval'
+Requires-Dist: ms-vlmeval>=0.0.5; extra == "vlmeval"
 
 
 
@@ -139,6 +139,7 @@ Requires-Dist: ms-vlmeval (>=0.0.5) ; extra == 'vlmeval'
  <a href="https://evalscope.readthedocs.io/en/latest/">📖 Documents</a>
 <p>
 
+> ⭐ If you like this project, please click the "Star" button at the top right to support us. Your support is our motivation to keep going!
 
 ## 📋 Table of Contents
 - [Introduction](#introduction)
@@ -164,7 +165,7 @@ EvalScope is the official model evaluation and performance benchmarking framewor
 The architecture includes the following modules:
 1. **Model Adapter**: The model adapter is used to convert the outputs of specific models into the format required by the framework, supporting both API call models and locally run models.
 2. **Data Adapter**: The data adapter is responsible for converting and processing input data to meet various evaluation needs and formats.
-3. **Evaluation Backend**: 
+3. **Evaluation Backend**:
     - **Native**: EvalScope’s own **default evaluation framework**, supporting various evaluation modes, including single model evaluation, arena mode, baseline model comparison mode, etc.
     - **OpenCompass**: Supports [OpenCompass](https://github.com/open-compass/opencompass) as the evaluation backend, providing advanced encapsulation and task simplification, allowing you to submit tasks for evaluation more easily.
     - **VLMEvalKit**: Supports [VLMEvalKit](https://github.com/open-compass/VLMEvalKit) as the evaluation backend, enabling easy initiation of multi-modal evaluation tasks, supporting various multi-modal models and datasets.
@@ -251,7 +252,7 @@ You can execute this command from any directory:
 python -m evalscope.run \
  --model qwen/Qwen2-0.5B-Instruct \
  --template-type qwen \
- --datasets arc 
+ --datasets arc
 ```
 
 #### Install from source
@@ -358,13 +359,13 @@ EvalScope supports using third-party evaluation frameworks to initiate evaluatio
 EvalScope supports custom dataset evaluation. For detailed information, please refer to the Custom Dataset Evaluation [📖User Guide](https://evalscope.readthedocs.io/en/latest/advanced_guides/custom_dataset.html)
 
 ## Offline Evaluation
-You can use local dataset to evaluate the model without internet connection. 
+You can use local dataset to evaluate the model without internet connection.
 
 Refer to: Offline Evaluation [📖 User Guide](https://evalscope.readthedocs.io/en/latest/user_guides/offline_evaluation.html)
 
 
 ## Arena Mode
-The Arena mode allows multiple candidate models to be evaluated through pairwise battles, and can choose to use the AI Enhanced Auto-Reviewer (AAR) automatic evaluation process or manual evaluation to obtain the evaluation report. 
+The Arena mode allows multiple candidate models to be evaluated through pairwise battles, and can choose to use the AI Enhanced Auto-Reviewer (AAR) automatic evaluation process or manual evaluation to obtain the evaluation report.
 
 Refer to: Arena Mode [📖 User Guide](https://evalscope.readthedocs.io/en/latest/user_guides/arena.html)
 

{evalscope-0.6.0rc0.dist-info → evalscope-0.6.1.dist-info}/RECORD

@@ -6,7 +6,7 @@ evalscope/run.py,sha256=uAXtaxIBcR94jyfHGFAecuzn0y71oLgu-d9VOohCJAw,18738
 evalscope/run_arena.py,sha256=BCWCAiX0BQ9pLMIq08svEcd-IoFr75gFShpV88robIY,8963
 evalscope/run_ms.py,sha256=UtJoGnah64SXigTawJQWTi_TEGjr7Td0rjCTaO-htL8,6028
 evalscope/summarizer.py,sha256=rIyML8HpjQxIpXg8KvQ0CzOS6xMS-JHZh6kUZzkaRsk,6640
-evalscope/version.py,sha256=HbExGw191bJuKShYz5RiaxbmdfvIqJQ_bjIjXZhfMDw,121
+evalscope/version.py,sha256=o4SLhBjhMLzVbUK1flGxf-kiqIBLnLnJbxG06BmvkyU,118
 evalscope/backend/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 evalscope/backend/base.py,sha256=5BLrDNNwxsGp35zorD-kphmN15tlBbkuuqwkz8jWZq0,876
 evalscope/backend/opencompass/__init__.py,sha256=UP_TW5KBq6V_Nvqkeb7PGvGGX3rVYussT43npwCwDgE,135
@@ -14,7 +14,7 @@ evalscope/backend/opencompass/api_meta_template.py,sha256=sBW0XbVDOKeJ7mVUDLhmcG
 evalscope/backend/opencompass/backend_manager.py,sha256=_eg82FLAVxQ6t5e1OqlyuxZcngqD8rxvI5EijLUh_zI,10294
 evalscope/backend/opencompass/tasks/__init__.py,sha256=I_ANdxdcIHpkIzIXc1yKOlWwzb4oY0FwTPq1kYtgzQw,50
 evalscope/backend/opencompass/tasks/eval_api.py,sha256=12lrgDpMzZ1XBRboq5TEOovDPCMDwwGCJoRT78Ox_yo,1108
-evalscope/backend/opencompass/tasks/eval_datasets.py,sha256=t2t3_dHZf-eMfNqpQaD2XIjWZejTN4AxVXITdj_4Y3o,5324
+evalscope/backend/opencompass/tasks/eval_datasets.py,sha256=3V67A2LSj_XaiGd9fqdKpxpzyNrfynCH3UnhaBtAaqc,5326
 evalscope/backend/rag_eval/__init__.py,sha256=8om6TVnTMmyTEQt1jBuUQA4UfIzyps-_-ih90H_Qjio,284
 evalscope/backend/rag_eval/backend_manager.py,sha256=jmO-UMu6_iOXMnl4--PrMWCsnIYEhsbiX017rtURqm0,2997
 evalscope/backend/rag_eval/clip_benchmark/__init__.py,sha256=gDXCiRUTSeGQHxd5SjQsnphMqHJ2si2jywRiHvujEOg,150
@@ -44,8 +44,13 @@ evalscope/backend/rag_eval/ragas/metrics/__init__.py,sha256=HgY5nrcNtWpQ7gBi5lCE
 evalscope/backend/rag_eval/ragas/metrics/multi_modal_faithfulness.py,sha256=Uqz5qWZ76Gos95_QlhwncbATXyk0YX4wkI0LiAdPElU,3838
 evalscope/backend/rag_eval/ragas/metrics/multi_modal_relevance.py,sha256=CdLnWHq1eTna6j3F5-pncW5YusxD_v3ScjzeCsZ7mng,3967
 evalscope/backend/rag_eval/ragas/tasks/__init__.py,sha256=WO2xja0g0JSiYGdu2uAEDQgDceuFcgPWwPoqFnwDU0s,172
-evalscope/backend/rag_eval/ragas/tasks/testset_generation.py,sha256=In-2VvZJIZvXl9idGUUQBTb7Gu-o1yFLjaqj-eJkWw0,8437
+evalscope/backend/rag_eval/ragas/tasks/testset_generation.py,sha256=nX-dG0Fm1629pSASujuEmMODFZf1955WncNNykRrNtI,9305
 evalscope/backend/rag_eval/ragas/tasks/translate_prompt.py,sha256=bXOqik6qKWzbrEz21ykdkqeqqPrmoUIhTwW6eRQXy0M,2222
+evalscope/backend/rag_eval/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+evalscope/backend/rag_eval/utils/clip.py,sha256=frafvJ1soUtjFUmi-053_Fhg6ERRwyvczQBlLWAX9vE,5104
+evalscope/backend/rag_eval/utils/embedding.py,sha256=RZf0JlovZY_cCBsq8MMUqC_Sy78WtKLY_rBAlRA_udo,6239
+evalscope/backend/rag_eval/utils/llm.py,sha256=9tFwMNoTf3jNomgDu5qqVLO92HtEtelH3DXpny9_B2g,2552
+evalscope/backend/rag_eval/utils/tools.py,sha256=LpcYoeIBj1btzQ1_P84u1dYCdRWhMtiltxihmZCvWKk,1528
 evalscope/backend/vlm_eval_kit/__init__.py,sha256=xTgHM95lWzh4s0W7zxLwYkgUbPAZfAb0UoGGmyyBXrs,83
 evalscope/backend/vlm_eval_kit/backend_manager.py,sha256=ewhpE9yzsqf5ED6kqsqek2YEgg96GBQOupxtVNhaXxI,6046
 evalscope/backend/vlm_eval_kit/custom_dataset.py,sha256=Yz2A5kB1E8DYBnjuVCA6TTPtLjhg8vYKeJTh6FU_Ecw,1645
@@ -132,7 +137,7 @@ evalscope/metrics/math_accuracy.py,sha256=1PCy1VUNYg48JcGy-6SUmUDZNwPeAkMW1QQ_lX
 evalscope/metrics/metrics.py,sha256=sDZljGiZwgHsFZ5eNi65-3z3BLCdIwWUzPcq2QpKf1k,12545
 evalscope/metrics/rouge_metric.py,sha256=sN0r-sXXc-nJUdFrthQPAv1VFdOCrF6zzIYDKaLSgrU,4522
 evalscope/metrics/bundled_rouge_score/__init__.py,sha256=PwbTdk8168FwDJe_l8XIqDuBgZQooDsP31vj7di05Fs,650
-evalscope/metrics/bundled_rouge_score/rouge_scorer.py,sha256=vhzIMSQezhZuJzGndymWjB_iRbDdECoEidOIdNL3NAM,12213
+evalscope/metrics/bundled_rouge_score/rouge_scorer.py,sha256=MXcHwmsXnh9mQZR1Bt5St6DNwXY-mfz4dNM8y6a23dc,12236
 evalscope/models/__init__.py,sha256=zG27J2HSeKPGiAIUE7QLPHEPLyXLsfaDwYI_TDXjpCg,145
 evalscope/models/dummy_chat_model.py,sha256=xE8wcFVSCkvizEJ-B8ojX0Ir01Q5KrN5mapjMQaQtbg,1325
 evalscope/models/model.py,sha256=ZzzVzZHVzuzdt5F1r-rEBT44ZfW9B7R1spsrV-T8nSw,3020
@@ -204,8 +209,8 @@ evalscope/utils/logger.py,sha256=cf3U400Mx1speMMNXorjwEE8noDz5Mbd-9PNgaulGeY,301
 evalscope/utils/task_cfg_parser.py,sha256=LiNQ2X8lbZU0cODpaY_PbKyUhNoxZIC495UsLJigX64,138
 evalscope/utils/task_utils.py,sha256=IMtBSBUp3H95Ko0vn8Q55Wmz2SFZXSfjVy49tyomL_g,537
 evalscope/utils/utils.py,sha256=zHo9hfxGBUVKE2xNMR7lDoEvfRnk4V4946DEfXQhlq4,20509
-evalscope-0.6.0rc0.dist-info/METADATA,sha256=w2k8y1h3gVVNAI7Ey-mc4RWsaSjNBlokuu0hw4e-3aI,21242
-evalscope-0.6.0rc0.dist-info/WHEEL,sha256=2wepM1nk4DS4eFpYrW1TTqPcoGNfHhhO_i5m4cOimbo,92
-evalscope-0.6.0rc0.dist-info/entry_points.txt,sha256=Qr4oTgGhg_K-iUtKwVH6lWUhFHDUiH9trIqydHGTEug,56
-evalscope-0.6.0rc0.dist-info/top_level.txt,sha256=jNR-HMn3TR8Atolq7_4rW8IWVX6GhvYV5_1Y_KbJKlY,10
-evalscope-0.6.0rc0.dist-info/RECORD,,
+evalscope-0.6.1.dist-info/METADATA,sha256=n4CpTzJGnhgqEsfbL1UfZtXHULmeNCGnKChyi6eT8Fw,21237
+evalscope-0.6.1.dist-info/WHEEL,sha256=bFJAMchF8aTQGUgMZzHJyDDMPTO3ToJ7x23SLJa1SVo,92
+evalscope-0.6.1.dist-info/entry_points.txt,sha256=Qr4oTgGhg_K-iUtKwVH6lWUhFHDUiH9trIqydHGTEug,56
+evalscope-0.6.1.dist-info/top_level.txt,sha256=jNR-HMn3TR8Atolq7_4rW8IWVX6GhvYV5_1Y_KbJKlY,10
+evalscope-0.6.1.dist-info/RECORD,,

{evalscope-0.6.0rc0.dist-info → evalscope-0.6.1.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.38.4)
+Generator: bdist_wheel (0.45.0)
 Root-Is-Purelib: true
 Tag: py3-none-any