evalscope 0.8.0__py3-none-any.whl → 0.10.1__py3-none-any.whl

evalscope/backend/rag_eval/ragas/prompts/chinese/SingleHopSpecificQuerySynthesizer/theme_persona_matching_prompt_chinese.json

@@ -1,39 +0,0 @@
-{
-  "ragas_version": "0.2.7",
-  "original_hash": 4608101540215877909,
-  "language": "chinese",
-  "instruction": "给定一个主题和角色列表，根据角色描述将每个角色与相关主题关联起来。",
-  "examples": [
-    {
-      "input": {
-        "themes": [
-          "同理心",
-          "包容性",
-          "远程工作"
-        ],
-        "personas": [
-          {
-            "name": "人力资源经理",
-            "role_description": "专注于包容性和员工支持。"
-          },
-          {
-            "name": "远程团队负责人",
-            "role_description": "管理远程团队沟通。"
-          }
-        ]
-      },
-      "output": {
-        "mapping": {
-          "HR Manager": [
-            "包容性",
-            "同理心"
-          ],
-          "Remote Team Lead": [
-            "远程工作",
-            "同理心"
-          ]
-        }
-      }
-    }
-  ]
-}

evalscope/backend/rag_eval/ragas/prompts/chinese/SummaryExtractor/prompt_chinese.json

@@ -1,16 +0,0 @@
-{
-  "ragas_version": "0.2.7",
-  "original_hash": -2203889341293275650,
-  "language": "chinese",
-  "instruction": "将给定文本总结为不超过10个句子。",
-  "examples": [
-    {
-      "input": {
-        "text": "人工智能\n\n人工智能正在通过自动化以前需要人类智能的任务来改变各个行业。从医疗保健到金融，人工智能正在被用来快速准确地分析大量数据。这项技术还推动了自动驾驶汽车和个性化推荐等领域的创新。"
-      },
-      "output": {
-        "text": "人工智能通过自动化任务、分析数据和推动自动驾驶汽车和个性化推荐等创新，正在革新各个行业。"
-      }
-    }
-  ]
-}

evalscope/backend/rag_eval/ragas/prompts/chinese/ThemesExtractor/prompt_chinese.json

@@ -1,24 +0,0 @@
-{
-  "ragas_version": "0.2.7",
-  "original_hash": -7344189172470926110,
-  "language": "chinese",
-  "instruction": "从给定的文本中提取主要主题和概念。",
-  "examples": [
-    {
-      "input": {
-        "text": "人工智能通过自动化需要人类智能的任务来改变行业。人工智能快速准确地分析大量数据，推动了自动驾驶汽车和个性化推荐等创新。",
-        "max_num": 10
-      },
-      "output": {
-        "output": [
-          "人工智能",
-          "自动化",
-          "数据分析",
-          "创新",
-          "自动驾驶汽车",
-          "个性化推荐"
-        ]
-      }
-    }
-  ]
-}

evalscope/evaluator/humaneval_evaluator.py

@@ -1,158 +0,0 @@
-import json
-import os
-import re
-from tqdm import tqdm
-from typing import List, Optional
-
-from evalscope.constants import OutputsStructure
-from evalscope.evaluator.evaluator import logger
-from evalscope.models.model_adapter import BaseModelAdapter
-from evalscope.tools.combine_reports import gen_table
-from evalscope.utils import normalize_score
-
-
-class HumanevalEvaluator(object):
-
-    def __init__(
-        self,
-        problem_file: str,
-        model_id: str,
-        model_revision: str,
-        model_adapter: BaseModelAdapter,
-        outputs: Optional[OutputsStructure] = None,
-        k: List[int] = [1, 10, 100],
-        n_workers: int = 4,
-        timeout: float = 3.0,
-    ):
-        try:
-            from human_eval.data import read_problems, write_jsonl
-            from human_eval.evaluation import evaluate_functional_correctness
-        except ImportError:
-            raise ImportError('Please install human_eval:'
-                              'https://github.com/openai/human-eval/tree/master#installation , '
-                              'Note that you need to enable the execution code in the human_eval/execution.py first.')
-
-        self.problem_file = problem_file
-        self.k = k
-        self.num_workers = n_workers
-        self.timeout = timeout
-        self.model_adapter = model_adapter
-
-        self.read_problems_func = read_problems
-        self.write_jsonl_func = write_jsonl
-        self.eval_func = evaluate_functional_correctness
-
-        # {'task_id': {'task_id': '', 'prompt': '', 'entry_point': '', 'canonical_solution': '', 'test': ''}, ...}
-        self.problems = self.read_problems_func(self.problem_file)
-
-        # Deal with the output paths
-        self.outputs_structure = OutputsStructure(outputs)
-
-    def get_answers(self, infer_cfg: dict) -> List[dict]:
-        ans_list: list = []
-        system_prompt: str = 'Complete the following python code:\n'
-        for task_id, data_d in tqdm(self.problems.items(), total=len(self.problems), desc='Predicting(problems)'):
-            prompt: str = system_prompt + data_d['prompt']
-            inputs: dict = {'data': [prompt]}
-            # pred_res: dict = self.model_adapter.predict(inputs)
-
-            pred_res: dict = self.model_adapter.predict(inputs=inputs, infer_cfg=infer_cfg)
-
-            pred_ans: str = pred_res['choices'][0]['message']['content']
-            pred_ans = self._postprocess(pred_ans)
-
-            ans_list.append({'task_id': task_id, 'completion': pred_ans})
-
-        return ans_list
-
-    def eval(self, infer_cfg: dict, **kwargs):
-
-        # predict
-        ans_list: list = self.get_answers(infer_cfg)
-        ans_out_file: str = os.path.join(self.outputs_structure.predictions_dir, 'human_eval_predictions.jsonl')
-
-        self.write_jsonl_func(filename=ans_out_file, data=ans_list)
-        # logger.info(f'** Dump predictions to {ans_out_file} successfully.')
-        logger.info('** Dump predictions successfully.')
-
-        # evaluate  results: e.g. {'pass@1': 0.333, 'pass@10': 0.111}
-        results = self.eval_func(
-            sample_file=ans_out_file,
-            k=self.k,
-            n_workers=self.num_workers,
-            timeout=self.timeout,
-            problem_file=self.problem_file)
-
-        # output: report
-        report_map: dict = self.gen_report(results=results)
-        report_dir: str = self.outputs_structure.reports_dir
-        report_file: str = os.path.join(report_dir, 'human_eval_report.json')
-
-        with open(report_file, 'w') as f:
-            f.write(json.dumps(report_map, ensure_ascii=False, indent=4))
-        # logger.info(f'** Dump report to {report_file} \n')
-        logger.info('** Dump report \n')
-
-        try:
-            # Make table
-            report_table: str = gen_table([report_dir])
-            logger.info(f'** Report table: \n {report_table} \n')
-        except Exception:
-            logger.error('Failed to generate report table.')
-
-    def gen_report(self, results: dict) -> dict:
-        """
-        Generate report from evaluation results.
-
-        Returns:
-            {
-            "name":"ARC-Challenge",
-            "metric":"WeightedAverageAccuracy",
-            "score":0.3389,
-            "category":[
-                {
-                    "name":"DEFAULT",
-                    "score":0.3389,
-                    "subset":[
-                        {
-                            "name":"ARC-Challenge",
-                            "score":0.3389
-                        },
-                    ]
-                }
-            ],
-            "total_num":100
-        }
-        """
-        results = {k: normalize_score(score=v) for k, v in results.items()}
-
-        category_d = dict(name='DEFAULT', score=results, subset=[])
-
-        res_map = dict(
-            name='HumanEval', metric='pass@k', score=results, category=[category_d], total_num=len(self.problems))
-
-        return res_map
-
-    @classmethod
-    def _postprocess(cls, text: str) -> str:
-        if '```' in text:
-            blocks = re.findall(r'```(.*?)```', text, re.DOTALL)
-            if len(blocks) == 0:
-                text = text.split('```')[1]  # fall back to default strategy
-            else:
-                text = blocks[0]  # fetch the first code block
-                if not text.startswith('\n'):  # in case starting with ```python
-                    text = text[max(text.find('\n') + 1, 0):]
-        if text.strip().startswith('from') or text.strip().startswith('import'):
-            def_idx = text.find('def')
-            if def_idx != -1:
-                text = text[max(text.find('\n', def_idx) + 1, 0):]
-        text = text.split('\n\n')[0]
-        if text.strip().startswith('def'):
-            text = '\n'.join(text.split('\n')[1:])
-        if not text.startswith('    '):
-            if text.startswith(' '):
-                text = '    ' + text.lstrip()
-            else:
-                text = '\n'.join(['    ' + line for line in text.split('\n')])
-        return text

evalscope/models/api/__init__.py

@@ -1,3 +0,0 @@
-# Copyright (c) Alibaba, Inc. and its affiliates.
-
-from evalscope.models.api.openai_api import OpenaiApi

evalscope/models/dummy_chat_model.py

@@ -1,49 +0,0 @@
-# Copyright (c) Alibaba, Inc. and its affiliates.
-
-import random
-import time
-
-from evalscope.models import ChatBaseModel
-from evalscope.utils.logger import get_logger
-
-logger = get_logger()
-
-
-class DummyChatModel(ChatBaseModel):
-
-    MODEL_ID = 'dummy_chat_model_0801'
-    REVISION = 'v1.0.0'
-
-    def __init__(self, model_cfg: dict, **kwargs):
-        model_cfg['model_id'] = self.MODEL_ID
-        model_cfg['revision'] = self.REVISION
-        super(DummyChatModel, self).__init__(model_cfg=model_cfg)
-
-    def predict(self, inputs: dict, **kwargs) -> dict:
-
-        debug: bool = False
-        if debug:
-            messages = inputs['messages']
-            history = inputs['history']
-
-            logger.info(f'** messages: {messages}')
-            logger.info(f'** history: {history}')
-
-        choice = random.choice(['A', 'B', 'C', 'D'])
-
-        # Build response
-        res = {
-            'choices': [{
-                'index': 0,
-                'message': {
-                    'content': choice,
-                    'role': 'assistant'
-                }
-            }],
-            'created': time.time(),
-            'model': self.MODEL_ID + '-' + self.REVISION,
-            'object': 'chat.completion',
-            'usage': {}
-        }
-
-        return res