evalscope 0.10.1__py3-none-any.whl → 0.11.0__py3-none-any.whl

evalscope/models/chat_adapter.py

@@ -1,7 +1,7 @@
 import os
 import time
 import torch
-from typing import Union
+from typing import List, Union
 
 from evalscope.models.base_adapter import BaseModelAdapter
 from evalscope.models.local_model import LocalModel
@@ -57,84 +57,104 @@ class ChatGenerationModelAdapter(BaseModelAdapter):
 
         return generation_config
 
-    def _model_generate(self, query: str, system_prompt: str = None, infer_cfg: dict = {}) -> str:
+    def _model_generate(self, queries: List[str], system_prompts: List[str] = None, infer_cfg: dict = {}) -> List[str]:
         """
         Args:
-            query: The input query.
-            system_prompt: The system prompt.
+            queries: The input queries.
+            system_prompts: The system prompts.
             infer_cfg: The inference configuration.
         Returns:
-            The prediction result.
+            The prediction results.
         """
-        # For chat model, use the chat template to format the input
-        if self.tokenizer.chat_template is not None:
-            messages = [ChatMessage(role='user', content=query)]
-            if system_prompt:
-                messages = [ChatMessage(role='system', content=system_prompt)] + messages
-            formatted_prompt = self.tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
-        else:
-            # For base model, use the query as the input
-            formatted_prompt = query
-
-        logger.debug(f'formatted_prompt: {formatted_prompt}')
-
-        inputs = self.tokenizer(formatted_prompt, return_tensors='pt', padding=True).to(self.device)
-        input_ids = inputs['input_ids']
-
         # Process infer_cfg
-        if isinstance(infer_cfg.get('num_return_sequences'), int) and infer_cfg['num_return_sequences'] > 1:
+        num_return_sequences = infer_cfg.get('num_return_sequences', 1)
+        if num_return_sequences > 1:
             infer_cfg['do_sample'] = True
 
         # stop settings
-        stop = infer_cfg.get('stop', None)
-        eos_token_id = self.tokenizer.encode(stop, add_special_tokens=False)[0] \
-            if stop else self.tokenizer.eos_token_id
+        stop = infer_cfg.get('stop', [])
+        if stop:
+            eos_token_id = self.tokenizer.encode(stop, add_special_tokens=False)[0]
+        else:
+            eos_token_id = self.tokenizer.eos_token_id
 
         if eos_token_id is not None:
             infer_cfg['eos_token_id'] = eos_token_id
-            infer_cfg['pad_token_id'] = eos_token_id  # setting eos_token_id as pad token
 
         self.generation_config.update(**infer_cfg)
         fix_do_sample_warning(self.generation_config)
 
+        # For chat model, use the chat template to format the input
+        if self.tokenizer.chat_template is not None:
+            formatted_prompts = []
+            for i, query in enumerate(queries):
+                messages = [ChatMessage(role='user', content=query)]
+                if i < len(system_prompts) and system_prompts[i]:
+                    messages = [ChatMessage(role='system', content=system_prompts[i])] + messages
+                formatted_prompts.append(
+                    self.tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True))
+        else:
+            # For base model, use the queries as the input
+            formatted_prompts = queries
+
+        logger.debug(f'formatted_prompts: {formatted_prompts}')
+
+        # Get input ids
+        inputs = self.tokenizer(
+            formatted_prompts, return_tensors='pt', padding=True, truncation=True,
+            padding_side='left').to(self.device)  # padding_side='left' is important for chat model
+        input_ids = inputs['input_ids']
+
         # Run inference
         output_ids = self.model.generate(**inputs, generation_config=self.generation_config)
 
-        response = self.tokenizer.decode(output_ids[0, len(input_ids[0]):], skip_special_tokens=True)
-        return response
+        responses = []
+        for i in range(0, len(output_ids), num_return_sequences):
+            query_responses = []
+            for j in range(num_return_sequences):
+                output = output_ids[i + j]
+                response = self.tokenizer.decode(
+                    output[len(input_ids[i // num_return_sequences]):], skip_special_tokens=True)
+                query_responses.append(response)
+            responses.append(query_responses)
+
+        return responses
 
     @torch.no_grad()
-    def predict(self, inputs: Union[str, dict, list], infer_cfg: dict = {}) -> dict:
+    def predict(self, inputs: List[dict], infer_cfg: dict = {}) -> List[dict]:
         """
         Args:
             inputs: The input data.
             infer_cfg: The inference configuration.
         Returns:
-            The prediction result.
+            The prediction results.
         """
 
         # Process inputs
-        if isinstance(inputs, str):
-            query = inputs
-            system_prompt = None
-        elif isinstance(inputs, dict):
-            query = inputs['data'][0]
-            system_prompt = inputs.get('system_prompt', None)
-        elif isinstance(inputs, list):
-            query = '\n'.join(inputs)
-            system_prompt = None
-        else:
-            raise TypeError(f'Unsupported inputs type: {type(inputs)}')
+        queries = []
+        system_prompts = []
+
+        for input_item in inputs:
+            queries.append(input_item['data'][0])
+            system_prompts.append(input_item.get('system_prompt', None))
+
+        responses = self._model_generate(queries, system_prompts, infer_cfg)
 
-        response = self._model_generate(query, system_prompt, infer_cfg)
+        results = []
+        for response in responses:
+            choices_list = [
+                ChatCompletionResponseChoice(
+                    index=index, message=ChatMessage(content=one_response, role='assistant'), finish_reason='stop')
+                for index, one_response in enumerate(response)
+            ]
 
-        choices_list = [
-            ChatCompletionResponseChoice(
-                index=0, message=ChatMessage(content=response, role='assistant'), finish_reason='stop')
-        ]
+            res_d = ChatCompletionResponse(
+                model=self.model_id,
+                choices=choices_list,
+                object='chat.completion',
+                created=int(time.time()),
+                usage=None).model_dump(exclude_unset=True)
 
-        res_d = ChatCompletionResponse(
-            model=self.model_id, choices=choices_list, object='chat.completion', created=int(time.time()),
-            usage=None).model_dump(exclude_unset=True)
+            results.append(res_d)
 
-        return res_d
+        return results

evalscope/models/choice_adapter.py

@@ -33,12 +33,12 @@ class MultiChoiceModelAdapter(BaseModelAdapter):
         return self._DEFAULT_MAX_LENGTH
 
     @torch.no_grad()
-    def predict(self, inputs: dict, infer_cfg: dict = None) -> dict:
+    def predict(self, inputs: List[dict], infer_cfg: dict = None) -> dict:
         """
         Multi-choice model prediction func.
 
         Args:
-            inputs (dict): The inputs for a doc. Format:
+            inputs (List[dict]): The inputs for a doc. Format:
                 {'data': [full_prompt], 'multi_choices': ['A', 'B', 'C', 'D']}
 
             infer_cfg (dict): inference configuration.
@@ -69,37 +69,39 @@ class MultiChoiceModelAdapter(BaseModelAdapter):
         infer_cfg = infer_cfg or {}
         self.model.generation_config.update(**infer_cfg)
 
-        input_data = inputs['data']
-        multi_choices = inputs['multi_choices']
+        input_data = [inp['data'][0] for inp in inputs]
+        multi_choices = [inp['multi_choices'] for inp in inputs]
 
-        output, input_info = self._get_logits(self.tokenizer, self.model, input_data)
-        assert output.shape[0] == 1
-        logits = output.flatten()
+        outputs, input_info = self._get_logits(self.tokenizer, self.model, input_data)
 
-        choice_logits = [logits[self.tokenizer(ch)['input_ids'][-1:]] for ch in multi_choices]
-        softval = torch.nn.functional.softmax(torch.tensor(choice_logits).float(), dim=0)
+        results = []
+        for i, (logits, choices) in enumerate(zip(outputs, multi_choices)):
+            choice_logits = [logits[self.tokenizer(ch)['input_ids'][-1:]] for ch in choices]
+            softval = torch.nn.functional.softmax(torch.tensor(choice_logits).float(), dim=0)
 
-        if softval.dtype in {torch.bfloat16, torch.float16}:
-            softval = softval.to(dtype=torch.float32)
-        probs = softval.detach().cpu().numpy()
-        pred: str = multi_choices[int(np.argmax(probs))]  # Format: A or B or C or D
+            if softval.dtype in {torch.bfloat16, torch.float16}:
+                softval = softval.to(dtype=torch.float32)
+            probs = softval.detach().cpu().numpy()
+            pred: str = choices[int(np.argmax(probs))]  # Format: A or B or C or D
 
-        res_d = ChatCompletionResponse(
-            model=self.model_id,
-            choices=[
-                ChatCompletionResponseChoice(
-                    index=0, message=ChatMessage(content=pred, role='assistant'), finish_reason='stop')
-            ],
-            object='chat.completion',
-            created=int(time.time()),
-            usage=None).model_dump(exclude_unset=True)
+            res_d = ChatCompletionResponse(
+                model=self.model_id,
+                choices=[
+                    ChatCompletionResponseChoice(
+                        index=0, message=ChatMessage(content=pred, role='assistant'), finish_reason='stop')
+                ],
+                object='chat.completion',
+                created=int(time.time()),
+                usage=None).model_dump(exclude_unset=True)
 
-        return res_d
+            results.append(res_d)
+
+        return results
 
     @staticmethod
     def _get_logits(tokenizer, model, inputs: List[str]):
-        input_ids = tokenizer(inputs, padding=False)['input_ids']
-        input_ids = torch.tensor(input_ids, device=model.device)
+        input_ids = tokenizer(
+            inputs, padding=True, return_tensors='pt', padding_side='left')['input_ids'].to(model.device)
         tokens = {'input_ids': input_ids}
 
         outputs = model(input_ids)['logits']
@@ -117,11 +119,11 @@ class ContinuationLogitsModelAdapter(MultiChoiceModelAdapter):
         super().__init__(model, **kwargs)
 
     @torch.no_grad()
-    def predict(self, inputs: dict, infer_cfg: dict = None) -> dict:
+    def predict(self, inputs: List[dict], infer_cfg: dict = None) -> dict:
         """
         Multi-choice model prediction func.
         Args:
-            inputs (dict): The inputs for a doc. Format:
+            inputs (List[dict]): The inputs for a doc. Format:
                 {'data': [(context, continuation), ...]}
             infer_cfg (dict): inference configuration.
         Returns:
@@ -149,24 +151,29 @@ class ContinuationLogitsModelAdapter(MultiChoiceModelAdapter):
         """
         infer_cfg = infer_cfg or {}
 
-        pred_list: list = self.loglikelihood(inputs=inputs['data'], infer_cfg=infer_cfg)
-
-        res_d = ChatCompletionResponse(
-            model=self.model_id,
-            choices=[{
-                'index': 0,
-                'message': {
-                    'content': pred_list,
-                    'role': 'assistant'
-                }
-            }],
-            object='chat.completion',
-            created=int(time.time()),
-            usage=None).model_dump(exclude_unset=True)
-
-        return res_d
-
-    def loglikelihood(self, inputs: list, infer_cfg: dict = None) -> list:
+        pred_list: list = []
+        for inp in inputs:
+            pred_list.append(self.loglikelihood(inputs=inp['data'], infer_cfg=infer_cfg))
+
+        results = []
+        for pred in pred_list:
+            res_d = ChatCompletionResponse(
+                model=self.model_id,
+                choices=[{
+                    'index': 0,
+                    'message': {
+                        'content': pred,
+                        'role': 'assistant'
+                    }
+                }],
+                object='chat.completion',
+                created=int(time.time()),
+                usage=None).model_dump(exclude_unset=True)
+            results.append(res_d)
+
+        return results
+
+    def loglikelihood(self, inputs: List[tuple], infer_cfg: dict = None) -> list:
         self.model.generation_config.update(**infer_cfg)
         # To predict one doc
         doc_ele_pred = []

evalscope/models/custom_adapter.py

@@ -17,12 +17,12 @@ class CustomModelAdapter(BaseModelAdapter):
         self.custom_model = custom_model
         super(CustomModelAdapter, self).__init__(model=custom_model)
 
-    def predict(self, inputs: Union[str, dict, list], **kwargs) -> List[Dict[str, Any]]:
+    def predict(self, inputs: List[Union[str, dict, list]], **kwargs) -> List[Dict[str, Any]]:
         """
         Model prediction func.
 
         Args:
-            inputs (Union[str, dict, list]): The input data. Depending on the specific model.
+            inputs (List[Union[str, dict, list]]): The input data. Depending on the specific model.
                 str: 'xxx'
                 dict: {'data': [full_prompt]}
                 list: ['xxx', 'yyy', 'zzz']

evalscope/models/local_model.py

@@ -37,6 +37,10 @@ class LocalModel:
             cache_dir=model_cache_dir,
         )
 
+        # Fix no padding
+        if self.tokenizer.pad_token is None:
+            self.tokenizer.pad_token = self.tokenizer.eos_token
+
         self.model = AutoModelForCausalLM.from_pretrained(
             self.model_id,
             revision=model_revision,

evalscope/models/server_adapter.py

@@ -1,6 +1,6 @@
 import requests
 import time
-from typing import Optional, Union
+from typing import List, Optional, Union
 
 from evalscope.models.base_adapter import BaseModelAdapter
 from evalscope.utils.chat_service import ChatMessage
@@ -28,36 +28,35 @@ class ServerModelAdapter(BaseModelAdapter):
         self.model_cfg = {'api_url': api_url, 'model_id': model_id, 'api_key': api_key}
         super().__init__(model=None, model_cfg=self.model_cfg, **kwargs)
 
-    def predict(self, inputs: Union[str, dict, list], infer_cfg: dict = None) -> dict:
+    def predict(self, inputs: List[Union[str, dict, list]], infer_cfg: dict = None) -> List[dict]:
         """
         Model prediction func.
 
         Args:
-            inputs (Union[str, dict, list]): The input data.
+            inputs (List[Union[str, dict, list]]): The input data.
             infer_cfg (dict): Inference configuration.
 
         Returns:
-            res (dict): The model prediction results.
+            res (List[dict]): The model prediction results.
         """
         infer_cfg = infer_cfg or {}
+        results = []
 
-        # Process inputs
-        if isinstance(inputs, str):
-            query = inputs
-            system_prompt = None
-        elif isinstance(inputs, dict):
-            data: list = inputs['data']
-            if isinstance(data[0], tuple):  # for truthful_qa and hellaswag
-                query = '\n'.join(''.join(item) for item in data)
-                system_prompt = inputs.get('system_prompt', None)
-            else:
-                query = data[0]
-                system_prompt = inputs.get('system_prompt', None)
-        elif isinstance(inputs, list):
-            query = '\n'.join(inputs)
-            system_prompt = None
+        for input_item in inputs:
+            response = self.process_single_input(input_item, infer_cfg)
+            results.append(response)
+
+        return results
+
+    def process_single_input(self, input_item: dict, infer_cfg: dict) -> dict:
+        """Process a single input item."""
+        data: list = input_item['data']
+        if isinstance(data[0], tuple):  # for truthful_qa and hellaswag
+            query = '\n'.join(''.join(item) for item in data)
+            system_prompt = input_item.get('system_prompt', None)
         else:
-            raise TypeError(f'Unsupported inputs type: {type(inputs)}')
+            query = data[0]
+            system_prompt = input_item.get('system_prompt', None)
 
         content = self.make_request_content(query, system_prompt)
         request_json = self.make_request(content, infer_cfg)
@@ -68,7 +67,7 @@ class ServerModelAdapter(BaseModelAdapter):
         """
         Make request content for API.
         """
-        if system_prompt is not None:
+        if system_prompt:
             messages = [
                 ChatMessage(role='system', content=system_prompt).model_dump(exclude_unset=True),
                 ChatMessage(role='user', content=query).model_dump(exclude_unset=True)
@@ -80,19 +79,14 @@ class ServerModelAdapter(BaseModelAdapter):
     def make_request(self, content: dict, infer_cfg: dict = {}) -> dict:
         """Make request to remote API."""
         # Format request JSON according to OpenAI API format
-        do_sample = infer_cfg.get('do_sample', False)
-        temperature = infer_cfg.get('temperature', 0.0) if do_sample else 0.0
-
-        request_json = {
-            **content, 'model': self.model_id,
-            'max_tokens': infer_cfg.get('max_tokens', 2048),
-            'temperature': temperature,
-            'top_p': infer_cfg.get('top_p', 1.0),
-            'n': infer_cfg.get('num_return_sequences', 1),
-            'stop': infer_cfg.get('stop', None)
-        }
-        if self.seed is not None:
-            request_json['seed'] = self.seed
+        from evalscope.config import DEFAULT_GENERATION_CONFIG
+        if infer_cfg == DEFAULT_GENERATION_CONFIG:
+            infer_cfg = {
+                'max_tokens': 2048,
+                'temperature': 0.0,
+            }
+
+        request_json = {'model': self.model_id, **content, **infer_cfg}
         logger.debug(f'Request to remote API: {request_json}')
         return request_json
 

evalscope/report/app.py

@@ -6,6 +6,7 @@ import os
 import pandas as pd
 import plotly.express as px
 import plotly.graph_objects as go
+import re
 from dataclasses import dataclass
 from typing import Any, List, Union
 
@@ -218,7 +219,16 @@ def dict_to_markdown(data) -> str:
     return '\n\n'.join(markdown_lines)
 
 
+def convert_html_tags(text):
+    # match begin label
+    text = re.sub(r'<(\w+)>', r'[\1]', text)
+    # match end label
+    text = re.sub(r'</(\w+)>', r'[/\1]', text)
+    return text
+
+
 def process_string(string: str, max_length: int = 2048) -> str:
+    string = convert_html_tags(string)  # for display labels e.g. `<think>`
     if len(string) > max_length:
         return f'{string[:max_length // 2]}......{string[-max_length // 2:]}'
     return string
@@ -226,9 +236,11 @@ def process_string(string: str, max_length: int = 2048) -> str:
 
 def process_model_prediction(item: Any):
     if isinstance(item, dict):
-        return dict_to_markdown(item)
+        res = dict_to_markdown(item)
+        return process_string(res)
     elif isinstance(item, list):
-        return '\n'.join([process_model_prediction(item) for item in item])
+        res = '\n'.join([process_model_prediction(item) for item in item])
+        return process_string(res)
     else:
         return process_string(str(item))
 
@@ -257,19 +269,20 @@ def get_model_prediction(work_dir: str, model_name: str, dataset_name: str, subs
     ds = []
     for i, item in origin_df.iterrows():
         raw_input = item['raw_input']
-        raw_pred_answer = item['choices'][0]['message']['content']
-        parsed_gold_answer = item['choices'][0]['review']['gold']
-        parsed_pred_answer = item['choices'][0]['review']['pred']
-        score = item['choices'][0]['review']['result']
-        raw_d = {
-            'Input': raw_input,
-            'Generated': raw_pred_answer,
-            'Gold': parsed_gold_answer if parsed_gold_answer != raw_input else '*Same as Input*',
-            'Pred': parsed_pred_answer if parsed_pred_answer != raw_pred_answer else '*Same as Generated*',
-            'Score': score,
-            'NScore': normalize_score(score)
-        }
-        ds.append(raw_d)
+        for choice in item['choices']:
+            raw_pred_answer = choice['message']['content']
+            parsed_gold_answer = choice['review']['gold']
+            parsed_pred_answer = choice['review']['pred']
+            score = choice['review']['result']
+            raw_d = {
+                'Input': raw_input,
+                'Generated': raw_pred_answer,
+                'Gold': parsed_gold_answer if parsed_gold_answer != raw_input else '*Same as Input*',
+                'Pred': parsed_pred_answer if parsed_pred_answer != raw_pred_answer else '*Same as Generated*',
+                'Score': score,
+                'NScore': normalize_score(score)
+            }
+            ds.append(raw_d)
 
     df_subset = pd.DataFrame(ds)
     return df_subset
@@ -284,6 +297,8 @@ def get_table_data(data_review_df: pd.DataFrame, page: int = 1, rows_per_page: i
     end = start + rows_per_page
     df_subset = data_review_df.iloc[start:end].copy()
     df_subset['Input'] = df_subset['Input'].map(process_model_prediction).astype(str)
+    df_subset['Generated'] = df_subset['Generated'].map(process_model_prediction).astype(str)
+    df_subset['Pred'] = df_subset['Pred'].map(process_model_prediction).astype(str)
     df_subset['Score'] = df_subset['Score'].map(process_model_prediction).astype(str)
     styler = style_df(df_subset, columns=['NScore'])
     return df_subset, styler

evalscope/run.py

@@ -46,11 +46,13 @@ def run_single_task(task_cfg: TaskConfig, run_time: str) -> dict:
 
 def setup_work_directory(task_cfg: TaskConfig, run_time: str):
     """Set the working directory for the task."""
+    # use cache
     if task_cfg.use_cache:
         task_cfg.work_dir = task_cfg.use_cache
         logger.info(f'Set resume from {task_cfg.work_dir}')
     # elif are_paths_same(task_cfg.work_dir, DEFAULT_WORK_DIR):
-    task_cfg.work_dir = os.path.join(task_cfg.work_dir, run_time)
+    else:
+        task_cfg.work_dir = os.path.join(task_cfg.work_dir, run_time)
 
     outputs = OutputsStructure(outputs_dir=task_cfg.work_dir)
 
@@ -112,7 +114,7 @@ def evaluate_model(task_cfg: TaskConfig, outputs: OutputsStructure) -> dict:
     logger.info(task_cfg)
 
     for evaluator in evaluators:
-        res_dict = evaluator.eval(infer_cfg=task_cfg.generation_config, debug=task_cfg.debug, limit=task_cfg.limit)
+        res_dict = evaluator.eval()
         eval_results[dataset_name] = res_dict
 
     return eval_results
@@ -124,21 +126,22 @@ def create_evaluator(task_cfg: TaskConfig, dataset_name: str, outputs: OutputsSt
     from evalscope.evaluator import Evaluator
     from evalscope.models import initialize_model_adapter
 
+    benchmark: BenchmarkMeta = Benchmark.get(dataset_name)
+    # Initialize data adapter
+    data_adapter = benchmark.get_data_adapter(config=task_cfg.dataset_args.get(dataset_name, {}))
+
     if dataset_name == DataCollection.NAME:
         # EvaluatorCollection is a collection of evaluators
         from evalscope.collections import EvaluatorCollection
-        return EvaluatorCollection(task_cfg, outputs)
+        return EvaluatorCollection(task_cfg, data_adapter, outputs)
 
-    benchmark: BenchmarkMeta = Benchmark.get(dataset_name)
-
-    data_adapter = benchmark.get_data_adapter(config=task_cfg.dataset_args.get(dataset_name, {}))
+    # Initialize model adapter
     model_adapter = initialize_model_adapter(task_cfg, benchmark.model_adapter, base_model)
 
     # update task_cfg.dataset_args
     task_cfg.dataset_args[dataset_name] = benchmark.to_string_dict()
 
     return Evaluator(
-        dataset_name_or_path=benchmark.dataset_id,
         data_adapter=data_adapter,
         model_adapter=model_adapter,
         outputs=outputs,

evalscope/utils/chat_service.py

@@ -174,7 +174,7 @@ class ChatService:
         )
 
     def _prepare_text_inputs(self, request: TextCompletionRequest):
-        inputs = self.tokenizer(request.prompt, return_tensors='pt', padding=True).to(self.device)
+        inputs = self.tokenizer(request.prompt, return_tensors='pt', padding=False).to(self.device)
         prompt_tokens = len(inputs['input_ids'][0])
         return inputs, prompt_tokens
 
@@ -204,7 +204,7 @@ class ChatService:
     def _prepare_chat_inputs(self, request: ChatCompletionRequest):
         formatted_prompt = self.tokenizer.apply_chat_template(
             request.messages, tokenize=False, add_generation_prompt=True)
-        inputs = self.tokenizer(formatted_prompt, return_tensors='pt', padding=True).to(self.device)
+        inputs = self.tokenizer(formatted_prompt, return_tensors='pt', padding=False).to(self.device)
         prompt_tokens = len(inputs['input_ids'][0])
         return formatted_prompt, inputs, prompt_tokens
 

evalscope/utils/io_utils.py

@@ -135,7 +135,7 @@ def dict_to_yaml(d: dict, yaml_file: str):
     Dump dict to yaml file.
     """
     with open(yaml_file, 'w') as f:
-        yaml.dump(d, f, default_flow_style=False)
+        yaml.dump(d, f, default_flow_style=False, allow_unicode=True)
 
 
 def json_to_dict(json_file) -> dict:

evalscope/version.py

@@ -1,4 +1,4 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 
-__version__ = '0.10.1'
-__release_datetime__ = '2025-01-23 13:00:00'
+__version__ = '0.11.0'
+__release_datetime__ = '2025-02-13 12:00:00'