crfm-helm 0.5.0__py3-none-any.whl → 0.5.2__py3-none-any.whl

helm/benchmark/metrics/evaluate_reference_metrics.py

@@ -10,6 +10,7 @@ from helm.benchmark.metrics.metric_service import MetricService
 from helm.benchmark.metrics.statistic import Stat
 from helm.benchmark.scenarios.code_scenario import CodeReference
 from helm.benchmark.scenarios.scenario import Reference
+from helm.common.optional_dependencies import handle_module_not_found_error
 from helm.common.request import GeneratedOutput
 from helm.benchmark.scenarios.math_scenario import is_equiv, is_equiv_chain_of_thought
 from nltk.metrics.scores import f_measure
@@ -21,6 +22,7 @@ import string
 from . import code_metrics_helper
 import nltk
 
+
 try:
     nltk.data.find("tokenizers/punkt")
 except LookupError:
@@ -188,6 +190,19 @@ def bleu_4(gold: str, pred: str) -> float:
     return sentence_bleu([word_tokenize(gold)], word_tokenize(pred), weights=(0, 0, 0, 1))
 
 
+def cider(gold: str, pred: str) -> float:
+    try:
+        from pycocoevalcap.cider.cider import Cider
+    except ModuleNotFoundError as e:
+        handle_module_not_found_error(e, ["vlm"])
+
+    cider_evaluator = Cider()
+    candidate = {"caption": [pred]}
+    reference = {"caption": [gold]}
+    average_score, _ = cider_evaluator.compute_score(reference, candidate)
+    return average_score
+
+
 def extract_set_from_text(
     set_str: str,
     set_start_str: str = " is ",
@@ -325,6 +340,7 @@ def compute_reference_metrics(
         "math_equiv_chain_of_thought": is_equiv_chain_of_thought,
         "code_eval_acc": code_eval,
         "pass": code_eval,
+        "cider": cider,
         "f1_score": f1_score,
         "rouge_1": get_rouge_function("rouge1"),
         "rouge_2": get_rouge_function("rouge2"),

helm/benchmark/metrics/fin_qa_metrics.py

@@ -0,0 +1,60 @@
+import math
+import json
+from typing import List, Union
+
+from helm.benchmark.adaptation.adapter_spec import AdapterSpec
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.benchmark.metrics.metric import Metric
+from helm.benchmark.metrics.metric_name import MetricName
+from helm.benchmark.metrics.metric_service import MetricService
+from helm.benchmark.metrics.statistic import Stat
+from helm.benchmark.metrics.fin_qa_metrics_helper import (  # type: ignore
+    equal_program,
+    eval_program,
+    program_tokenization,
+)
+
+
+def _get_program_accuracy(reference_program: List[str], generated_program: List[str]) -> float:
+    return 1.0 if equal_program(reference_program, generated_program) else 0.0
+
+
+def _get_execution_accuracy(reference_execution: str, generated_program: List[str], table: List[List[str]]) -> float:
+    invalid_flag: int
+    generated_result: Union[str, float]
+    invalid_flag, generated_result = eval_program(generated_program, table)
+    if invalid_flag:
+        return 0.0
+    if reference_execution == "yes" or reference_execution == "no":
+        return 1.0 if reference_execution == generated_result else 0
+    else:
+        if not isinstance(generated_result, float):
+            return 0.0
+        return 1.0 if math.isclose(float(reference_execution), generated_result) else 0
+
+
+class FinQAMetric(Metric):
+    def evaluate_generation(
+        self,
+        adapter_spec: AdapterSpec,
+        request_state: RequestState,
+        metric_service: MetricService,
+        eval_cache_path: str,
+    ) -> List[Stat]:
+        assert len(request_state.instance.references) == 3
+        reference_text = request_state.instance.references[0].output.text
+        reference_program = program_tokenization(reference_text)
+        reference_execution = request_state.instance.references[1].output.text
+        table: List[List[str]] = json.loads(request_state.instance.references[2].output.text)
+
+        assert request_state.result
+        assert len(request_state.result.completions) == 1
+        generated_text = request_state.result.completions[0].text.strip()
+        generated_program = program_tokenization(generated_text)
+
+        return [
+            Stat(MetricName("program_accuracy")).add(_get_program_accuracy(reference_program, generated_program)),
+            Stat(MetricName("execution_accuracy")).add(
+                _get_execution_accuracy(reference_execution, generated_program, table)
+            ),
+        ]

helm/benchmark/metrics/fin_qa_metrics_helper.py

@@ -0,0 +1,398 @@
+# type: ignore
+# flake8: noqa
+# fmt: off
+"""Evaluation metrics for FinQA.
+
+This evaluation code is reproduced from the following URL with the following license.
+
+URL: https://github.com/czyssrs/FinQA/blob/0f16e2867befa6840783e58be38c9efb9229d742/code/evaluate/evaluate.py
+
+License: MIT License
+
+Copyright (c) 2021 Zhiyu Chen
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE."""
+
+from sympy import simplify
+
+
+all_ops = ["add", "subtract", "multiply", "divide", "exp", "greater", "table_max", \
+"table_min", "table_sum", "table_average"]
+
+
+def str_to_num(text):
+    
+    text = text.replace(",", "")
+    try:
+        num = float(text)
+    except ValueError:
+        if "%" in text:
+            text = text.replace("%", "")
+            try:
+                num = float(text)
+                num = num / 100.0
+            except ValueError:
+                num = "n/a"
+        elif "const" in text:
+            text = text.replace("const_", "")
+            if text == "m1":
+                text = "-1"
+            num = float(text)
+        else:
+            num = "n/a"
+    return num
+
+def process_row(row_in):
+    
+    row_out = []
+    invalid_flag = 0
+    
+    for num in row_in:
+        num = num.replace("$", "").strip()
+        num = num.split("(")[0].strip()
+        
+        num = str_to_num(num)
+        
+        if num == "n/a":
+            invalid_flag = 1
+            break
+        
+        row_out.append(num)
+        
+    if invalid_flag:
+        return "n/a"
+    
+    return row_out
+
+
+def eval_program(program, table):
+    '''
+    calculate the numerical results of the program
+    '''
+
+    invalid_flag = 0
+    this_res = "n/a"
+    
+    try:
+        program = program[:-1] # remove EOF
+        # check structure
+        for ind, token in enumerate(program):
+            if ind % 4 == 0:
+                if token.strip("(") not in all_ops:
+                    return 1, "n/a"
+            if (ind + 1) % 4 == 0:
+                if token != ")":
+                    return 1, "n/a"
+
+
+        program = "|".join(program)
+        steps = program.split(")")[:-1]
+        
+        
+        res_dict = {}
+        
+        # print(program)
+        
+        for ind, step in enumerate(steps):
+            step = step.strip()
+            
+            if len(step.split("(")) > 2:
+                invalid_flag = 1
+                break
+            op = step.split("(")[0].strip("|").strip()
+            args = step.split("(")[1].strip("|").strip()
+            
+            # print(args)
+            # print(op)
+            
+            arg1 = args.split("|")[0].strip()
+            arg2 = args.split("|")[1].strip()
+            
+            if op == "add" or op == "subtract" or op == "multiply" or op == "divide" or op == "exp" or op == "greater":
+                
+                if "#" in arg1:
+                    arg1 = res_dict[int(arg1.replace("#", ""))]
+                else:
+                    # print(arg1)
+                    arg1 = str_to_num(arg1)
+                    if arg1 == "n/a":
+                        invalid_flag = 1
+                        break
+                
+                if "#" in arg2:
+                    arg2 = res_dict[int(arg2.replace("#", ""))]
+                else:
+                    arg2 = str_to_num(arg2)
+                    if arg2 == "n/a":
+                        invalid_flag = 1
+                        break
+                
+                if op == "add":
+                    this_res = arg1 + arg2
+                elif op == "subtract":
+                    this_res = arg1 - arg2
+                elif op == "multiply":
+                    this_res = arg1 * arg2
+                elif op == "divide":
+                    this_res = arg1 / arg2
+                elif op == "exp":
+                    this_res = arg1 ** arg2
+                elif op == "greater":
+                    this_res = "yes" if arg1 > arg2 else "no"
+
+                    
+                # print("ind: ", ind)
+                # print(this_res)
+                res_dict[ind] = this_res
+
+
+            elif "table" in op:
+                table_dict = {}
+                for row in table:
+                    table_dict[row[0]] = row[1:]
+                    
+                if "#" in arg1:
+                    arg1 = res_dict[int(arg1.replace("#", ""))]
+                else:
+                    if arg1 not in table_dict:
+                        invalid_flag = 1
+                        break
+                    
+                    cal_row = table_dict[arg1]
+                    num_row = process_row(cal_row)
+                    
+                if num_row == "n/a":
+                    invalid_flag = 1
+                    break
+                if op == "table_max":
+                    this_res = max(num_row)
+                elif op == "table_min":
+                    this_res = min(num_row)
+                elif op == "table_sum":
+                    this_res = sum(num_row)
+                elif op == "table_average":
+                    this_res = sum(num_row) / len(num_row)
+                    
+                # this_res = round(this_res, 5)
+
+                res_dict[ind] = this_res
+
+            # print(this_res)
+
+        if this_res != "yes" and this_res != "no" and this_res != "n/a":
+            # print(this_res)
+            this_res = round(this_res, 5)
+
+    except:
+        invalid_flag = 1
+        
+
+    return invalid_flag, this_res
+
+
+def equal_program(program1, program2):
+    '''
+    symbolic program if equal
+    program1: gold
+    program2: pred
+    '''
+    
+    sym_map = {}
+    
+    program1 = program1[:-1] # remove EOF
+    program1 = "|".join(program1)
+    steps = program1.split(")")[:-1]
+    
+    invalid_flag = 0
+    sym_ind = 0
+    step_dict_1 = {}
+    
+    # symbolic map
+    for ind, step in enumerate(steps):
+        
+        step = step.strip()
+
+        assert len(step.split("(")) <= 2
+
+        op = step.split("(")[0].strip("|").strip()
+        args = step.split("(")[1].strip("|").strip()
+        
+        arg1 = args.split("|")[0].strip()
+        arg2 = args.split("|")[1].strip()
+        
+        step_dict_1[ind] = step
+
+        if "table" in op:
+            if step not in sym_map:
+                sym_map[step] = "a" + str(sym_ind)
+                sym_ind += 1
+                
+        else:
+            if "#" not in arg1:
+                if arg1 not in sym_map:
+                    sym_map[arg1] = "a" + str(sym_ind)
+                    sym_ind += 1
+                    
+            if "#" not in arg2:
+                if arg2 not in sym_map:
+                    sym_map[arg2] = "a" + str(sym_ind)
+                    sym_ind += 1
+
+
+    # check program 2
+    step_dict_2 = {}
+    try:
+        program2 = program2[:-1] # remove EOF
+        # check structure
+        for ind, token in enumerate(program2):
+            if ind % 4 == 0:
+                if token.strip("(") not in all_ops:
+                    print("structure error")
+                    return False
+            if (ind + 1) % 4 == 0:
+                if token != ")":
+                    print("structure error")
+                    return False
+
+        program2 = "|".join(program2)
+        steps = program2.split(")")[:-1]
+        
+        for ind, step in enumerate(steps):
+            step = step.strip()
+            
+            if len(step.split("(")) > 2:
+                return False
+            op = step.split("(")[0].strip("|").strip()
+            args = step.split("(")[1].strip("|").strip()
+            
+            # print(args)
+            # print(op)
+            
+            arg1 = args.split("|")[0].strip()
+            arg2 = args.split("|")[1].strip()
+            
+            step_dict_2[ind] = step
+
+            if "table" in op:
+                if step not in sym_map:
+                    return False
+                    
+            else:
+                if "#" not in arg1:
+                    if arg1 not in sym_map:
+                        return False
+                else:
+                    if int(arg1.strip("#")) >= ind:
+                        return False
+                        
+                if "#" not in arg2:
+                    if arg2 not in sym_map:
+                        return False
+                else:
+                    if int(arg2.strip("#")) >= ind:
+                        return False
+    except:
+        return False
+
+    def symbol_recur(step, step_dict):
+        
+        step = step.strip()
+        op = step.split("(")[0].strip("|").strip()
+        args = step.split("(")[1].strip("|").strip()
+        
+        arg1 = args.split("|")[0].strip()
+        arg2 = args.split("|")[1].strip()
+        
+        # print(op)
+        # print(arg1)
+        # print(arg2)
+        
+        if "table" in op:
+            # as var
+            return sym_map[step]
+        
+        if "#" in arg1:
+            arg1_ind = int(arg1.replace("#", ""))
+            arg1_part = symbol_recur(step_dict[arg1_ind], step_dict)
+        else:
+            arg1_part = sym_map[arg1]
+            
+            
+        if "#" in arg2:
+            arg2_ind = int(arg2.replace("#", ""))
+            arg2_part = symbol_recur(step_dict[arg2_ind], step_dict)
+        else:
+            arg2_part = sym_map[arg2]
+            
+        if op == "add":
+            return "( " + arg1_part + " + " + arg2_part + " )"
+        elif op == "subtract":
+            return "( " + arg1_part + " - " + arg2_part + " )"
+        elif op == "multiply":
+            return "( " + arg1_part + " * " + arg2_part + " )"
+        elif op == "divide":
+            return "( " + arg1_part + " / " + arg2_part + " )"
+        elif op == "exp":
+            return "( " + arg1_part + " ** " + arg2_part + " )"
+        elif op == "greater":
+            return "( " + arg1_part + " > " + arg2_part + " )"
+
+
+    # # derive symbolic program 1
+    # print(program1)
+    steps = program1.split(")")[:-1]
+    # print(steps)
+    # print(steps)
+    # print(sym_map)
+    sym_prog1 = symbol_recur(steps[-1], step_dict_1)
+    sym_prog1 = simplify(sym_prog1, evaluate=False)
+    # print("########")
+    # print(sym_prog1)
+    
+    try:
+        # derive symbolic program 2
+        steps = program2.split(")")[:-1]
+        sym_prog2 = symbol_recur(steps[-1], step_dict_2)
+        sym_prog2 = simplify(sym_prog2, evaluate=False)
+        # print(sym_prog2)
+    except:
+        return False
+
+    return sym_prog1 == sym_prog2
+
+
+def program_tokenization(original_program):
+    original_program = original_program.split(', ')
+    program = []
+    for tok in original_program:
+        cur_tok = ''
+        for c in tok:
+            if c == ')':
+                if cur_tok != '':
+                    program.append(cur_tok)
+                    cur_tok = ''
+            cur_tok += c
+            if c in ['(', ')']:
+                program.append(cur_tok)
+                cur_tok = ''
+        if cur_tok != '':
+            program.append(cur_tok)
+    program.append('EOF')
+    return program
+# fmt: on

helm/benchmark/metrics/gpt4v_originality_critique_metrics.py

@@ -0,0 +1,126 @@
+from typing import Dict, List
+
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.benchmark.adaptation.scenario_state import ScenarioState
+from helm.benchmark.adaptation.adapter_spec import AdapterSpec
+from helm.benchmark.metrics.metric import MetricInterface, MetricResult, PerInstanceStats, add_context
+from helm.benchmark.metrics.metric_name import MetricContext, MetricName
+from helm.benchmark.metrics.metric_service import MetricService
+from helm.benchmark.metrics.statistic import Stat, merge_stat
+from helm.common.critique_request import CritiqueTaskTemplate, CritiqueQuestionTemplate, CritiqueRequest, QuestionType
+from helm.common.hierarchical_logger import hlog
+from helm.common.request import RequestResult, Request, GeneratedOutput
+from helm.common.media_object import MultimediaObject, IMAGE_TYPE, MediaObject
+
+
+class GPT4VCritiqueMetric(MetricInterface):
+    """
+    Critique evaluation for evaluating how original the generated text are given the image by GPT4V.
+    """
+
+    # We can add more evaluation aspects here
+    ORIGINALITY_NAME: str = "originality_gpt4v"
+    ORIGINALITY_ANSWER_TO_SCORE: Dict[str, int] = {
+        "I’ve seen something like this before to the point it’s become tiresome.": 1,
+        "The text is not really original, but it has some originality to it.": 2,
+        "Neutral.": 3,
+        "I find the text to be fresh and original.": 4,
+        "I find the text to be extremely creative and out of this world.": 5,
+    }
+
+    def __init__(self, num_respondents: int):
+        self._num_respondents = num_respondents
+
+    def __repr__(self) -> str:
+        return "GPT4CritiqueMetric()"
+
+    def evaluate(
+        self,
+        scenario_state: ScenarioState,
+        metric_service: MetricService,
+        eval_cache_path: str,
+        parallelism: int,
+    ) -> MetricResult:
+        request_states: List[RequestState] = scenario_state.request_states
+
+        all_stats: Dict[MetricName, Stat] = {}
+        per_instance_stats: List[PerInstanceStats] = []
+        for request_state in request_states:
+            context = MetricContext.from_instance(request_state.instance)
+            stats_without_context = self.evaluate_generation(
+                scenario_state.adapter_spec,
+                request_state,
+                metric_service,
+                eval_cache_path,
+            )
+            stats = [add_context(stat_without_context, context) for stat_without_context in stats_without_context]
+            for stat in stats:
+                merge_stat(all_stats, stat)
+            assert request_state.instance.id is not None
+            per_instance_stats.append(
+                PerInstanceStats(
+                    instance_id=request_state.instance.id,
+                    perturbation=request_state.instance.perturbation,
+                    train_trial_index=request_state.train_trial_index,
+                    stats=stats,
+                )
+            )
+        return MetricResult(aggregated_stats=list(all_stats.values()), per_instance_stats=per_instance_stats)
+
+    def evaluate_generation(
+        self,
+        adapter_spec: AdapterSpec,
+        request_state: RequestState,
+        metric_service: MetricService,
+        eval_cache_path: str,
+    ) -> List[Stat]:
+        input_request: Request = request_state.request
+        # Predicted outputs and their originality scores
+        assert request_state.result is not None
+        request_result: RequestResult = request_state.result
+        # Get input image and generated response for the originality evaluation
+        assert input_request.multimodal_prompt is not None
+        completions: List[GeneratedOutput] = request_result.completions
+        input_text: str = completions[0].text
+        input_media: MultimediaObject = input_request.multimodal_prompt
+        image_objects: List[MediaObject] = [
+            item for item in input_media.media_objects if item.is_type(IMAGE_TYPE) and item.location
+        ]
+
+        template = CritiqueTaskTemplate(
+            name="vhelm_gpt4v_originality",
+            # TODO: Add proper instructions
+            instructions="Answer the multiple choice question by just giving the letter of the correct "
+            "answer.\n\n{{prompt}}",
+            num_respondents=self._num_respondents,
+            questions=[
+                CritiqueQuestionTemplate(
+                    name=self.ORIGINALITY_NAME,
+                    question_type=QuestionType.MULTIPLE_CHOICE,
+                    text="How original is the text, given it was created with the image?",
+                    options=list(self.ORIGINALITY_ANSWER_TO_SCORE.keys()),
+                    media_object=image_objects[0],  # we only take the first image as input
+                )
+            ],
+        )
+        request = CritiqueRequest(template=template, fields={"prompt": input_text})
+
+        # send to critique request
+        result = metric_service.make_critique_request(request)
+        if not result or not result.responses:
+            # Skip computing metrics if there aren't any responses yet
+            hlog("Waiting for responses to be generated.")
+            return []
+
+        stats: Dict[str, Stat] = {}
+        for question in template.questions:
+            stats[question.name] = Stat(MetricName(question.name))
+
+        for response in result.responses:
+            for answer_name, answer in response.answers.items():
+                assert isinstance(answer, str)
+                answer_value: float
+                answer_value = self.ORIGINALITY_ANSWER_TO_SCORE[answer]
+                stats[answer_name].add(answer_value)
+
+        return list(stats.values())

helm/benchmark/metrics/instruction_following_critique_metrics.py

@@ -1,3 +1,4 @@
+# noqa: E501
 from typing import Dict, List
 
 from helm.benchmark.adaptation.request_state import RequestState

helm/benchmark/metrics/live_qa_metrics.py

@@ -0,0 +1,23 @@
+from typing import List
+
+from helm.benchmark.adaptation.adapter_spec import AdapterSpec
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.benchmark.metrics.metric import Metric
+from helm.benchmark.metrics.metric_name import MetricName
+from helm.benchmark.metrics.metric_service import MetricService
+from helm.benchmark.metrics.statistic import Stat
+
+
+class LiveQAScoreMetric(Metric):
+    """Score metrics for LiveQA."""
+
+    def evaluate_generation(
+        self,
+        adapter_spec: AdapterSpec,
+        request_state: RequestState,
+        metric_service: MetricService,
+        eval_cache_path: str,
+    ) -> List[Stat]:
+        assert request_state.annotations
+        score = request_state.annotations["live_qa"]["score"]
+        return [Stat(MetricName("live_qa_score")).add(score)]

helm/benchmark/metrics/medication_qa_metrics.py

@@ -0,0 +1,23 @@
+from typing import List
+
+from helm.benchmark.adaptation.adapter_spec import AdapterSpec
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.benchmark.metrics.metric import Metric
+from helm.benchmark.metrics.metric_name import MetricName
+from helm.benchmark.metrics.metric_service import MetricService
+from helm.benchmark.metrics.statistic import Stat
+
+
+class MedicationQAScoreMetric(Metric):
+    """Score metrics for MedicationQA."""
+
+    def evaluate_generation(
+        self,
+        adapter_spec: AdapterSpec,
+        request_state: RequestState,
+        metric_service: MetricService,
+        eval_cache_path: str,
+    ) -> List[Stat]:
+        assert request_state.annotations
+        score = request_state.annotations["medication_qa"]["score"]
+        return [Stat(MetricName("medication_qa_score")).add(score)]