eval-framework 0.2.7__py3-none-any.whl

eval_framework/main.py

@@ -0,0 +1,166 @@
+import gc
+import json
+import logging
+import os
+import shutil
+from collections.abc import Callable
+from pathlib import Path
+from typing import Any, Literal
+
+import wandb
+
+from eval_framework.evaluation_generator import EvaluationGenerator, Result
+from eval_framework.llm.base import BaseLLM
+from eval_framework.response_generator import ResponseGenerator
+from eval_framework.result_processors.hf_uploader import HFUploader
+from eval_framework.result_processors.result_processor import ResultsFileProcessor, generate_output_dir
+from eval_framework.result_processors.wandb_uploader import WandbUploader
+from eval_framework.tasks.eval_config import EvalConfig
+from eval_framework.utils.constants import RED, RESET
+from eval_framework.utils.logging import setup_logging
+
+logger = logging.getLogger(__name__)
+
+
+def main(
+    llm: BaseLLM,
+    config: EvalConfig,
+    should_preempt_callable: Callable[[], bool] | None = None,
+    trial_id: int | None = None,
+    *args: Any,
+    resource_cleanup: bool = False,
+    verbosity: int = 1,
+) -> list[Result]:
+    """Runs the entire evaluation process: responses generation and evaluation."""
+    # Set up centralized logging early
+    output_dir = generate_output_dir(llm.name, config)
+    setup_logging(output_dir=output_dir, log_level=verbosity, log_filename="evaluation.log")
+    logger.info(f"Output directory for evaluation: {output_dir}")
+
+    logger.info(f"{RED}[ Running full evaluation process ------- ]{RESET}")
+    logger.info(f"Evaluating {llm.name} on {config.task_name}")
+    logger.info(f"Configuration: num_fewshot={config.num_fewshot}, num_samples={config.num_samples}")
+    logger.info(f"Output directory: {output_dir}")
+
+    if not should_preempt_callable:
+        should_preempt_callable = lambda: False  # noqa: E731
+    preemption_data = None
+
+    if trial_id:
+        preemption_data = _read_preemption_data(config, trial_id)
+
+    if preemption_data is None:
+        output_dir = generate_output_dir(llm.name, config)
+        wandb_run_id = config.wandb_run_id  # defaults to none, if no run_id is provided then it starts a new one
+    else:
+        logger.info("Found preempted run restarting ...")
+        output_dir = preemption_data["output_dir"]
+        wandb_run_id = preemption_data.get("wandb_run_id", None)
+
+    logger.info(f"Output directory: {output_dir}")
+    assert output_dir is not None
+
+    file_processor = ResultsFileProcessor(output_dir)
+    response_generator = ResponseGenerator(llm, config, file_processor)
+
+    with wandb.init(
+        entity=config.wandb_entity,
+        project=config.wandb_project,
+        group=llm.name[:127],
+        job_type=config.task_name[:63],
+        id=wandb_run_id,  # (potentially resuming run after preemption)
+        config=response_generator._get_metadata(),
+        resume="allow",
+        mode=_wandb_mode(config.wandb_project),
+        settings=wandb.Settings(disable_code=True),  # ("wandb-history" artifacts not needed)
+    ) as run:
+        artifact = getattr(llm, "artifact", None)
+        if artifact is not None:
+            wandb.use_artifact(artifact)
+        for additional_artifact in os.getenv("WANDB_ADDITIONAL_ARTIFACT_REFERENCES", "").split(","):
+            if additional_artifact.strip():
+                wandb.use_artifact(additional_artifact.strip())
+
+        _, preempted = response_generator.generate(should_preempt_callable)
+
+        if preempted:
+            logger.info("Response generation was preempted")
+            assert trial_id is not None
+            run.mark_preempting()
+            _save_preemption_data(config, trial_id, output_dir, wandb_run_id=run.id)
+            wandb.finish(exit_code=1)
+            return []
+        # update config from response generator with get metadata
+        if trial_id is not None:
+            _delete_preemption_file(config, trial_id)
+
+        if resource_cleanup:
+            del response_generator
+            gc.collect()
+
+        evaluator = EvaluationGenerator(config, file_processor)
+        results = evaluator.run_eval()
+
+        upload_success = False
+        for uploader in [HFUploader(config), WandbUploader(config)]:
+            upload_success |= uploader.upload(llm.name, config, output_dir)
+
+        if config.delete_output_dir_after_upload and upload_success:
+            logger.warning(f"Deleting output directory '{output_dir}' after successful upload(s)!")
+            shutil.rmtree(output_dir, ignore_errors=True)
+            if output_dir.exists() and any(output_dir.iterdir()):
+                logger.warning("Could not delete output directory, some files remain.")
+
+    return results
+
+
+def _read_preemption_data(config: EvalConfig, trial_id: int) -> dict[str, Any] | None:
+    preemption_file = config.output_dir / f"preemption_trial_{trial_id}.json"
+    if not preemption_file.is_file():
+        return None
+    with open(preemption_file, "rb") as f:
+        preemption_data = json.load(f)
+        preemption_data["output_dir"] = Path(preemption_data["output_dir"])
+        preemption_data["wandb_run_id"] = preemption_data.get("wandb_run_id", "")
+        logger.info(f"Loaded preemption data from {preemption_file}")
+        return preemption_data
+
+
+def _save_preemption_data(config: EvalConfig, trial_id: int, output_dir: Path, wandb_run_id: str = "") -> None:
+    preemption_file = config.output_dir / f"preemption_trial_{trial_id}.json"
+    with open(preemption_file, "w") as f:
+        json.dump({"output_dir": str(output_dir), "wandb_run_id": wandb_run_id}, f)
+
+
+def _delete_preemption_file(config: EvalConfig, trial_id: int) -> None:
+    preemption_file = config.output_dir / f"preemption_trial_{trial_id}.json"
+    if preemption_file.is_file():
+        preemption_file.unlink()
+        logger.info(f"Deleted preemption file: {preemption_file}")
+    else:
+        logger.info(f"No preemption file found to delete: {preemption_file}")
+    logger.info(f"Saved preemption data to {preemption_file}")
+
+
+def _wandb_mode(project: str | None) -> Literal["online", "disabled"] | None:
+    """
+    Checks to see if a WandB API key is found. If not, wandb starts in offline mode.
+    """
+    if project is None:
+        logger.warning("No WandB project specified, disabling logging.")
+        return "disabled"
+    else:
+        try:
+            api_key = wandb.api.api_key
+            if api_key is None:
+                logger.warning(
+                    """No wandb API key found. Disabling Wandb logging.
+                    If you have a WandB account set the environment variable 'WANDB_API_KEY'"""
+                )
+                return "disabled"
+            else:
+                logger.info("Wandb login detected. Using online mode.")
+        except Exception as e:
+            logger.warning(f"Wandb login check failed: {e}. Disabling Wandb logging.")
+            return "disabled"
+    return "online"

eval_framework/metrics/base.py

@@ -0,0 +1,40 @@
+from abc import ABC, abstractmethod
+from typing import Any
+
+from pydantic import BaseModel, ConfigDict
+
+from eval_framework.shared.types import Error
+
+
+class MetricResult(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+    metric_name: str
+    value: float | None
+    higher_is_better: bool
+    llm_judge_prompt: str | None = None
+    llm_judge_response: str | None = None
+    code_execution_trace: str | None = None
+    error: Error | None = None
+
+
+class classproperty:
+    def __init__(self, method: Any) -> None:
+        self.method = method
+
+    def __get__(self, instance: Any, cls: Any) -> Any:
+        return self.method(cls)
+
+
+class BaseMetric[Response](ABC):
+    NAME: str
+    KEYS: list[str] | None = None
+
+    @classproperty
+    def NAMES(cls) -> list[str]:
+        if cls.KEYS is None:
+            return [cls.NAME]
+        return [f"{cls.NAME}/{k}" for k in cls.KEYS]
+
+    @abstractmethod
+    def calculate(self, response: Response) -> list[MetricResult]:
+        raise NotImplementedError

eval_framework/metrics/completion/__init__.py

@@ -0,0 +1 @@
+from .accuracy_completion import AccuracyCompletion

eval_framework/metrics/completion/accuracy_completion.py

@@ -0,0 +1,16 @@
+from eval_framework.metrics.base import BaseMetric, MetricResult
+from eval_framework.shared.types import Completion
+
+
+class AccuracyCompletion(BaseMetric[Completion]):
+    NAME = "Accuracy Completion"
+
+    def calculate(self, response: Completion) -> list[MetricResult]:
+        if response.error is not None:
+            return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=response.error)]
+
+        ground_truths = response.ground_truth_list
+        is_correct = any(response.completion == gt for gt in ground_truths)
+        return [
+            MetricResult(metric_name=self.NAME, value=float(is_correct), higher_is_better=True, error=response.error)
+        ]

eval_framework/metrics/completion/aidanbench.py

@@ -0,0 +1,28 @@
+import logging
+
+from eval_framework.metrics.base import BaseMetric, MetricResult
+from eval_framework.shared.types import Completion
+
+logger = logging.getLogger(__name__)
+
+
+class AidanBenchMetric(BaseMetric[Completion]):
+    NAME = "AidanBench"
+
+    def calculate(self, response: Completion) -> list[MetricResult]:
+        # subtract 2 to not count 1) initial instruction and 2) the latest model response, which caused the stop
+        # i.e. was not (unique && coherent)
+        num_unique_responses = len(response.messages) - 2 if response.messages is not None else 0
+        if num_unique_responses < 0:
+            logger.warning(
+                "Number of unique responses calculated as negative, setting to 0."
+                "Likely something went wrong during answer generation."
+            )
+            num_unique_responses = 0
+        return [
+            MetricResult(
+                metric_name=f"{self.NAME}/num_responses",
+                value=num_unique_responses,
+                higher_is_better=True,
+            )
+        ]

eval_framework/metrics/completion/bleu.py

@@ -0,0 +1,76 @@
+import sacrebleu
+
+from eval_framework.exceptions import LogicError
+from eval_framework.metrics.base import BaseMetric, MetricResult
+from eval_framework.shared.types import Completion
+
+
+class BLEU(BaseMetric[Completion]):
+    """The Bilingual Evaluation Understudy Score, or BLEU for short, is a metric
+    for evaluating a generated sentence to a reference sentence. It counts matching
+    n-grams in the candidate translation to n-grams in the reference text, where
+    1-gram or unigram would be each token and a bigram comparison would be each
+    word pair. The comparison is made regardless of word order
+    Source: https://machinelearningmastery.com/calculate-bleu-score-for-text-python/
+    Paper: https://www.aclweb.org/anthology/P02-1040/
+    """
+
+    NAME = "BLEU"
+
+    def calculate(self, response: Completion) -> list[MetricResult]:
+        if response.error is not None:
+            return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=response.error)]
+
+        scores = []
+        for ground_truth in response.ground_truth_list:
+            if ground_truth == "" or ground_truth is None:
+                raise LogicError("When calculating BLEU we need a ground truth.")
+
+            sacre_formatted_completion = [response.completion]
+            sacre_formatted_ground_truth = [[ground_truth]]
+            scores.append(sacrebleu.corpus_bleu(sacre_formatted_completion, sacre_formatted_ground_truth).score)
+
+        return [
+            MetricResult(metric_name=self.NAME, value=float(max(scores)), higher_is_better=True, error=response.error)
+        ]
+
+
+class LINEWISE_BLEU(BaseMetric[Completion]):
+    """Maximum Line-level BLEU score."""
+
+    NAME = "Linewise BLEU"
+
+    def calculate(self, response: Completion) -> list[MetricResult]:
+        if response.error is not None:
+            return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=response.error)]
+
+        scores = []
+        for ground_truth in response.ground_truth_list:
+            for sentence in response.completion.split("\n"):
+                if sentence == "":
+                    continue
+
+                if ground_truth == "" or ground_truth is None:
+                    raise LogicError("When calculating BLEU we need a ground truth.")
+
+                sacre_formatted_completion = [sentence]
+                sacre_formatted_ground_truth = [[ground_truth]]
+                scores.append(sacrebleu.corpus_bleu(sacre_formatted_completion, sacre_formatted_ground_truth).score)
+
+        return [
+            MetricResult(
+                metric_name=self.NAME, value=float(max(scores, default=0)), higher_is_better=True, error=response.error
+            )
+        ]
+
+
+class ResponseToOriginalBLEU(BaseMetric[Completion]):
+    NAME = "Response to Original BLEU"
+
+    def calculate(self, response: Completion) -> list[MetricResult]:
+        if response.error is not None:
+            return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=response.error)]
+
+        score = sacrebleu.corpus_bleu([response.completion], [[response.last_user_instruction]]).score
+        # scaled to [0, 1] to make aggregation easier
+        return [MetricResult(metric_name=self.NAME, value=score / 100, higher_is_better=True, error=response.error)]

eval_framework/metrics/completion/chrf.py

@@ -0,0 +1,62 @@
+import sacrebleu
+
+from eval_framework.exceptions import LogicError
+from eval_framework.metrics.base import BaseMetric, MetricResult
+from eval_framework.shared.types import Completion
+
+
+class CHRF(BaseMetric[Completion]):
+    """chrF++ is a tool for automatic evaluation of machine translation output
+    based on character n-gram precision and recall enhanced with word n-grams.
+    Source: https://github.com/m-popovic/chrF
+    Paper: https://www.aclweb.org/anthology/W15-3049.pdf
+    """
+
+    NAME = "chrF"
+
+    def calculate(self, response: Completion) -> list[MetricResult]:
+        if response.error is not None:
+            return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=response.error)]
+
+        scores = []
+        for ground_truth in response.ground_truth_list:
+            if ground_truth == "" or ground_truth is None:
+                raise LogicError("When calculating chrF we need a ground truth.")
+
+            sacre_formatted_completion = [response.completion]
+            sacre_formatted_ground_truth = [[ground_truth]]
+            scores.append(sacrebleu.corpus_chrf(sacre_formatted_completion, sacre_formatted_ground_truth).score)
+
+        return [
+            MetricResult(metric_name=self.NAME, value=float(max(scores)), higher_is_better=True, error=response.error)
+        ]
+
+
+class LINEWISE_CHRF(BaseMetric[Completion]):
+    """Maximum Line-level chrF++ (Character n-gram F-score) score.
+    Paper: https://aclanthology.org/W15-3049/"""
+
+    NAME = "Linewise chrF"
+
+    def calculate(self, response: Completion) -> list[MetricResult]:
+        if response.error is not None:
+            return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=response.error)]
+
+        scores = []
+        for ground_truth in response.ground_truth_list:
+            for sentence in response.completion.split("\n"):
+                if sentence == "":
+                    continue
+
+                if ground_truth == "" or ground_truth is None:
+                    raise LogicError("When calculating chrF we need a ground truth.")
+
+                sacre_formatted_completion = [sentence]
+                sacre_formatted_ground_truth = [[ground_truth]]
+                scores.append(sacrebleu.corpus_chrf(sacre_formatted_completion, sacre_formatted_ground_truth).score)
+
+        return [
+            MetricResult(
+                metric_name=self.NAME, value=float(max(scores, default=0)), higher_is_better=True, error=response.error
+            )
+        ]

eval_framework/metrics/completion/code_assertion.py

@@ -0,0 +1,44 @@
+from eval_framework.metrics.base import BaseMetric, MetricResult
+from eval_framework.shared.types import Completion, Error
+from eval_framework.tasks.utils import run_python_code
+
+
+class CodeCompletionAssertion(BaseMetric[Completion]):
+    NAME = "Code Completion Accuracy"
+
+    def calculate(self, response: Completion) -> list[MetricResult]:
+        if response.error is not None:
+            return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=response.error)]
+
+        # this will always be a list, if return is "" this will be an empty list
+        code = response.completion
+        output = run_python_code(code, image="python:3.12-slim")
+
+        # Split and filter out empty strings
+        output_parts = [part for part in output.split() if part.strip()]
+
+        if not output_parts:
+            last_output = ""
+        else:
+            last_output = output_parts[-1]
+
+        success = last_output == "True"
+        error = (
+            None
+            if success
+            else Error(
+                error_class="CodeCompletionAssertionError",
+                message=f"Expected 'True' but got '{last_output}'",
+                traceback=output,
+            )
+        )
+
+        return [
+            MetricResult(
+                metric_name=self.NAME,
+                value=1.0 if success else 0.0,
+                higher_is_better=True,
+                error=error,
+                code_execution_trace=output,
+            )
+        ]

eval_framework/metrics/completion/code_execution_pass_at_one.py

@@ -0,0 +1,126 @@
+import traceback
+from collections.abc import Callable
+from typing import Self
+
+from pydantic import Field
+
+from eval_framework.metrics.base import BaseMetric, MetricResult
+from eval_framework.shared.types import BaseMetricContext, Completion, Error, extract_context_metric
+from eval_framework.tasks.utils import CallableSerializer, ExecutionResult, execute_python_code_with_tests
+
+
+class CodeExecutionBaseContext(BaseMetricContext):
+    run_env: str = Field(description="Name of docker image to run unit-tests inside")
+    code_prompt: str = Field(description="Prompt to LLM for code generation")
+    test_code: str = Field(description="Python code that contains logic for unit test execution")
+    benchmark_timeout: int = Field(default=60, description="Time in seconds for the full test execution run")
+    package_downloads: dict[str, str | None] = Field(
+        description="a dictionary listing the packages and their respective names in PyPiinto the LLM sandbox"
+    )
+
+
+class CodeExecutionPassAtOneContext(CodeExecutionBaseContext):
+    snippet_merge_fn: str = Field(
+        description="logic for merging LLM generated code with test execution code;"
+        "this code will be passed into the sandbox to run the testing process"
+        "This code is serialized"
+    )
+    output_parse_fn: str = Field(
+        description="logic for parsing the output of test code execution run within the LLM sandbox"
+        "This code is serialized"
+    )
+
+
+class RealtimeCodeExectionContext(CodeExecutionBaseContext):
+    snippet_merge_fn: Callable[[str, str], str] = Field(
+        description="logic for merging LLM generated code with test execution code;"
+        "this code will be passed into the sandbox to run the testing process"
+        "This code is deserialized"
+    )
+    output_parse_fn: Callable[[str], ExecutionResult] = Field(
+        description="logic for parsing the output of test code execution run within the LLM sandbox"
+        "This code is deserialized"
+    )
+
+    @classmethod
+    def from_context(cls, context: CodeExecutionPassAtOneContext) -> Self:
+        return cls(
+            run_env=context.run_env,
+            code_prompt=context.code_prompt,
+            test_code=context.test_code,
+            benchmark_timeout=context.benchmark_timeout,
+            snippet_merge_fn=CallableSerializer.decode(context.snippet_merge_fn),
+            output_parse_fn=CallableSerializer.decode(context.output_parse_fn),
+            package_downloads=context.package_downloads,
+        )
+
+
+class CodeExecutionPassAtOne(BaseMetric[Completion]):
+    NAME = "code-execution-pass@1"
+
+    def __init__(self) -> None:
+        self.k = 1
+        # NOTE : this serializer should be the same class as initialized in the benchmark
+        self.serializer = CallableSerializer()
+
+    def calculate(self, response: Completion) -> list[MetricResult]:
+        if response.error is not None:
+            return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=response.error)]
+        try:
+            context = extract_context_metric(response, CodeExecutionPassAtOneContext)
+            parsed_context = RealtimeCodeExectionContext.from_context(context)
+        except Exception as e:
+            raise Exception(f"Failed to rebuild parsing functions => {e}")
+
+        n = 1  # we only support N=1 at the moment
+        try:
+            c, output = self._count_correct_samples(response.completion, parsed_context)
+        except Exception as e:
+            error = Error(error_class=e.__class__.__name__, message=str(e), traceback=traceback.format_exc())
+            return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=error)]
+
+        pass_at_k_value = estimate_pass_at_k(n, c, self.k)
+        return [
+            MetricResult(
+                metric_name=self.NAME,
+                value=pass_at_k_value,
+                higher_is_better=True,
+                error=response.error,
+                code_execution_trace=output,
+            )
+        ]
+
+    def _count_correct_samples(self, completion: str, context: RealtimeCodeExectionContext) -> tuple[int, str]:
+        result = execute_python_code_with_tests(
+            code=completion,
+            test_code=context.test_code,
+            package_mapping=context.package_downloads,
+            merge_code_fn=context.snippet_merge_fn,
+            image=context.run_env,
+            timeout=context.benchmark_timeout,
+            parse_output_fn=context.output_parse_fn,
+        )
+        return (1 if result.success else 0), result.output
+
+
+def estimate_pass_at_k(n: int, c: int, k: int) -> float:
+    """
+    Estimates pass@k for a single problem.
+
+    Parameters:
+    n (int): Total number of generated samples.
+    c (int): Number of correct samples.
+    k (int): Number of attempts or samples considered.
+
+    Returns:
+    float: The pass@k value.
+    """
+    if n - c < k:
+        return 1.0
+
+    # Calculate the probability that at least one of the k samples is correct
+    probability = 1.0
+    for i in range(k):
+        probability *= (n - c - i) / (n - i)
+
+    return 1.0 - probability

eval_framework/metrics/completion/comet.py

@@ -0,0 +1,56 @@
+import torch
+from comet import download_model, load_from_checkpoint
+
+from eval_framework.exceptions import LogicError
+from eval_framework.metrics.base import BaseMetric, MetricResult
+from eval_framework.shared.types import Completion, UntemplatedPrompt
+from eval_framework.utils.constants import ROOT_DIR
+
+SAVING_DIR = ROOT_DIR / "comet_model"
+
+
+class COMET(BaseMetric[Completion]):
+    """COMET is a neural, multilingual framework for evaluating machine translation quality by leveraging cross-lingual
+    pretrained language models to achieve state-of-the-art correlation with human judgments
+    Note: this requires a Hugging Face token with access to the model: https://huggingface.co/Unbabel/XCOMET-XL
+    Source: https://github.com/Unbabel/COMET
+    Paper: https://arxiv.org/abs/2009.09025
+    """
+
+    NAME = "COMET"
+
+    def __init__(self) -> None:
+        checkpoint_path = download_model("Unbabel/XCOMET-XL", saving_directory=SAVING_DIR)
+        self.model = load_from_checkpoint(checkpoint_path)
+        assert torch.cuda.is_available(), "COMET requires a GPU"
+
+    def calculate(self, response: Completion) -> list[MetricResult]:
+        if response.error is not None:
+            return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=response.error)]
+
+        if (
+            response.context is None
+            or not isinstance(response.context, UntemplatedPrompt)
+            or response.context.untemplated_prompt == ""
+        ):
+            raise LogicError("When calculating COMET we need an untemplated prompt.")
+
+        scores = []
+        for ground_truth in response.ground_truth_list:
+            if ground_truth == "" or ground_truth is None:
+                raise LogicError("When calculating COMET we need a ground truth.")
+
+            data = [
+                {
+                    "src": response.context.untemplated_prompt.strip(),
+                    "mt": response.completion.strip(),
+                    "ref": ground_truth.strip(),
+                },
+            ]
+            with torch.no_grad():
+                model_output = self.model.predict(data, gpus=1)
+            scores.append(model_output.system_score)
+
+        return [
+            MetricResult(metric_name=self.NAME, value=float(max(scores)), higher_is_better=True, error=response.error)
+        ]

eval_framework/metrics/completion/concordance_index.py

@@ -0,0 +1,38 @@
+import ast
+
+from eval_framework.metrics.base import BaseMetric, MetricResult
+from eval_framework.shared.types import Completion
+
+
+class ConcordanceIndex(BaseMetric[Completion]):
+    NAME = "ConcordanceIndex"
+
+    def calculate(self, response: Completion) -> list[MetricResult]:
+        if response.error is not None:
+            return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=response.error)]
+
+        ground_truths = [gt for gt in response.ground_truth_list if gt is not None]
+        if not ground_truths:
+            return [MetricResult(metric_name=self.NAME, value=0.0, higher_is_better=True, error=response.error)]
+
+        concordance_count = max([calculate_concordance_index(gt, response.completion) for gt in ground_truths])
+        return [
+            MetricResult(metric_name=self.NAME, value=concordance_count, higher_is_better=True, error=response.error)
+        ]
+
+
+def calculate_concordance_index(
+    ground_truth: str,
+    completion: str,
+) -> float:
+    ground_truth_arr = ast.literal_eval(ground_truth)
+    completion_arr = ast.literal_eval(completion)
+
+    if len(ground_truth_arr) != len(completion_arr):
+        return 0
+
+    concordance_count = 0
+    for gt, c in zip(ground_truth_arr, completion_arr):
+        concordance_count += 1 if gt == c else 0
+
+    return concordance_count / len(ground_truth_arr)