eval-framework 0.2.7__py3-none-any.whl

eval_framework/metrics/llm/llm_judge_sql.py

@@ -0,0 +1,394 @@
+import atexit
+import logging
+import random
+import re
+import signal
+import sqlite3
+import threading
+from enum import Enum
+from time import sleep
+from typing import Any
+from uuid import uuid4
+
+import docker
+import mysql.connector
+import mysql.connector.abstracts
+import psycopg2  # type: ignore
+from pydantic import BaseModel
+
+from eval_framework.llm.base import BaseLLM
+from eval_framework.metrics.base import MetricResult
+from eval_framework.metrics.llm.base import BaseLLMJudgeMetric
+from eval_framework.metrics.llm.graders.language import Language
+from eval_framework.metrics.llm.graders.sql_quality_grader import SqlQualityGrader
+from eval_framework.shared.types import Completion, LanguageMetricContext, extract_context_metric
+from eval_framework.tasks.utils import get_docker_address
+
+logger = logging.getLogger(__name__)
+
+
+class SqlDialects(Enum):
+    sqlite = "sqlite"
+    postgres = "postgresql"
+    mysql = "mysql"
+    standard_sql = "standard_sql"
+
+
+class SqlOutputComparison(BaseModel):
+    matches_results_count: bool
+    matches_column_count: bool
+    results_equal: bool
+
+
+class SqlValidationResult(BaseModel):
+    success: bool
+    schema_error: str | None = None
+    query_error: str | None = None
+    results: list[Any] = []
+
+
+class LLMJudgeSqlMetricContext(LanguageMetricContext):
+    dialect: str
+    db_schema: str
+
+
+_DOCKER_LAUNCH_LOCK = threading.Lock()
+_MYSQL_PORT = 0
+_POSTGRES_PORT = 0
+
+
+class LLMJudgeSql(BaseLLMJudgeMetric):
+    NAME = "SQL Quality"
+
+    def __init__(self, llm_judge: BaseLLM):
+        super().__init__(llm_judge)
+        self._grader = SqlQualityGrader(llm_judge)
+
+        self.postgres_password = "mysecretpassword"
+        self.postgres_user = "postgres"
+
+        self.mysql_password = "mysecretpassword"
+        self.mysql_user = "root"
+        self.mysql_db_name = "mysql"
+
+        with _DOCKER_LAUNCH_LOCK:
+            if _MYSQL_PORT != 0 and _POSTGRES_PORT != 0:
+                return
+            self.client = docker.from_env()
+            atexit.register(self._shutdown_dbs)
+            signal.signal(signal.SIGTERM, lambda *_: self._shutdown_dbs())
+            self._start_postgres_db()
+            self._start_mysql_db()
+            self._wait_for_db_containers()
+
+    def calculate(self, response: Completion) -> list[MetricResult]:
+        if response.error is not None:
+            return [
+                MetricResult(metric_name=f"{self.NAME}/{k}", value=None, higher_is_better=True, error=response.error)
+                for k in [
+                    "successfully_runs",
+                    "is_just_sql",
+                    "matches_results_count",
+                    "matches_column_count",
+                    "results_equal",
+                    "llm_quality_score",
+                ]
+            ]
+
+        context = extract_context_metric(response, LLMJudgeSqlMetricContext)
+
+        assert isinstance(response.ground_truth, str)
+
+        schema_id = str(uuid4()).replace("-", "_")
+
+        expected_result = self.validate_query(
+            SqlDialects(context.dialect),
+            context.db_schema,
+            response.ground_truth,
+            f"golden_{schema_id}",
+        )
+        completion_stripped = response.completion.strip().strip("```sql").strip("```")
+        completion_query = extract_query_from_completions(completion_stripped)
+        if completion_query:
+            result = self.validate_query(
+                SqlDialects(context.dialect),
+                context.db_schema,
+                completion_query,
+                f"completion_{schema_id}",
+            )
+        else:
+            result = None
+
+        results = [
+            MetricResult(
+                metric_name=f"{self.NAME}/successfully_runs",
+                value=float(result is not None and result.success),
+                higher_is_better=True,
+                error=response.error,
+            ),
+            MetricResult(
+                metric_name=f"{self.NAME}/is_just_sql",
+                value=float(completion_query == completion_stripped),
+                higher_is_better=True,
+                error=response.error,
+            ),
+        ]
+
+        if result is not None and result.success:
+            output_comparison = SqlOutputComparison(
+                matches_results_count=len(expected_result.results) == len(result.results),
+                matches_column_count=count_result_columns(expected_result.results)
+                == count_result_columns(result.results),
+                results_equal=expected_result.results == result.results,
+            )
+            results.extend(
+                [
+                    MetricResult(
+                        metric_name=f"{self.NAME}/matches_results_count",
+                        value=float(output_comparison.matches_results_count),
+                        higher_is_better=True,
+                        error=response.error,
+                    ),
+                    MetricResult(
+                        metric_name=f"{self.NAME}/matches_column_count",
+                        value=float(output_comparison.matches_column_count),
+                        higher_is_better=True,
+                        error=response.error,
+                    ),
+                    MetricResult(
+                        metric_name=f"{self.NAME}/results_equal",
+                        value=float(output_comparison.results_equal),
+                        higher_is_better=True,
+                        error=response.error,
+                    ),
+                ]
+            )
+
+        grading = self._grader.grade(
+            prompt=response.user_instruction,
+            completion=completion_stripped,
+            result=result.results if result and result.success else None,
+            language=Language(response.get_instruction_language()),
+        )
+        results.append(
+            MetricResult(
+                metric_name=f"{self.NAME}/llm_quality_score",
+                # [0, 1] normalization required for visualizer
+                value=(float(grading.query_quality) - 1) / 4 if grading.query_quality is not None else None,
+                higher_is_better=True,
+                llm_judge_prompt=grading.judge_prompt,
+                llm_judge_response=grading.judge_response,
+                error=response.error,
+            )
+        )
+
+        return results
+
+    def _start_postgres_db(self) -> None:
+        global _POSTGRES_PORT
+        for _ in range(10):  # find a free port
+            try:
+                _POSTGRES_PORT = random.randint(1000, 65535)
+                self.postgres_docker = self.client.containers.run(
+                    "docker.io/postgres",
+                    environment={"POSTGRES_PASSWORD": self.postgres_password},
+                    ports={5432: _POSTGRES_PORT},
+                    tty=True,
+                    auto_remove=True,
+                    detach=True,
+                    network_mode="bridge",
+                )
+                break
+            except docker.errors.APIError as e:
+                if "port is already allocated" not in str(e):
+                    raise e
+                continue
+
+    def _start_mysql_db(self) -> None:
+        global _MYSQL_PORT
+        for _ in range(10):  # find a free port
+            try:
+                _MYSQL_PORT = random.randint(1000, 65535)
+                self.mysql_docker = self.client.containers.run(
+                    "docker.io/mysql:latest",
+                    environment={"MYSQL_ROOT_PASSWORD": self.mysql_password, "MYSQL_DATABASE": self.mysql_db_name},
+                    ports={3306: _MYSQL_PORT},
+                    tty=True,
+                    auto_remove=True,
+                    detach=True,
+                    network_mode="bridge",
+                )
+                break
+            except docker.errors.APIError as e:
+                if "port is already allocated" not in str(e):
+                    raise e
+                continue
+
+    def _wait_for_db_containers(self) -> None:
+        for _ in range(600):
+            try:
+                con = self.connect_to_postgres()
+                con.close()
+                con = self.connect_to_mysql()
+                con.close()
+                return
+            except Exception:
+                logger.info("Could not connect to DBs yet...")
+                sleep(1)
+        raise Exception("DBs not available.")
+
+    def _shutdown_dbs(self) -> None:
+        if hasattr(self, "postgres_docker"):
+            self.postgres_docker.kill()
+        if hasattr(self, "mysql_docker"):
+            self.mysql_docker.kill()
+
+    def validate_query(
+        self,
+        dialect: SqlDialects,
+        create_db_statements: str,
+        sql_query: str,
+        db_schema: str,
+    ) -> SqlValidationResult:
+        match dialect:
+            case SqlDialects.sqlite | SqlDialects.standard_sql:
+                return self.validate_query_sqlite(create_db_statements, sql_query, f"{dialect.value}_{db_schema}")
+            case SqlDialects.postgres:
+                return self.validate_query_postgres(create_db_statements, sql_query, f"{dialect.value}_{db_schema}")
+            case SqlDialects.mysql:
+                return self.validate_query_mysql(create_db_statements, sql_query, f"{dialect.value}_{db_schema}")
+            case _:
+                raise NotImplementedError(f"Query validation not implemented for {dialect.value}.")
+
+    def validate_query_sqlite(self, create_db_statements: str, sql_query: str, db_schema: str) -> SqlValidationResult:
+        con = sqlite3.connect(":memory:")
+        cur = con.cursor()
+        try:
+            statements = separate_statements(create_db_statements)
+            for statement in statements:
+                cur.execute(statement)
+                con.commit()
+        except Exception as e:
+            logger.info(f"Create statements are not compatible with SQLite. Reason: {e}")
+            return SqlValidationResult(success=False, schema_error=str(e))
+        try:
+            queries = separate_statements(sql_query)
+            for query in queries:
+                cur.execute(query)
+                con.commit()
+            results = cur.fetchall()
+        except Exception as e:
+            logger.info(f"SQL query is not compatible with SQLite. Reason: {e}")
+            return SqlValidationResult(success=False, query_error=str(e))
+
+        con.close()
+        return SqlValidationResult(success=True, results=results)
+
+    def connect_to_postgres(self) -> psycopg2.extensions.connection:
+        conn_params = {
+            "dbname": "postgres",
+            "user": self.postgres_user,
+            "password": self.postgres_password,
+            "host": get_docker_address(),
+            "port": _POSTGRES_PORT,
+        }
+        return psycopg2.connect(**conn_params)
+
+    def validate_query_postgres(self, create_db_statements: str, sql_query: str, db_schema: str) -> SqlValidationResult:
+        con = self.connect_to_postgres()
+        cur = con.cursor()
+        cur.execute(f"CREATE SCHEMA {db_schema};")
+        con.commit()
+        cur.execute(f"ALTER USER {self.postgres_user} set SEARCH_PATH = {db_schema};")
+        con.commit()
+        try:
+            statements = separate_statements(create_db_statements)
+            for statement in statements:
+                cur.execute(statement)
+            con.commit()
+        except Exception as e:
+            logger.info(f"Create statements are not compatible with PostgreSQL. Reason: {e}")
+            return SqlValidationResult(success=False, schema_error=str(e))
+        try:
+            queries = separate_statements(sql_query)
+            for query in queries:
+                cur.execute(query)
+                con.commit()
+            results = cur.fetchall()
+        except Exception as e:
+            logger.info(f"SQL query is not compatible with PostgreSQL. Reason: {e}")
+            return SqlValidationResult(success=False, query_error=str(e))
+
+        con.commit()
+
+        con.close()
+        return SqlValidationResult(success=True, results=results)
+
+    def connect_to_mysql(
+        self,
+    ) -> mysql.connector.pooling.PooledMySQLConnection | mysql.connector.abstracts.MySQLConnectionAbstract:
+        conn_params = {
+            "database": self.mysql_db_name,
+            "user": self.mysql_user,
+            "password": self.mysql_password,
+            "host": get_docker_address(),
+            "port": _MYSQL_PORT,
+        }
+        return mysql.connector.connect(**conn_params)
+
+    def validate_query_mysql(self, create_db_statements: str, sql_query: str, db_schema: str) -> SqlValidationResult:
+        con = self.connect_to_mysql()
+        cur = con.cursor(buffered=True)
+        cur.execute(f"CREATE SCHEMA {db_schema};")
+        con.commit()
+        cur.execute(f"USE {db_schema};")
+        try:
+            statements = separate_statements(create_db_statements)
+            for statement in statements:
+                cur.execute(statement)
+                con.commit()
+        except Exception as e:
+            logger.info(f"Create statements are not compatible with MySQL. Reason: {e}")
+            con.close()
+            return SqlValidationResult(success=False, schema_error=str(e))
+        try:
+            queries = separate_statements(sql_query)
+            for query in queries:
+                cur.execute(query)
+                con.commit()
+            results = cur.fetchall()
+        except Exception as e:
+            logger.info(f"SQL query is not compatible with MySQL. Reason: {e}")
+            con.close()
+            return SqlValidationResult(success=False, query_error=str(e))
+
+        cur.close()
+        con.close()
+        return SqlValidationResult(success=True, results=results)
+
+
+def separate_statements(statements: str) -> list[str]:
+    return statements.split(";")[:-1]
+
+
+def is_create_table_statement(statement: str) -> bool:
+    return "CREATE TABLE" in statement
+
+
+def count_result_columns(result: list[Any]) -> int:
+    if len(result) == 0:
+        return 0
+    return len(result[0])
+
+
+def extract_query_from_completions(completion: str) -> str | None:
+    # Match SQL blocks starting with SELECT or WITH at line start
+    # (allowing punctuation/whitespace), ending at first semicolon
+    pattern = re.compile(r"(?:^|\n)[^a-zA-Z0-9_]*((?:select|with)\b.*?;)", re.IGNORECASE | re.DOTALL)
+
+    matches = pattern.findall(completion)
+
+    # Return the query only if exactly one match is found
+    if len(matches) == 1:
+        return matches[0].strip()
+    return None

eval_framework/metrics/llm/llm_judge_world_knowledge.py

@@ -0,0 +1,37 @@
+from eval_framework.llm.base import BaseLLM
+from eval_framework.metrics.base import MetricResult
+from eval_framework.metrics.llm.base import BaseLLMJudgeMetric
+from eval_framework.metrics.llm.graders.language import Language
+from eval_framework.metrics.llm.graders.summary_world_knowledge_grader import SummarizationWorldKnowledgeGrader
+from eval_framework.shared.types import Completion
+
+
+class LLMJudgeWorldKnowledge(BaseLLMJudgeMetric):
+    NAME = "World Knowledge"
+
+    def __init__(self, llm_judge: BaseLLM):
+        super().__init__(llm_judge)
+        self._grader = SummarizationWorldKnowledgeGrader(llm_judge)
+
+    def calculate(self, response: Completion) -> list[MetricResult]:
+        if response.error is not None:
+            return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=False, error=response.error)]
+
+        language = Language(response.get_instruction_language())
+
+        grading = self._grader.grade(
+            reference_input=response.user_instruction,
+            completion=response.sanitized_completion,
+            language=language,
+        )
+
+        return [
+            MetricResult(
+                metric_name=self.NAME,
+                value=float(grading.contains_world_knowledge) if grading.contains_world_knowledge is not None else None,
+                higher_is_better=False,
+                llm_judge_prompt=grading.judge_prompt,
+                llm_judge_response=grading.judge_response,
+                error=response.error,
+            )
+        ]

eval_framework/metrics/llm/utils.py

@@ -0,0 +1,20 @@
+"""Utility functions for LLM-based metrics."""
+
+
+def order_answers_for_comparison(candidate: str, reference: str, swap: bool) -> tuple[str, str]:
+    """Order candidate and reference answers for A/B comparison.
+
+    This function is used to mitigate position bias in LLM-as-judge evaluations
+    by optionally swapping the order in which answers are presented.
+
+    Args:
+        candidate: The candidate completion to evaluate.
+        reference: The reference/baseline completion.
+        swap: If True, swap the order (reference becomes A, candidate becomes B).
+
+    Returns:
+        Tuple of (answer_a, answer_b) in the correct order.
+    """
+    if swap:
+        return reference, candidate
+    return candidate, reference

eval_framework/metrics/loglikelihood/accuracy_loglikelihood.py

@@ -0,0 +1,51 @@
+from eval_framework.metrics.base import BaseMetric, MetricResult
+from eval_framework.shared.types import Loglikelihood
+
+
+class AccuracyLoglikelihood(BaseMetric[Loglikelihood]):
+    NAME = "Accuracy Loglikelihood"
+
+    def calculate(self, response: Loglikelihood) -> list[MetricResult]:
+        if response.error is not None:
+            return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=response.error)]
+
+        ground_truth_list = response.ground_truth_list
+        completion_text = max(response.loglikelihoods, key=response.loglikelihoods.get)  # type: ignore[arg-type]
+
+        return [
+            MetricResult(
+                metric_name=self.NAME,
+                value=float(completion_text in ground_truth_list),
+                higher_is_better=True,
+                error=response.error,
+            )
+        ]
+
+
+class AccuracyNormLoglikelihood(BaseMetric[Loglikelihood]):
+    NAME = "Accuracy Normalized Loglikelihood"
+
+    def calculate(self, response: Loglikelihood) -> list[MetricResult]:
+        if response.error is not None:
+            return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=response.error)]
+
+        ground_truth_list = response.ground_truth_list
+
+        output_len_normalized = {}
+        for k, v in response.loglikelihoods.items():
+            completion_length = len(k)
+
+            if completion_length != 0:
+                output_len_normalized[k] = v / completion_length
+            else:
+                output_len_normalized[k] = v
+
+        model_output_len_normalized = max(output_len_normalized, key=output_len_normalized.get)  # type:ignore
+        return [
+            MetricResult(
+                metric_name=self.NAME,
+                value=float(model_output_len_normalized in ground_truth_list),
+                higher_is_better=True,
+                error=response.error,
+            )
+        ]

eval_framework/metrics/loglikelihood/base.py

@@ -0,0 +1,50 @@
+import math
+
+from eval_framework.metrics.base import BaseMetric
+from eval_framework.shared.types import Loglikelihood
+
+
+class BaseLoglikelihoodMetric(BaseMetric[Loglikelihood]):
+    """Base class for metrics that operate on loglikelihood responses."""
+
+    def __init__(
+        self,
+        *,
+        len_normalised: bool = True,
+    ) -> None:
+        self.len_normalised = len_normalised
+
+    def _normalise_text(self, text: str) -> str:
+        return text.strip().lower()
+
+    def _length_normalise_loglikelihoods(self, loglikelihoods: dict) -> dict:
+        """Return a dict of length-normalised loglikelihoods."""
+        output = {}
+        for k, v in loglikelihoods.items():
+            length = len(k)
+            output[k] = v / length if length > 0 else v
+        return output
+
+    def _compute_probabilities(self, loglikelihoods: dict) -> tuple[dict, dict]:
+        """Compute probabilities from loglikelihoods, with optional length normalisation."""
+        if self.len_normalised:
+            loglikelihoods = self._length_normalise_loglikelihoods(loglikelihoods)
+        return loglikelihoods, self._softmax(loglikelihoods)
+
+    def _gather_ground_truths(self, response: Loglikelihood) -> set[str]:
+        """Extract and normalize ground truth completions from a Loglikelihood response."""
+        ground_truths = set(
+            self._normalise_text(gt)
+            for gt in (response.ground_truth if isinstance(response.ground_truth, list) else [response.ground_truth])
+        )
+        return ground_truths
+
+    def _softmax(self, log_probs: dict) -> dict:
+        """Convert log-likelihoods to probabilities with softmax."""
+        vals = list(log_probs.values())
+        if not vals:  # no valid entries
+            return {}
+        m = max(vals)
+        exp_vals = [math.exp(x - m) for x in vals]
+        total = sum(exp_vals)
+        return {k: ev / total for k, ev in zip(log_probs.keys(), exp_vals)}

eval_framework/metrics/loglikelihood/confidence_weighted_accuracy.py

@@ -0,0 +1,25 @@
+from eval_framework.metrics.base import MetricResult
+from eval_framework.metrics.loglikelihood.base import BaseLoglikelihoodMetric
+from eval_framework.shared.types import Loglikelihood
+
+
+class ConfidenceWeightedAccuracy(BaseLoglikelihoodMetric):
+    NAME = "Confidence-weighted Accuracy"
+
+    def __init__(self, *, len_normalised: bool = True) -> None:
+        super().__init__(len_normalised=len_normalised)
+
+    def calculate(self, response: Loglikelihood) -> list[MetricResult]:
+        if response.error is not None:
+            return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=response.error)]
+
+        loglikelihoods, probs = self._compute_probabilities(response.loglikelihoods)
+        ground_truths = self._gather_ground_truths(response)
+
+        best_key = max(loglikelihoods, key=loglikelihoods.get)  # type: ignore[arg-type]
+        best_key_norm = self._normalise_text(best_key)
+        p_c = probs.get(best_key, 0.0)
+
+        accuracy = p_c if best_key_norm in ground_truths else 0.0
+
+        return [MetricResult(metric_name=self.NAME, value=accuracy, higher_is_better=True, error=response.error)]

eval_framework/metrics/loglikelihood/dcs.py

@@ -0,0 +1,43 @@
+from eval_framework.metrics.base import MetricResult
+from eval_framework.metrics.loglikelihood.base import BaseLoglikelihoodMetric
+from eval_framework.shared.types import Loglikelihood
+
+
+class DistributionalCorrectnessScore(BaseLoglikelihoodMetric):
+    """Based on Burns (2025) Measuring Language Model Hallucinations Through Distributional Correctness."""
+
+    NAME = "Distributional Correctness Score"
+
+    def __init__(
+        self,
+        *,
+        lc: float = 1.0,  # Default reward weight for correct answers
+        lw: float = 1.0,  # Default penalty weight for wrong answers
+        len_normalised: bool = True,
+    ) -> None:
+        super().__init__(len_normalised=len_normalised)
+        self._lc = float(lc)
+        self._lw = float(lw)
+        if not (self._lc >= 0 and self._lw >= 0 and self._lc >= self._lw):
+            raise ValueError(f"Invalid DCS loadings: lc={self._lc}, lw={self._lw}. Require lc>=0, lw>=0, and lc>=lw.")
+
+    def calculate(self, response: Loglikelihood) -> list[MetricResult]:
+        if response.error is not None:
+            return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=response.error)]
+
+        loglikelihoods, probs = self._compute_probabilities(response.loglikelihoods)
+        ground_truths = self._gather_ground_truths(response)
+
+        idk_key = self._normalise_text(list(response.loglikelihoods.keys())[-1])  # assumes last key is "IDK" option
+
+        p_c = sum(p for k, p in probs.items() if self._normalise_text(k) in ground_truths)
+        p_idk = probs.get(idk_key, 0.0)
+        p_w = sum(
+            p
+            for k, p in probs.items()
+            if (self._normalise_text(k) not in ground_truths and self._normalise_text(k) != idk_key)
+        )
+
+        dcs = (self._lc * p_c - self._lw * p_w) * (1.0 - p_idk)
+
+        return [MetricResult(metric_name=self.NAME, value=float(dcs), higher_is_better=True, error=response.error)]

eval_framework/metrics/loglikelihood/probability_mass.py

@@ -0,0 +1,53 @@
+import numpy as np
+
+from eval_framework.metrics.base import BaseMetric, MetricResult
+from eval_framework.shared.types import Loglikelihood
+
+
+class ProbabilityMass(BaseMetric[Loglikelihood]):
+    NAME = "Probability Mass"
+
+    def calculate(self, response: Loglikelihood) -> list[MetricResult]:
+        if response.error is not None:
+            return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=response.error)]
+
+        assert isinstance(response.ground_truth, list)
+        # https://docs.python.org/3.10/library/stdtypes.html?highlight=dictview#dictionary-view-objects
+        in_ground_truths = [completion in response.ground_truth for completion in response.loglikelihoods]
+        log_probs = list(response.loglikelihoods.values())
+
+        probs = np.exp(log_probs) / np.sum(np.exp(log_probs))
+        prob_mass = np.sum(probs[in_ground_truths])
+
+        return [
+            MetricResult(metric_name=self.NAME, value=float(prob_mass), higher_is_better=True, error=response.error)
+        ]
+
+
+class ProbabilityMassNorm(BaseMetric[Loglikelihood]):
+    NAME = "Probability Mass Normalized"
+
+    def calculate(self, response: Loglikelihood) -> list[MetricResult]:
+        if response.error is not None:
+            return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=response.error)]
+
+        assert isinstance(response.ground_truth, list)
+        # len normalized
+
+        output_len_normalized = {}
+        for k, v in response.loglikelihoods.items():
+            completion_length = len(k)
+
+            if completion_length != 0:
+                output_len_normalized[k] = v / completion_length
+            else:
+                output_len_normalized[k] = v
+
+        log_probs = list(output_len_normalized.values())
+        in_ground_truths = [completion in response.ground_truth for completion in response.loglikelihoods]
+        log_probs = list(output_len_normalized.values())
+
+        probs = np.exp(log_probs) / np.sum(np.exp(log_probs))
+        prob_mass_norm = np.sum(probs[in_ground_truths])
+
+        return [MetricResult(metric_name=self.NAME, value=prob_mass_norm, higher_is_better=True, error=response.error)]