crfm-helm 0.4.0__py3-none-any.whl → 0.5.10__py3-none-any.whl

helm/benchmark/metrics/efficiency_metrics.py

@@ -0,0 +1,287 @@
+from typing import Dict, List, Optional
+
+import json
+import importlib_resources as resources
+
+from helm.common.hierarchical_logger import hwarn
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.benchmark.adaptation.adapter_spec import AdapterSpec
+from helm.benchmark.window_services.window_service import WindowService
+from helm.benchmark.window_services.window_service_factory import WindowServiceFactory
+from helm.benchmark.window_services.tokenizer_service import TokenizerService
+from helm.benchmark.metrics.metric import MetricMetadata
+from helm.benchmark.metrics.metric_name import MetricName
+from helm.benchmark.metrics.metric_service import MetricService
+from helm.benchmark.metrics.statistic import Stat
+
+
+EFFICIENCY_DATA_PACKAGE: str = "helm.benchmark.efficiency_data"
+
+INFERENCE_IDEALIZED_RUNTIMES_JSON_FILENAME: str = "inference_idealized_runtimes.json"
+INFERENCE_DENOISED_RUNTIMES_JSON_FILENAME: str = "inference_denoised_runtimes.json"
+TRAINING_EFFICIENCY_JSON_FILENAME: str = "training_efficiency.json"
+
+
+# TODO Actually make this work like a Metric. The current state is just trying to split
+# it out of other Metrics to make refactoring easier.
+class EfficiencyMetric:
+    def __init__(self):
+        # For Efficiency metrics:
+        # The `inference_efficiency.json` file contains a `runtime_per_output_token` value
+        # (the estimated runtime of generating one output token) and a
+        # `runtime_for_prompt_tokens` dict (a mapping from various num_prompt_tokens values to
+        # the estimated runtime of encoding a prompt with that many tokens).
+        # For example:
+        # "openai/davinci": {
+        #   "runtime_per_output_token": 0.080,
+        #   "runtime_for_prompt_tokens": {
+        #     "1": 0.016,
+        #     "16": 0.018,
+        #     "32": 0.020,
+        #     ...
+        #
+        # These runtimes are generated by initializing Megatron with a model of the right size,
+        # obtaining end-to-end generation times for different numbers of prompt and output tokens,
+        # and then fitting a linear regression model to the runtimes: the resulting slope is the
+        # runtime_per_output_token, which is the processing time for generating each output token,
+        # and the y-intercept is the runtime_for_prompt_tokens, with different values for different
+        # num_prompt_tokens values.
+        # Profiling code and logs, and code to fit the regression model is available at
+        # https://github.com/stanford-crfm/benchmarking_efficiency.
+        data_package = resources.files(EFFICIENCY_DATA_PACKAGE)
+        with data_package.joinpath(INFERENCE_IDEALIZED_RUNTIMES_JSON_FILENAME).open("r") as f:
+            self.inference_idealized_runtimes_dict = json.load(f)
+        with data_package.joinpath(INFERENCE_DENOISED_RUNTIMES_JSON_FILENAME).open("r") as f:
+            self.inference_denoised_runtimes_dict = json.load(f)
+        # We use estimated emitted CO2 during training (in tons of CO2) as a proxy metric
+        # for training efficiency. We use reported metrics where applicable, otherwise
+        # we estimate them from runtime information, type and number of hardware accelerators
+        # used, region, etc.
+        with data_package.joinpath(TRAINING_EFFICIENCY_JSON_FILENAME).open("r") as f:
+            self.training_efficiency_dict = json.load(f)
+
+    def compute_efficiency_metrics(
+        self, adapter_spec: AdapterSpec, request_state: RequestState, metric_service: MetricService
+    ) -> List[Stat]:
+        """Compute efficiency metrics for both inference and training.
+        For inference, we record both the actual runtime and an estimated idealized runtime
+        for the given request with an optimized software implementation run on A100 GPU(s),
+        taking into account both the number of tokens in the prompt of the request, and the
+        number of generated output tokens.
+        For training, we report the estimated total metric tons of CO2 emitted to train the
+        model. This is the same for each request."""
+        # Compute efficiency metrics for inference.
+        assert request_state.result is not None
+
+        runtime: Optional[float] = None
+        batch_size: Optional[int] = None
+        # Compute efficiency metrics for inference.
+        if request_state.result.request_time is not None:
+            runtime = request_state.result.request_time
+            batch_size = 1
+        # For models that perform offline batch inference, effective runtime is batch_request_time, but also
+        # record batch_size to provide nuance.
+        if request_state.result.batch_request_time is not None and request_state.result.batch_size is not None:
+            runtime = request_state.result.batch_request_time
+            batch_size = request_state.result.batch_size
+
+        # Compute total number of prompt and output tokens.
+        # Fetch the right `Tokenizer` depending on the model defined in `AdapterSpec`
+        # and calculate the number of tokens in the prompt.
+        tokenizer_service: TokenizerService = metric_service
+        window_service: WindowService = WindowServiceFactory.get_window_service(
+            adapter_spec.model_deployment, tokenizer_service
+        )
+
+        prompt: str
+        num_prompt_tokens: int
+        if request_state.request.multimodal_prompt is not None:
+            prompt = request_state.request.multimodal_prompt.text
+            num_prompt_tokens = window_service.get_num_tokens(prompt)
+        else:
+            prompt = request_state.request.prompt
+            num_prompt_tokens = window_service.get_num_tokens(prompt)
+
+        # Total number of tokens in the completion.
+        num_completion_tokens: int = sum([len(completion.tokens) for completion in request_state.result.completions])
+        # Don't include prompt in number of generated tokens (e.g., for language modeling).
+        # Assume that tokens for different completions are generated sequentially (instead of batched) when
+        # computing num_output_tokens (for the purpose of runtime estimation).
+        num_output_tokens: int = num_completion_tokens
+        if request_state.request.echo_prompt:
+            # num_prompt_tokens > num_output_tokens can happen if tokenizer doesn't round trip.
+            if num_prompt_tokens <= num_output_tokens:
+                num_output_tokens -= num_prompt_tokens
+            else:
+                hwarn(
+                    f"num_prompt_tokens ({num_prompt_tokens}) > num_output_tokens ({num_output_tokens}) "
+                    f"for prompt: {prompt}"
+                )
+                num_output_tokens = 0
+
+        idealized_runtime: Optional[float] = _compute_estimated_time_from_prompt_size_and_num_output_tokens(
+            request_state, self.inference_idealized_runtimes_dict, num_prompt_tokens, num_output_tokens
+        )
+
+        denoised_runtime: Optional[float] = _compute_estimated_time_from_prompt_size_and_num_output_tokens(
+            request_state, self.inference_denoised_runtimes_dict, num_prompt_tokens, num_output_tokens
+        )
+        # Denoised runtime for offline models is just runtime.
+        # We divide by batch_size to get approximate per-input runtime.
+        if runtime is not None and request_state.result.batch_size is not None:
+            denoised_runtime = runtime / request_state.result.batch_size
+
+        # Compute efficiency metrics for training.
+        training_co2_cost: Optional[float]
+        if request_state.request.model_deployment in self.training_efficiency_dict["carbon"]:
+            training_co2_cost = self.training_efficiency_dict["carbon"][request_state.request.model_deployment]["value"]
+        else:
+            training_co2_cost = None
+
+        training_energy_cost: Optional[float]
+        if request_state.request.model_deployment in self.training_efficiency_dict["energy"]:
+            training_energy_cost = self.training_efficiency_dict["energy"][request_state.request.model_deployment][
+                "value"
+            ]
+        else:
+            training_energy_cost = None
+
+        stats = [
+            Stat(MetricName("num_prompt_tokens")).add(num_prompt_tokens),
+            Stat(MetricName("num_completion_tokens")).add(num_completion_tokens),
+            Stat(MetricName("num_output_tokens")).add(num_output_tokens),
+            Stat(MetricName("training_co2_cost")).add(training_co2_cost),
+            Stat(MetricName("training_energy_cost")).add(training_energy_cost),
+        ]
+        if runtime is not None:
+            stats.append(Stat(MetricName("inference_runtime")).add(runtime))
+        if batch_size is not None:
+            stats.append(Stat(MetricName("batch_size")).add(batch_size))
+        if denoised_runtime is not None:
+            stats.append(Stat(MetricName("inference_denoised_runtime")).add(denoised_runtime))
+        if idealized_runtime is not None:
+            stats.append(Stat(MetricName("inference_idealized_runtime")).add(idealized_runtime))
+        return stats
+
+    def get_metadata(self) -> List[MetricMetadata]:
+        return [
+            MetricMetadata(
+                name="num_prompt_tokens",
+                display_name="# prompt tokens",
+                short_display_name=None,
+                description="Number of tokens in the prompt.",
+                lower_is_better=None,
+                group="general_information",
+            ),
+            MetricMetadata(
+                name="num_completion_tokens",
+                display_name="# completion tokens",
+                description="Actual number of completion tokens (over all completions).",
+                lower_is_better=None,
+            ),
+            MetricMetadata(
+                name="num_output_tokens",
+                display_name="# output tokens",
+                description="Actual number of output tokens.",
+                lower_is_better=None,
+            ),
+            MetricMetadata(
+                name="training_co2_cost",
+                display_name="Estimated training emissions (kg CO2)",
+                short_display_name="Training emissions (kg CO2)",
+                description="Estimate of the CO2 emissions from training the model.",
+                lower_is_better=True,
+                group="efficiency_detailed",
+            ),
+            MetricMetadata(
+                name="training_energy_cost",
+                display_name="Estimated training energy cost (MWh)",
+                short_display_name="Training energy (MWh)",
+                description="Estimate of the amount of energy used to train the model.",
+                lower_is_better=True,
+                group="efficiency_detailed",
+            ),
+            MetricMetadata(
+                name="inference_runtime",
+                display_name="Observed inference runtime (s)",
+                short_display_name="Observed inference time (s)",
+                description="Average observed time to process a request to the model (via an API, and thus depends on "
+                "particular deployment).",
+                lower_is_better=True,
+                group="efficiency_detailed",
+            ),
+            MetricMetadata(
+                name="batch_size",
+                display_name="Batch size",
+                description="For batch jobs, how many requests are in a batch.",
+                lower_is_better=None,
+            ),
+            MetricMetadata(
+                name="inference_denoised_runtime",
+                display_name="Denoised inference runtime (s)",
+                short_display_name="Denoised inference time (s)",
+                description="Average time to process a request to the model minus performance contention by using "
+                "profiled runtimes from multiple trials of SyntheticEfficiencyScenario.",
+                lower_is_better=True,
+                group="efficiency_detailed",
+            ),
+            MetricMetadata(
+                name="inference_idealized_runtime",
+                display_name="Idealized inference runtime (s)",
+                short_display_name="Idealized inference time (s)",
+                description="Average time to process a request to the model based solely on the model architecture "
+                "(using Megatron-LM).",
+                lower_is_better=True,
+                group="efficiency_detailed",
+            ),
+        ]
+
+
+def _compute_estimated_time_from_prompt_size_and_num_output_tokens(
+    request_state: RequestState,
+    inference_runtimes_dict: Dict[str, Dict],
+    num_prompt_tokens: int,
+    num_output_tokens: int,
+) -> Optional[float]:
+    estimated_runtime: Optional[float]
+    if request_state.request.model_deployment in inference_runtimes_dict:
+        inference_runtimes_dict_for_model = inference_runtimes_dict[request_state.request.model_deployment]
+        runtime_per_output_token: float = inference_runtimes_dict_for_model["runtime_per_output_token"]
+        raw_runtimes_for_prompt_tokens: Dict[str, float] = inference_runtimes_dict_for_model[
+            "runtime_for_prompt_tokens"
+        ]
+        runtimes_for_prompt_tokens: Dict[int, float] = {int(k): v for (k, v) in raw_runtimes_for_prompt_tokens.items()}
+
+        runtime_for_prompt_tokens: Optional[float] = None
+        largest_num_tokens_in_efficiency_dict: int = max(runtimes_for_prompt_tokens.keys())
+        # Find the smallest num_prompt_tokens larger than the number of tokens in the given prompt,
+        # then scale runtime in dict by (num_prompt_tokens / key) to get more accurate estimate: we
+        # assume that we can encode the prompt at the same throughput as the smallest key larger than
+        # num_prompt_tokens, and number of compute operations scales linearly with num_prompt_tokens.
+        for key in sorted(runtimes_for_prompt_tokens.keys()):
+            if num_prompt_tokens <= key:
+                runtime_for_prompt_tokens = runtimes_for_prompt_tokens[key] * (num_prompt_tokens / key)
+                break
+        # If number of tokens in the prompt exceeds the largest key in the efficiency dict, then
+        # estimate the prompt encoding time by linearly scaling up the runtime for the largest
+        # key (this is reasonably accurate under certain simplifying assumptions).
+        if runtime_for_prompt_tokens is None:
+            runtime_for_prompt_tokens = runtimes_for_prompt_tokens[largest_num_tokens_in_efficiency_dict] * (
+                num_prompt_tokens / largest_num_tokens_in_efficiency_dict
+            )
+        overhead: Optional[float] = inference_runtimes_dict_for_model.get("overhead")
+
+        # Idealized runtime is sum of the runtime of encoding the input tokens, the runtime of
+        # generating `num_output_tokens` (`runtime_per_output_token` * (`num_output_tokens` - 1))
+        # if number of output tokens is greater than 0, otherwise just `runtime_for_prompt_tokens`,
+        # and the overhead if available.
+        estimated_runtime = runtime_for_prompt_tokens
+        if num_output_tokens > 0:
+            estimated_runtime += runtime_per_output_token * (num_output_tokens - 1)
+        # Add overhead if it is available.
+        if overhead is not None:
+            estimated_runtime += overhead
+    else:
+        estimated_runtime = None
+
+    return estimated_runtime

helm/benchmark/metrics/ehr_sql_metrics.py

@@ -0,0 +1,159 @@
+from typing import List
+from helm.benchmark.adaptation.adapter_spec import AdapterSpec
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.benchmark.metrics.metric import Metric, MetricMetadata
+from helm.benchmark.metrics.metric_name import MetricName
+from helm.benchmark.metrics.metric_service import MetricService
+from helm.benchmark.metrics.statistic import Stat
+from helm.common.hierarchical_logger import hlog
+
+
+class EhrSqlMetric(Metric):
+    """
+    Metric for evaluating the EHR SQL dataset, focusing on:
+    1. Execution Accuracy – Whether the generated SQL query produces the same results as the ground truth.
+    2. Query Validity – Whether the generated SQL executes without errors.
+    3. Precision for Answerable Questions (Pans).
+    4. Recall for Answerable Questions (Rans).
+    """
+
+    def evaluate_generation(
+        self,
+        adapter_spec: AdapterSpec,
+        request_state: RequestState,
+        metric_service: MetricService,
+        eval_cache_path: str,
+    ) -> List[Stat]:
+        """
+        Evaluate execution accuracy, query validity, and answerability metrics.
+        """
+
+        if not request_state.annotations:
+            hlog(f"Warning: Request state missing annotations for instance {request_state.instance}")
+            return []
+
+        if "ehr_sql" not in request_state.annotations:
+            hlog(f"Warning: 'ehr_sql' key missing in annotations for instance {request_state.instance}")
+            return []
+
+        # Extract execution results
+        predicted_result = request_state.annotations["ehr_sql"].get("predicted_result", [])
+        ground_truth_result = request_state.annotations["ehr_sql"].get("ground_truth_result", [])
+        query_error = request_state.annotations["ehr_sql"].get("query_error", None)
+
+        # Extract predictions from the model output
+        if request_state.result is None:
+            predictions = []
+        else:
+            predictions = [completion.text.strip() for completion in request_state.result.completions]
+        if not predictions:
+            hlog(f"Warning: No predictions found in the completions for instance {request_state.instance}")
+            return []
+
+        # Process the first prediction as the primary output
+        prediction = predictions[0].strip()
+
+        # Extract references and input text
+        references = getattr(request_state.instance, "references", None)
+
+        if not references or len(references) == 0:
+            hlog(f"Warning: Missing references for instance {request_state.instance}")
+            return []
+
+        # Check if the ground truth is answerable based on `is_impossible` flag
+        ground_truth_query = references[0].output.text.strip() if references else None
+        is_impossible = (
+            request_state.instance.extra_data.get("is_impossible", False)
+            if request_state.instance.extra_data
+            else False
+        )
+
+        is_answerable = not is_impossible and bool(ground_truth_query)  # True if the ground truth is answerable
+        is_predicted_answerable = bool(prediction)  # True if the model generated a non-empty SQL query
+        correct_answerable = int(is_answerable and is_predicted_answerable)  # Correct if both are answerable
+
+        # **Execution Accuracy Fix:**
+        execution_accuracy = 0
+
+        if ground_truth_query:
+            if ground_truth_result and predicted_result:
+                execution_accuracy = int(set(predicted_result) == set(ground_truth_result))  # Compare sets.
+            elif not ground_truth_result and not predicted_result and not prediction:
+                execution_accuracy = 1  # Both empty and no query was generated.
+        elif not ground_truth_query and prediction:
+            execution_accuracy = 0  # LLM generated a query when no gold query exists.
+
+        # **Query Validity Fix:**
+        if not prediction:  # No SQL query was generated
+            query_validity = 0
+        elif query_error is None:
+            query_validity = 1  # Query executed successfully.
+        else:
+            query_validity = 0  # Execution error occurred.
+
+        return [
+            # Execution-based Metrics
+            Stat(MetricName("ehr_sql_execution_accuracy")).add(execution_accuracy),
+            Stat(MetricName("ehr_sql_query_validity")).add(query_validity),
+            # Answerability Metrics
+            Stat(MetricName("ehr_sql_precision_answerable")).add(correct_answerable if is_predicted_answerable else 0),
+            Stat(MetricName("ehr_sql_recall_answerable")).add(correct_answerable if is_answerable else 0),
+            Stat(MetricName("ehr_sql_total_predicted_answerable")).add(int(is_predicted_answerable)),
+            Stat(MetricName("ehr_sql_total_ground_truth_answerable")).add(int(is_answerable)),
+        ]
+
+    def get_metadata(self) -> List[MetricMetadata]:
+        return [
+            MetricMetadata(
+                name="ehr_sql_execution_accuracy",
+                display_name="Execution accuracy for Generated Query",
+                short_display_name="EHRSQLExeAcc",
+                description="Measures the proportion of correctly predicted answerable questions among all questions "
+                "predicted to be answerable.",
+                lower_is_better=False,
+                group=None,
+            ),
+            MetricMetadata(
+                name="ehr_sql_query_validity",
+                display_name="Validity of Generated Query",
+                short_display_name="EHRSQLQueryValid",
+                description="Measures the proportion of correctly predicted answerable questions among all answerable "
+                "questions in the dataset.",
+                lower_is_better=False,
+                group=None,
+            ),
+            MetricMetadata(
+                name="ehr_sql_precision_answerable",
+                display_name="Precision for Answerable Questions",
+                short_display_name="EHRSQLPreAns",
+                description="Measures the proportion of correctly predicted answerable questions among all questions "
+                "predicted to be answerable.",
+                lower_is_better=False,
+                group=None,
+            ),
+            MetricMetadata(
+                name="ehr_sql_recall_answerable",
+                display_name="Recall for Answerable Questions",
+                short_display_name="EHRSQLReAns",
+                description="Measures the proportion of correctly predicted answerable questions among all answerable "
+                "questions in the dataset.",
+                lower_is_better=False,
+                group=None,
+            ),
+            MetricMetadata(
+                name="ehr_sql_total_predicted_answerable",
+                display_name="Total Predicted Answerable",
+                short_display_name="Total Pred Ans",
+                description="Total number of questions predicted to be answerable by the model.",
+                lower_is_better=False,
+                group=None,
+            ),
+            MetricMetadata(
+                name="ehr_sql_total_ground_truth_answerable",
+                display_name="Total Ground Truth Answerable",
+                short_display_name="Total GT Ans",
+                description="Total number of answerable questions in the ground truth.",
+                lower_is_better=False,
+                group=None,
+            ),
+        ]

helm/benchmark/metrics/evaluate_instances_metric.py

@@ -0,0 +1,59 @@
+from abc import ABC, abstractmethod
+from collections import defaultdict
+from typing import List, Dict
+from helm.benchmark.metrics.metric import MetricInterface, MetricResult, add_context
+
+
+from helm.benchmark.adaptation.scenario_state import ScenarioState
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.benchmark.metrics.metric_name import MetricName, MetricContext
+from helm.benchmark.metrics.metric_service import MetricService
+from helm.benchmark.metrics.statistic import Stat, merge_stat
+
+
+class EvaluateInstancesMetric(MetricInterface, ABC):
+    """
+    Metric that needs to examine all request states for all instances in the same split with the same perturbations
+    in order to determine the Stats.
+    """
+
+    def evaluate(
+        self, scenario_state: ScenarioState, metric_service: MetricService, eval_cache_path: str, parallelism: int
+    ) -> MetricResult:
+        """Aggregate over calls to evaluate_instances, which is defined by the subclass.
+
+        1. Each call has all instances for the same train trial, split, and perturbations.
+        2. For each train trial, take the mean for each Stat.
+        3. Returns Stats built from those means (e.g. the mean in the result is the mean-of-means).
+        """
+        adapter_spec = scenario_state.adapter_spec
+        global_stats: Dict[MetricName, Stat] = {}
+
+        for train_trial_index in range(adapter_spec.num_train_trials):
+
+            # Aggregate these stats
+            trial_stats: Dict[MetricName, Stat] = {}  # Statistics just for this trial
+
+            # Compute statistics that depend on all the `RequestStates` (e.g., bias metrics).
+            # Aggregate request states and call evaluate_instances in case the metric needs it.
+            grouped_request_states: Dict[MetricContext, List[RequestState]] = defaultdict(list)
+            for instance in scenario_state.instances:
+                # TODO: do we need to support reference_index that is not None?
+                grouped_request_states[MetricContext.from_instance(instance)].extend(
+                    scenario_state.get_request_states(train_trial_index, instance, None)
+                )
+            for context, request_states in grouped_request_states.items():
+                for stat in self.evaluate_instances(request_states, eval_cache_path):
+                    merge_stat(trial_stats, add_context(stat, context))
+
+            # We take the mean value for each trial.
+            for stat in trial_stats.values():
+                merge_stat(global_stats, stat.take_mean())
+
+        # Wrap aggregated and per-instance stats in a MetricResult.
+        return MetricResult(list(global_stats.values()), [])
+
+    @abstractmethod
+    def evaluate_instances(self, request_states: List[RequestState], eval_cache_path: str) -> List[Stat]:
+        """Evaluate all request states directly. Use only if nothing else works."""
+        pass