text2sql-eval-toolkit 1.0.0__py3-none-any.whl

text2sql_eval_toolkit/evaluation/evaluation_tools.py

@@ -0,0 +1,759 @@
+#
+# Copyright IBM Corp. 2025 - 2026
+# SPDX-License-Identifier: Apache-2.0
+#
+
+import asyncio
+import json
+import pandas as pd
+from pathlib import Path
+from tqdm.asyncio import tqdm_asyncio
+from unitxt.text2sql_utils import (
+    compare_result_dfs,
+    compare_dfs_bird_eval_logic,
+    is_sqlglot_parsable,
+    is_sqlparse_parsable,
+    sqlglot_parsed_queries_equivalent,
+    sqlglot_optimized_equivalence,
+    sqlparse_queries_equivalent,
+    sql_exact_match,
+)
+from text2sql_eval_toolkit.utils import (
+    get_benchmark_info,
+    parse_dataframe,
+    truncate_dataframe,
+    get_gt_sqls,
+    get_question,
+    get_default_eval_filename,
+    add_summary_json_suffix,
+    add_summary_csv_suffix,
+)
+from text2sql_eval_toolkit.evaluation.llm_as_judge import (
+    evaluate_sql_prediction_with_llm,
+    load_llm_judge_config,
+)
+from text2sql_eval_toolkit.logging import get_logger
+
+
+logger = get_logger(__name__)
+
+
+def evaluate_prediction(record, prediction, llm_judge_config=None, force_rerun_llm_judge=False):
+    """
+    Evaluates a predicted SQL query against one or more ground truth SQL queries and their corresponding result dataframes.
+
+    This function supports multiple ground truth SQLs per record. It iterates through each ground truth SQL and its
+    associated result dataframe, comparing them to the predicted SQL and its result dataframe. Evaluation stops early
+    if a perfect (subset/super) execution match (subset_non_empty_execution_accuracy == 1) is found.
+
+    Parameters
+    ----------
+    record : dict
+        A dictionary containing the ground truth SQL(s) and their corresponding result dataframe(s).
+        Expected keys:
+            - "sql": str or List[str]
+                One or more ground truth SQL queries.
+            - "gt_df": dict or List[dict]
+                One or more serialized dataframes corresponding to the ground truth SQL queries.
+
+    prediction : dict
+        A dictionary containing the predicted SQL and its result dataframe.
+        Expected keys:
+            - "predicted_sql": str
+                The SQL query generated by the model.
+            - "predicted_df": dict
+                The serialized dataframe resulting from executing the predicted SQL.
+            - "sql_execution_error" (optional): str
+                An error message if the predicted SQL failed to execute.
+            - "evaluation" (optional): dict
+                Existing evaluation results. If present and contains valid LLM judge results,
+                they will be reused unless force_rerun_llm_judge is True.
+
+    llm_judge_config : dict, optional
+        dictionary config object loaded from the YAML configuration file containing model parameters
+        and prompt template for LLM-based evaluation. If not provided, LLM judge will not be used.
+
+    force_rerun_llm_judge : bool, optional
+        If True, forces re-evaluation with LLM judge even if cached results exist.
+        If False (default), reuses existing LLM judge results when available.
+
+    Returns
+    -------
+    result : dict
+        A dictionary containing evaluation metrics and flags. Keys include:
+            - "execution_accuracy": int
+                Whether the predicted result matches the ground truth result exactly.
+            - "non_empty_execution_accuracy": int
+                Whether the predicted result matches the ground truth result and is non-empty.
+            - "subset_non_empty_execution_accuracy": int
+                Whether the predicted result is a non-empty subset or superset of the ground truth result.
+            - "logic_execution_accuracy": int
+                Execution accuracy of SQL logic if record as logic_df
+                (result of running query with SELECT clause replaced with gt's SELECT clause).
+            - "bird_execution_accuracy": int
+                A relaxed match score based on BIRD evaluation logic.
+            - "llm_score" (optional): float
+                If llm_judge_config is provided, the score using LLM as judge
+            - "is_sqlglot_parsable": int
+                Whether the predicted SQL is parsable by SQLGlot.
+            - "is_sqlparse_parsable": int
+                Whether the predicted SQL is parsable by sqlparse.
+            - "sqlglot_equivalence": int
+                Whether the predicted SQL is equivalent to the ground truth SQL using SQLGlot parsing.
+            - "sqlglot_optimized_equivalence": int
+                Whether the predicted SQL is equivalent to the ground truth SQL using SQLGlot optimization.
+            - "sqlparse_equivalence": int
+                Whether the predicted SQL is equivalent to the ground truth SQL using sqlparse.
+            - "sql_exact_match": int
+                Whether the predicted SQL exactly matches the ground truth SQL string.
+            - "sql_syntactic_equivalence": int
+                Whether any of the syntactic equivalence checks passed.
+            - "df_error": int
+                Indicates if there was an error parsing the predicted dataframe.
+            - "df_error_message" (optional): str
+                Error message if dataframe parsing failed.
+            - "eval_error": int
+                Indicates if there was an error during evaluation.
+            - "eval_error_message" (optional): str
+                Error message if evaluation failed.
+            - "llm_explanation" (optional): str
+                If llm_judge_config is provided, LLM judge explanation of the accuracy of the prediction
+            - "gt_sql" (optional): str
+                The ground truth SQL query that was used for final evaluation, only present
+                if subset_non_empty_execution_accuracy == 1.
+            - "gt_df" (optional): DataFrame
+                The parsed ground truth dataframe that was used for final evaluation, only present
+                if subset_non_empty_execution_accuracy == 1.
+
+    Notes
+    -----
+    - If the predicted dataframe cannot be parsed, the function returns early with a dataframe error.
+    - If multiple ground truth SQLs are provided, the function evaluates them in order and stops at the first
+      one that results in a perfect execution match.
+    - The function uses several SQL equivalence and result comparison methods to assess prediction quality.
+    - The final result reflects the evaluation against the first ground truth SQL that yields
+      subset_non_empty_execution_accuracy == 1, or the last one evaluated if no perfect match is found.
+    - The "gt_sql" and "gt_df" fields are only included in the result if a perfect execution match is found.
+    - LLM judge caching: If the prediction already has an "evaluation" dict with valid "llm_score" and
+      "llm_explanation" fields (and no "llm_judge_error"), those cached results will be reused unless
+      force_rerun_llm_judge is True. This significantly improves performance when re-evaluating the same data.
+    """
+    result = {}
+    
+    # Check for inference error - skip evaluation if inference failed
+    if "inference_error" in prediction:
+        return {
+            "execution_accuracy": 0,
+            "non_empty_execution_accuracy": 0,
+            "subset_non_empty_execution_accuracy": 0,
+            "logic_execution_accuracy": 0,
+            "bird_execution_accuracy": 0,
+            "is_sqlglot_parsable": 0,
+            "is_sqlparse_parsable": 0,
+            "sqlglot_equivalence": 0,
+            "sqlglot_optimized_equivalence": 0,
+            "sqlparse_equivalence": 0,
+            "sql_exact_match": 0,
+            "sql_syntactic_equivalence": 0,
+            "df_error": 1,
+            "df_error_message": f"Inference failed: {prediction['inference_error']}",
+            "eval_error": 0,
+        }
+    
+    pred_df = None
+    predicted_sql = prediction["predicted_sql"]
+
+    try:
+        pred_df = parse_dataframe(prediction["predicted_df"])
+        result["df_error"] = 0
+    except Exception as e:
+        result["df_error"] = 1
+        result["df_error_message"] = prediction.get("sql_execution_error", str(e))
+
+    try:
+        gold_sqls = get_gt_sqls(record)
+        gold_dfs = record["gt_df"]
+
+        if not isinstance(gold_dfs, list):
+            gold_dfs = [gold_dfs]
+
+        for gold_sql, gold_df_raw in zip(gold_sqls, gold_dfs):
+            gold_df = parse_dataframe(gold_df_raw)
+
+            match, non_empty_match, subset_match = (
+                compare_result_dfs(gold_df, pred_df, gold_sql)
+                if gold_sql and pred_df is not None
+                else (0, 0, 0)
+            )
+            bird_match = (
+                compare_dfs_bird_eval_logic(gold_df, pred_df)
+                if gold_sql and pred_df is not None
+                else 0
+            )
+            logic_match = subset_match
+            if logic_match == 0:
+                logic_df_raw = prediction.get("logic_df")
+                if logic_df_raw is not None:
+                    logic_df = parse_dataframe(logic_df_raw)
+                    _, logic_match, _ = (
+                        compare_result_dfs(gold_df, logic_df, gold_sql)
+                        if gold_sql and logic_df is not None
+                        else (0, 0, 0)
+                    )
+
+            is_glot_parsable = is_sqlglot_parsable(predicted_sql)
+            sqlparse_parsable = is_sqlparse_parsable(predicted_sql)
+            sqlglot_equivalence_score = (
+                sqlglot_parsed_queries_equivalent(predicted_sql, gold_sql)
+                if is_glot_parsable
+                else 0
+            )
+            sqlglot_optimized_equivalence_score = (
+                sqlglot_optimized_equivalence(predicted_sql, gold_sql)
+                if is_glot_parsable
+                else 0
+            )
+            sqlparse_equivalance = (
+                sqlparse_queries_equivalent(predicted_sql, gold_sql)
+                if sqlparse_parsable
+                else 0
+            )
+            sql_exact_match_score = sql_exact_match(predicted_sql, gold_sql)
+
+            result.update(
+                {
+                    "execution_accuracy": int(match),
+                    "non_empty_execution_accuracy": int(non_empty_match),
+                    "subset_non_empty_execution_accuracy": int(subset_match),
+                    "logic_execution_accuracy": int(logic_match),
+                    "bird_execution_accuracy": int(bird_match),
+                    "is_sqlglot_parsable": int(is_glot_parsable),
+                    "is_sqlparse_parsable": int(sqlparse_parsable),
+                    "sqlglot_equivalence": int(sqlglot_equivalence_score),
+                    "sqlglot_optimized_equivalence": int(
+                        sqlglot_optimized_equivalence_score
+                    ),
+                    "sqlparse_equivalence": int(sqlparse_equivalance),
+                    "sql_exact_match": int(sql_exact_match_score),
+                    "sql_syntactic_equivalence": int(
+                        any(
+                            [
+                                sqlglot_equivalence_score,
+                                sqlglot_optimized_equivalence_score,
+                                sqlparse_equivalance,
+                                sql_exact_match_score,
+                            ]
+                        )
+                    ),
+                    "eval_error": 0,
+                }
+            )
+            result["df_error"] = result.pop("df_error")
+            
+            # Add token usage metrics from prediction to evaluation result
+            token_usage = prediction.get("token_usage")
+            if token_usage:
+                result["prompt_tokens"] = token_usage.get("prompt_tokens", 0)
+                result["completion_tokens"] = token_usage.get("completion_tokens", 0)
+                result["total_tokens"] = token_usage.get("total_tokens", 0)
+            
+            # Add timing metrics from prediction to evaluation result
+            inference_time = prediction.get("inference_time_ms")
+            if inference_time is not None:
+                result["inference_time_ms"] = inference_time
+            execution_time = prediction.get("execution_time_ms")
+            if execution_time is not None:
+                result["execution_time_ms"] = execution_time
+
+            if llm_judge_config:
+                try:
+                    llm_score = None
+                    llm_explanation = None
+                    
+                    # Check if we can reuse existing LLM judge results
+                    use_cached_results = False
+                    if not force_rerun_llm_judge:
+                        existing_eval = prediction.get("evaluation", {})
+                        if (
+                            "llm_score" in existing_eval
+                            and "llm_explanation" in existing_eval
+                            and "llm_judge_error" not in existing_eval
+                        ):
+                            # Validate that llm_score is a valid number
+                            try:
+                                cached_score = float(existing_eval["llm_score"])
+                                llm_score = cached_score
+                                llm_explanation = existing_eval["llm_explanation"]
+                                use_cached_results = True
+                                logger.info(
+                                    f"Reusing cached LLM judge results (score: {llm_score})"
+                                )
+                            except (ValueError, TypeError):
+                                logger.warning(
+                                    "Invalid cached llm_score, will re-run LLM judge"
+                                )
+                    
+                    if not use_cached_results:
+                        if pred_df is None:
+                            llm_score = 0.0
+                            llm_explanation = (
+                                "N/A (did not use LLM due to missing prediction dataframe)"
+                            )
+                        elif subset_match:
+                            llm_score = 1.0
+                            llm_explanation = "N/A (did not use LLM due to subset match)"
+                        else:
+                            question = get_question(record)
+                            ground_truth_sql = record["sql"]
+                            ground_truth_df = truncate_dataframe(gold_df)
+                            predicted_sql = prediction["predicted_sql"]
+                            predicted_df = truncate_dataframe(pred_df)
+
+                            # Get context for LLM judge
+                            # For agentic pipelines: use agent_trace (full conversation history)
+                            # For standard baseline: use prompt
+                            if "agent_trace" in prediction and prediction["agent_trace"]:
+                                # Agentic pipeline - use full trace as context
+                                trace = prediction["agent_trace"]
+                                trace_text = "Agent Interaction Trace:\n\n"
+                                for i, interaction in enumerate(trace, 1):
+                                    if interaction is None:
+                                        continue
+                                    trace_text += (
+                                        f"Step {i}: {interaction.get('step', 'unknown')}\n"
+                                    )
+                                    if "messages" in interaction:
+                                        for msg in interaction["messages"]:
+                                            role = msg.get("role", "unknown")
+                                            content = msg.get("content", "")[
+                                                :500
+                                            ]  # Truncate long content
+                                            trace_text += f"  [{role}]: {content}...\n"
+                                    if "response" in interaction:
+                                        trace_text += f"  [response]: {interaction['response'][:500]}...\n"
+                                    trace_text += "\n"
+                                prompt = trace_text
+                            elif "agent_reasoning" in prediction:
+                                # Fallback to agent_reasoning if trace not available
+                                reasoning_list = prediction["agent_reasoning"]
+                                prompt = "Agent Reasoning:\n" + "\n".join(
+                                    f"- {r}" for r in reasoning_list
+                                )
+                            elif "prompt" in prediction:
+                                # Standard baseline - use prompt
+                                prompt = prediction["prompt"]
+                            else:
+                                # Fallback - construct minimal context
+                                schema_info = record.get("schema", {})
+                                db_type = record.get("db_type", "SQL")
+                                prompt = f"Question: {question}\n\nDatabase Type: {db_type}\n\nSchema: {schema_info}\n\nGenerate SQL to answer the question."
+
+                            llm_as_judge_response = evaluate_sql_prediction_with_llm(
+                                question,
+                                ground_truth_sql,
+                                ground_truth_df,
+                                predicted_sql,
+                                predicted_df,
+                                prompt,
+                                llm_judge_config,
+                            )
+                            llm_score = float(llm_as_judge_response["score"])
+                            llm_explanation = llm_as_judge_response["explanation"]
+                    result["llm_score"] = llm_score
+                    result["llm_explanation"] = llm_explanation
+                except Exception as e:
+                    logger.error(f"LLM judge error: {repr(e)}")
+                    result["llm_judge_error"] = repr(e)
+
+            if result["subset_non_empty_execution_accuracy"] == 1:
+                result["gt_sql"] = gold_sql
+                result["gt_df"] = gold_df_raw
+                break
+
+    except Exception as e:
+        result["eval_error"] = 1
+        result["eval_error_message"] = repr(e)
+        # raise e
+
+    return result
+
+
+def compute_summary(metrics_by_model, llm_judge_config, token_usage_by_model=None):
+    summary = {}
+    for model, records in metrics_by_model.items():
+        num_records = len(records)
+        num_eval_errors = sum(1 for r in records if "eval_error_message" in r)
+        num_df_errors = sum(1 for r in records if "df_error_message" in r)
+        # Count records with inference errors (failed to generate SQL)
+        num_inference_errors = sum(
+            1 for r in records
+            if "df_error_message" in r and "Inference failed" in (r.get("df_error_message") or "")
+        )
+        # Count records with successful predictions (SQL was generated)
+        num_predictions = num_records - num_inference_errors
+        num_evaluated = num_records - num_eval_errors
+        num_correct_non_empty_execution_accuracy = sum(
+            r["non_empty_execution_accuracy"]
+            for r in records
+            if "eval_error_message" not in r
+        )
+        num_correct_subset_non_empty_execution_accuracy = sum(
+            r["subset_non_empty_execution_accuracy"]
+            for r in records
+            if "eval_error_message" not in r
+        )
+
+        df = None
+        if num_evaluated > 0:
+            df = pd.DataFrame([r for r in records if "eval_error_message" not in r])
+            # Calculate metrics based on num_records (total benchmark size) instead of num_evaluated
+            # This ensures that failures to generate predictions or evaluation errors count as 0
+            metric_stats = {}
+            for metric in df.columns:
+                if metric not in [
+                    "eval_error_message",
+                    "df_error_message",
+                    "llm_judge_error",
+                    "llm_explanation",
+                    "gt_sql",
+                    "gt_df",
+                ]:
+                    # For accuracy metrics, divide by num_records (not num_evaluated)
+                    # This penalizes pipelines that fail to generate predictions
+                    metric_sum = df[metric].sum()
+                    metric_stats[metric] = {
+                        "average": metric_sum / num_records,  # Changed from df[metric].mean()
+                        "stddev": df[metric].std()
+                    }
+            
+            # Token metrics are automatically calculated by pandas from the evaluation records
+            # The statistics (average, stddev) are already in metric_stats from lines above
+            # We just need to add the total sums as separate count metrics
+            if "total_tokens" in df.columns:
+                metric_stats["sum_total_tokens"] = int(df["total_tokens"].sum())
+                metric_stats["sum_prompt_tokens"] = int(df["prompt_tokens"].sum())
+                metric_stats["sum_completion_tokens"] = int(df["completion_tokens"].sum())
+            
+            # Timing metrics - add total sums
+            if "inference_time_ms" in df.columns:
+                metric_stats["sum_inference_time_ms"] = round(df["inference_time_ms"].sum(), 2)
+            if "execution_time_ms" in df.columns:
+                metric_stats["sum_execution_time_ms"] = round(df["execution_time_ms"].sum(), 2)
+        else:
+            metric_stats = {}
+
+        metric_stats["num_records"] = num_records
+        metric_stats["num_predictions"] = num_predictions
+        metric_stats["num_evaluated"] = num_evaluated
+        metric_stats["num_eval_errors"] = num_eval_errors
+        metric_stats["num_df_errors"] = num_df_errors
+        metric_stats["num_inference_errors"] = num_inference_errors
+        metric_stats["num_correct_non_empty_execution_accuracy"] = (
+            num_correct_non_empty_execution_accuracy
+        )
+        metric_stats["num_correct_subset_non_empty_execution_accuracy"] = (
+            num_correct_subset_non_empty_execution_accuracy
+        )
+        
+        if llm_judge_config:
+            metric_stats["num_correct_llm"] = sum(
+                1
+                for r in records
+                if "llm_judge_error" not in r
+                and "eval_error_message" not in r
+                and r.get("llm_score") == 1
+            )
+            metric_stats["num_llm_judge_errors"] = sum(
+                1 for r in records if "llm_judge_error" in r
+            )
+            if "llm_judge_config" not in summary:
+                summary["llm_judge_config"] = llm_judge_config
+
+        summary[model] = metric_stats
+
+    return summary
+
+
+def summary_to_df_csv(summary, output_path, use_llm):
+    rows = []
+    for model, metrics in summary.items():
+        if model == "llm_judge_config":
+            continue
+        row = {
+            "Model": model,
+            "Total": metrics.get("num_records", 0),
+            "Evaluated": metrics.get("num_evaluated", 0),
+            "Number of Correct Non-Empty Data Frames": metrics.get(
+                "num_correct_non_empty_execution_accuracy"
+            ),
+            "Number of Correct Subset/Superset Non-Empty Data Frames": metrics.get(
+                "num_correct_subset_non_empty_execution_accuracy"
+            ),
+            "Number of Correct Results According to LLM Judge": (
+                metrics["num_correct_llm"] if use_llm else "N/A"
+            ),
+            "Evaluation Errors": metrics.get("num_eval_errors", 0),
+            "Dataframe Errors": metrics.get("num_df_errors", 0),
+            "LLM Judge Errors": metrics.get("num_llm_judge_errors", 0),
+            "Total Tokens": metrics.get("sum_total_tokens", "N/A"),
+            "Avg Tokens/Question": (
+                round(metrics.get("total_tokens", {}).get("average", 0), 2)
+                if isinstance(metrics.get("total_tokens"), dict)
+                else "N/A"
+            ),
+            "Total Prompt Tokens": metrics.get("sum_prompt_tokens", "N/A"),
+            "Total Completion Tokens": metrics.get("sum_completion_tokens", "N/A"),
+            "Total Inference Time (ms)": metrics.get("sum_inference_time_ms", "N/A"),
+            "Avg Inference Time (ms)": (
+                round(metrics.get("inference_time_ms", {}).get("average", 0), 2)
+                if isinstance(metrics.get("inference_time_ms"), dict)
+                else "N/A"
+            ),
+            "Total Execution Time (ms)": metrics.get("sum_execution_time_ms", "N/A"),
+            "Avg Execution Time (ms)": (
+                round(metrics.get("execution_time_ms", {}).get("average", 0), 2)
+                if isinstance(metrics.get("execution_time_ms"), dict)
+                else "N/A"
+            ),
+        }
+
+        for metric, stats in metrics.items():
+            if isinstance(stats, dict):
+                row[f"{metric}_avg"] = round(stats.get("average", 0), 4)
+                row[f"{metric}_std"] = round(stats.get("stddev", 0), 4)
+
+        rows.append(row)
+
+    df = pd.DataFrame(rows)
+
+    sort_col = "subset_non_empty_execution_accuracy_avg"
+    if sort_col in df.columns:
+        df.sort_values(by=sort_col, ascending=False, inplace=True)
+
+    df.to_csv(output_path, index=False)
+    logger.info(f"\nSummary written to: {output_path}")
+    return df
+
+
+def print_summary(summary, use_llm):
+    print("\n=== Evaluation Summary ===")
+    for pipeline, metrics in summary.items():
+        if pipeline == "llm_judge_config":
+            continue
+        print(f"\n: {pipeline}")
+        num_records = metrics.get("num_records", 0)
+        num_evaluated = metrics.get("num_evaluated", 0)
+        num_eval_errors = metrics.get("num_eval_errors", 0)
+        num_df_errors = metrics.get("num_df_errors", 0)
+        num_correct_non_empty_execution_accuracy = metrics.get(
+            "num_correct_non_empty_execution_accuracy"
+        )
+        num_correct_subset_non_empty_execution_accuracy = metrics.get(
+            "num_correct_subset_non_empty_execution_accuracy"
+        )
+        print(f"  Total Records       : {num_records}")
+        print(f"  Successfully Evaluated: {num_evaluated}")
+        print(
+            f"  Number of Correct Non-Empty Data Frames: {num_correct_non_empty_execution_accuracy}"
+        )
+        print(
+            f"  Number of Correct Subset/Superset Non-Empty Data Frames: {num_correct_subset_non_empty_execution_accuracy}"
+        )
+        if use_llm:
+            print(
+                f"  Number of Correct Results According to LLM Judge: {metrics.get('num_correct_llm')}"
+            )
+            print(
+                f"  Number of LLM Judge errors: {metrics.get('num_llm_judge_errors')}"
+            )
+        print(f"  Evaluation Errors              : {num_eval_errors}")
+        print(f"  Dataframe Errors              : {num_df_errors}")
+        
+        # Print token usage metrics if available
+        if "sum_total_tokens" in metrics:
+            print(f"  Token Usage Metrics:")
+            print(f"    Total Tokens                 : {metrics.get('sum_total_tokens', 0):,}")
+            total_tokens_stats = metrics.get('total_tokens', {})
+            if isinstance(total_tokens_stats, dict):
+                avg_val = total_tokens_stats.get('average', 0)
+            else:
+                avg_val = 0
+            print(f"    Avg Tokens per Question      : {avg_val:.2f}")
+            print(f"    Total Prompt Tokens          : {metrics.get('sum_prompt_tokens', 0):,}")
+            print(f"    Total Completion Tokens      : {metrics.get('sum_completion_tokens', 0):,}")
+        
+        # Print timing metrics if available
+        if "sum_inference_time_ms" in metrics or "sum_execution_time_ms" in metrics:
+            print(f"  Performance Metrics:")
+            if "sum_inference_time_ms" in metrics:
+                inference_stats = metrics.get('inference_time_ms', {})
+                if isinstance(inference_stats, dict):
+                    avg_inference = inference_stats.get('average', 0)
+                else:
+                    avg_inference = 0
+                print(f"    Total Inference Time         : {metrics.get('sum_inference_time_ms', 0):,.2f} ms")
+                print(f"    Avg Inference Time per Query : {avg_inference:.2f} ms")
+            
+            if "sum_execution_time_ms" in metrics:
+                execution_stats = metrics.get('execution_time_ms', {})
+                if isinstance(execution_stats, dict):
+                    avg_execution = execution_stats.get('average', 0)
+                else:
+                    avg_execution = 0
+                print(f"    Total Execution Time         : {metrics.get('sum_execution_time_ms', 0):,.2f} ms")
+                print(f"    Avg Execution Time per Query : {avg_execution:.2f} ms")
+        
+        for metric, stats in metrics.items():
+            if metric in {
+                "num_records",
+                "num_predictions",
+                "num_evaluated",
+                "num_eval_errors",
+                "num_df_errors",
+                "num_inference_errors",
+                "num_correct_non_empty_execution_accuracy",
+                "num_correct_subset_non_empty_execution_accuracy",
+                "num_correct_llm",
+                "num_llm_judge_errors",
+                "sum_total_tokens",
+                "sum_prompt_tokens",
+                "sum_completion_tokens",
+                "sum_inference_time_ms",
+                "sum_execution_time_ms",
+                "inference_time_ms",
+                "execution_time_ms",
+            }:
+                continue
+            print(
+                f"  {metric:<30} Avg: {stats['average']:.4f}  StdDev: {stats['stddev']:.4f}"
+            )
+
+
+async def async_evaluate_predictions(
+    input_file: str,
+    output_file: str = None,
+    summary_file: str = None,
+    csv_summary_file: str = None,
+    llm_judge_config: dict = None,
+    max_concurrency: int = 16,
+    force_rerun_llm_judge: bool = False,
+    force_rerun: bool = False,
+):
+    output_file = output_file or get_default_eval_filename(input_file)
+    summary_file = summary_file or add_summary_json_suffix(output_file)
+    csv_summary_file = csv_summary_file or add_summary_csv_suffix(output_file)
+
+    semaphore = asyncio.Semaphore(max_concurrency)
+
+    async def worker(record, prediction, llm_judge_config, force_rerun_llm_judge):
+        async with semaphore:
+            return await asyncio.to_thread(
+                evaluate_prediction, record, prediction, llm_judge_config, force_rerun_llm_judge
+            )
+
+    with open(input_file, "r") as f:
+        data = json.load(f)
+    
+    # Load existing evaluations from output file if it exists (for caching)
+    existing_evaluations = {}
+    if not force_rerun and Path(output_file).exists():
+        try:
+            with open(output_file, "r") as f:
+                existing_data = json.load(f)
+                for record in existing_data:
+                    record_id = record.get("id") or record.get("question_id")
+                    if record_id:
+                        existing_evaluations[record_id] = record.get("predictions", {})
+        except Exception as e:
+            logger.warning(f"Could not load existing evaluations from {output_file}: {e}")
+
+    # Copy existing evaluations to predictions for caching
+    if not force_rerun:
+        for record in data:
+            record_id = record.get("id") or record.get("question_id")
+            if record_id and record_id in existing_evaluations:
+                predictions = record.get("predictions", {})
+                for model_name, prediction in predictions.items():
+                    if model_name in existing_evaluations[record_id]:
+                        existing_eval = existing_evaluations[record_id][model_name].get("evaluation", {})
+                        if existing_eval:
+                            prediction["evaluation"] = existing_eval
+
+    tasks = []
+    prediction_references = []
+    for record in data:
+        predictions = record.get("predictions", {})
+        for model_name, prediction in predictions.items():
+            task = worker(record, prediction, llm_judge_config, force_rerun_llm_judge)
+            tasks.append(task)
+            prediction_references.append((record, model_name, prediction))
+
+    evaluations = await tqdm_asyncio.gather(
+        *tasks, desc=f"Evaluating (concurrency limit: {max_concurrency})"
+    )
+
+    metrics_by_model = {}
+    token_usage_by_model = {}
+    for i, evaluation in enumerate(evaluations):
+        record, model_name, prediction = prediction_references[i]
+        prediction["evaluation"] = evaluation
+
+        if model_name not in metrics_by_model:
+            metrics_by_model[model_name] = []
+            token_usage_by_model[model_name] = []
+        metrics_by_model[model_name].append(evaluation)
+        
+        # Collect token usage from prediction
+        token_usage = prediction.get("token_usage")
+        if token_usage:
+            token_usage_by_model[model_name].append(token_usage)
+
+    summary = compute_summary(metrics_by_model, llm_judge_config, token_usage_by_model)
+
+    with open(output_file, "w") as f:
+        json.dump(data, f, indent=2, ensure_ascii=False)
+
+    with open(summary_file, "w") as f:
+        json.dump(summary, f, indent=2, ensure_ascii=False)
+
+    use_llm = True if llm_judge_config is not None else False
+    summary_df = summary_to_df_csv(summary, csv_summary_file, use_llm)
+    print_summary(summary, use_llm)
+
+    return data, summary_df
+
+
+def evaluate_predictions(
+    input_file: str,
+    output_file: str = None,
+    summary_file: str = None,
+    csv_summary_file: str = None,
+    use_llm: bool = False,
+    llm_judge_config_path: str = None,
+    force_rerun_llm_judge: bool = False,
+    force_rerun: bool = False,
+):
+    llm_judge_config = None
+    if use_llm or llm_judge_config_path is not None:
+        llm_judge_config = load_llm_judge_config(llm_judge_config_path)
+    return asyncio.run(
+        async_evaluate_predictions(
+            input_file, output_file, summary_file, csv_summary_file, llm_judge_config,
+            force_rerun_llm_judge=force_rerun_llm_judge,
+            force_rerun=force_rerun
+        )
+    )
+
+
+# For running from script
+def run_evaluation(
+    benchmark_id: str, use_llm: bool = False, llm_judge_config_path: str = None,
+    force_rerun_llm_judge: bool = False, force_rerun: bool = False
+):
+    benchmark_info = get_benchmark_info(benchmark_id)
+    predictions_path = str(Path(benchmark_info["predictions_path"]))
+    return evaluate_predictions(
+        predictions_path, use_llm=use_llm, llm_judge_config_path=llm_judge_config_path,
+        force_rerun_llm_judge=force_rerun_llm_judge or force_rerun,
+        force_rerun=force_rerun
+    )