crfm-helm 0.4.0__py3-none-any.whl → 0.5.10__py3-none-any.whl

helm/benchmark/presentation/table.py

@@ -4,26 +4,26 @@ from typing import Any, Optional, List, Dict
 
 @dataclass(frozen=True)
 class Cell:
-    # Semantic value (that can be used for sorting)
     value: Any
+    """Semantic value (that can be used for sorting)"""
 
-    # Optionally, if we want to render things specially (floating points to 3 decimal points)
     display_value: Optional[str] = None
+    """Optionally, if we want to render things specially (floating points to 3 decimal points)"""
 
-    # Detailed description if hover over the cell
     description: Optional[str] = None
+    """Detailed description if hover over the cell"""
 
-    # If we click on the link for this cell, it takes us somewhere
     href: Optional[str] = None
+    """If we click on the link for this cell, it takes us somewhere"""
 
-    # Styling
     style: Optional[Dict[str, Any]] = None
+    """Styling"""
 
-    # If the value or display_value is markdown that needs to be interpreted
     markdown: bool = False
+    """If the value or display_value is markdown that needs to be interpreted"""
 
-    # How much train-test contamination affects the cell's value (`contamination.CONTAMINATION_LEVEL_{WEAK/STRONG}`)
-    contamination_level: Optional[str] = None
+    run_spec_names: Optional[List[str]] = None
+    """The names of the runs that this cell's value was aggregated from, if the cell contains an aggregate value."""
 
 
 @dataclass(frozen=True)

helm/benchmark/presentation/taxonomy_info.py

@@ -0,0 +1,20 @@
+from dataclasses import dataclass
+from typing import Optional
+
+
+@dataclass(frozen=True)
+class TaxonomyInfo:
+    # Task (e.g., question answering)
+    task: Optional[str] = None
+
+    # Domain - genre (e.g., Wikipedia)
+    what: Optional[str] = None
+
+    # Domain - when it was written (e.g., 2010s)
+    when: Optional[str] = None
+
+    # Domain - demographics (e.g., web users)
+    who: Optional[str] = None
+
+    # Language (e.g., English)
+    language: Optional[str] = None

helm/benchmark/presentation/test_contamination.py

@@ -1,9 +1,9 @@
-from helm.benchmark.presentation.schema import read_schema, SCHEMA_CLASSIC_YAML_FILENAME
+from helm.benchmark.presentation.schema import read_schema, get_default_schema_path
 from helm.benchmark.presentation.contamination import read_contamination, validate_contamination
 
 
 def test_contamination_schema():
-    schema = read_schema(SCHEMA_CLASSIC_YAML_FILENAME)
+    schema = read_schema(get_default_schema_path())
     contamination = read_contamination()
     validate_contamination(contamination, schema)
 

helm/benchmark/presentation/test_create_plots.py

@@ -1,4 +1,7 @@
-# mypy: check_untyped_defs = False
+# type: ignore
+# flake8: noqa
+# fmt: off
+
 from helm.common.general import asdict_without_nones
 from helm.benchmark.presentation.table import Table, Cell, HeaderCell
 from helm.benchmark.presentation.create_plots import parse_table

helm/benchmark/presentation/test_run_entry.py

@@ -3,8 +3,7 @@ import pytest
 
 from helm.common.object_spec import parse_object_spec
 from helm.benchmark.presentation.run_entry import read_run_entries
-from helm.benchmark.run_specs import construct_run_specs
-from helm.benchmark import vlm_run_specs  # noqa
+from helm.benchmark.run_spec_factory import construct_run_specs
 
 
 def list_fnames():
@@ -17,6 +16,7 @@ class TestRunEntry:
 
     @pytest.mark.parametrize("fname", list_fnames())
     def test_read_all_specs(self, fname: str):
+        pytest.skip("Skipping slow tests")
         run_entries = read_run_entries([fname])
         for entry in run_entries.entries:
             construct_run_specs(parse_object_spec(entry.description))

helm/benchmark/presentation/test_schema.py

@@ -0,0 +1,11 @@
+from helm.benchmark.presentation.schema import get_adapter_fields
+
+
+def test_get_adapter_fields() -> None:
+    adapter_fields = get_adapter_fields()
+    assert adapter_fields
+    assert adapter_fields[0].name == "method"
+    assert (
+        adapter_fields[0].description
+        == "The high-level strategy for converting instances into a prompt for the language model."
+    )

helm/benchmark/presentation/test_summarize.py

@@ -1,8 +1,9 @@
 import os
 import tempfile
 
-from helm.benchmark.presentation.summarize import Summarizer
-from helm.benchmark.presentation.schema import SCHEMA_CLASSIC_YAML_FILENAME
+from helm.benchmark.presentation.summarize import Summarizer, compute_aggregate_row_win_rates
+from helm.benchmark.presentation.schema import get_default_schema_path
+from helm.benchmark.presentation.table import Cell, HeaderCell, Table
 from helm.common.general import ensure_directory_exists
 
 
@@ -13,13 +14,13 @@ def test_summarize_suite():
             release=None,
             suites=None,
             suite="test_suite",
-            schema_file=SCHEMA_CLASSIC_YAML_FILENAME,
+            schema_path=get_default_schema_path(),
             output_path=output_path,
             verbose=False,
             num_threads=4,
             allow_unknown_models=True,
         )
-        summarizer.run_pipeline(skip_completed=True, num_instances=1000)
+        summarizer.run_pipeline(skip_completed=True)
         assert os.path.isfile(os.path.join(output_path, "runs", "test_suite", "groups.json"))
 
 
@@ -31,11 +32,152 @@ def test_summarize_release():
             release="test_release",
             suites=["test_suite_1", "test_suite_2"],
             suite=None,
-            schema_file=SCHEMA_CLASSIC_YAML_FILENAME,
+            schema_path=get_default_schema_path(),
             output_path=output_path,
             verbose=False,
             num_threads=4,
             allow_unknown_models=True,
         )
-        summarizer.run_pipeline(skip_completed=True, num_instances=1000)
+        summarizer.run_pipeline(skip_completed=True)
         assert os.path.isfile(os.path.join(output_path, "releases", "test_release", "groups.json"))
+
+
+def test_compute_win_rates_one_scenario():
+    header = [
+        HeaderCell(value="Model"),
+        HeaderCell(value="Scenario A", lower_is_better=False),
+    ]
+    values = [
+        ["Model A", 1],
+        ["Model B", 2],
+        ["Model C", 3],
+        ["Model D", 4],
+        ["Model E", 5],
+    ]
+    rows = [[Cell(value) for value in row_values] for row_values in values]
+    table = Table(title="Test Table", header=header, rows=rows)
+    assert compute_aggregate_row_win_rates(table) == [0, 0.25, 0.5, 0.75, 1]
+
+
+def test_compute_win_rates_two_scenarios():
+    header = [
+        HeaderCell(value="Model"),
+        HeaderCell(value="Scenario A", lower_is_better=False),
+        HeaderCell(value="Scenario B", lower_is_better=False),
+    ]
+    values = [
+        ["Model A", 1, 3],
+        ["Model B", 2, 1],
+        ["Model C", 3, 2],
+        ["Model D", 4, 5],
+        ["Model E", 5, 4],
+    ]
+    rows = [[Cell(value) for value in row_values] for row_values in values]
+    table = Table(title="Test Table", header=header, rows=rows)
+    assert compute_aggregate_row_win_rates(table) == [0.25, 0.125, 0.375, 0.875, 0.875]
+
+
+def test_compute_win_rates_incomplete_values():
+    header = [
+        HeaderCell(value="Model"),
+        HeaderCell(value="Scenario A", lower_is_better=False),
+        HeaderCell(value="Scenario B", lower_is_better=False),
+    ]
+    values = [
+        ["Model A", 1, 3],
+        ["Model B", 2, 1],
+        ["Model C", 3, None],
+        ["Model D", 4, None],
+        ["Model E", 5, None],
+    ]
+    rows = [[Cell(value) for value in row_values] for row_values in values]
+    table = Table(title="Test Table", header=header, rows=rows)
+    assert compute_aggregate_row_win_rates(table) == [0.5, 0.125, 0.5, 0.75, 1]
+
+
+def test_compute_win_rates_ignore_nones():
+    header = [
+        HeaderCell(value="Model"),
+        HeaderCell(value="Scenario A", lower_is_better=False),
+        HeaderCell(value="Scenario B", lower_is_better=False),
+        HeaderCell(value="Scenario C", lower_is_better=False),
+    ]
+    values = [
+        ["Model A", 1, None, None],
+        ["Model B", 2, None, 1],
+        ["Model C", 3, None, None],
+        ["Model D", 4, None, None],
+        ["Model E", 5, None, None],
+    ]
+    rows = [[Cell(value) for value in row_values] for row_values in values]
+    table = Table(title="Test Table", header=header, rows=rows)
+    assert compute_aggregate_row_win_rates(table) == [0, 0.25, 0.5, 0.75, 1]
+
+
+def test_compute_win_rates_ignore_unset_lower_is_better():
+    header = [
+        HeaderCell(value="Model"),
+        HeaderCell(value="Scenario A", lower_is_better=False),
+        HeaderCell(value="Scenario B"),
+    ]
+    values = [
+        ["Model A", 1, 3],
+        ["Model B", 2, 1],
+        ["Model C", 3, 2],
+        ["Model D", 4, 5],
+        ["Model E", 5, 4],
+    ]
+    rows = [[Cell(value) for value in row_values] for row_values in values]
+    table = Table(title="Test Table", header=header, rows=rows)
+    assert compute_aggregate_row_win_rates(table) == [0, 0.25, 0.5, 0.75, 1]
+
+
+def test_compute_win_rates_no_win_rate():
+    header = [
+        HeaderCell(value="Model"),
+        HeaderCell(value="Scenario A", lower_is_better=False),
+    ]
+    values = [
+        ["Model A", None],
+        ["Model B", None],
+        ["Model C", None],
+        ["Model D", None],
+        ["Model E", None],
+    ]
+    rows = [[Cell(value) for value in row_values] for row_values in values]
+    table = Table(title="Test Table", header=header, rows=rows)
+    assert compute_aggregate_row_win_rates(table) == [None, None, None, None, None]
+
+
+def test_compute_win_rates_ties():
+    header = [
+        HeaderCell(value="Model"),
+        HeaderCell(value="Scenario A", lower_is_better=False),
+    ]
+    values = [
+        ["Model A", 1],
+        ["Model B", 1],
+        ["Model C", 1],
+        ["Model D", 4],
+        ["Model E", 5],
+    ]
+    rows = [[Cell(value) for value in row_values] for row_values in values]
+    table = Table(title="Test Table", header=header, rows=rows)
+    assert compute_aggregate_row_win_rates(table) == [0.25, 0.25, 0.25, 0.75, 1.0]
+
+
+def test_compute_win_rates_lower_is_better():
+    header = [
+        HeaderCell(value="Model"),
+        HeaderCell(value="Scenario A", lower_is_better=True),
+    ]
+    values = [
+        ["Model A", 1],
+        ["Model B", 2],
+        ["Model C", 3],
+        ["Model D", 4],
+        ["Model E", 5],
+    ]
+    rows = [[Cell(value) for value in row_values] for row_values in values]
+    table = Table(title="Test Table", header=header, rows=rows)
+    assert compute_aggregate_row_win_rates(table) == [1, 0.75, 0.5, 0.25, 0]

helm/benchmark/presentation/torr_robustness_summarizer.py

@@ -0,0 +1,178 @@
+import os
+from typing import Dict, List, Optional
+
+from helm.benchmark.metrics.metric import PerInstanceStats
+from helm.benchmark.presentation.schema import MetricNameMatcher, RunGroup
+from helm.benchmark.presentation.summarize import Run, Summarizer
+from helm.benchmark.presentation.table import Cell
+from helm.common.codec import from_json
+
+
+class ToRRRobustnessSummarizer(Summarizer):
+    """A Summarizer that computes robustness metrics.
+
+    This Summarizer computes a robustness metrics based on the definition in the ToRR paper.
+    The instance-level robustness score for a given model and instance is defined as
+    1 - (max_score - min_score) where max_score and min_scores are the maximum and minimum
+    scores for the model on that instance across all runs (i.e. across all augmentations
+    and serialization formats). The robustness score for a given model and scenario is
+    the mean of the model's instance-level robustness score across all instances in that scenario.
+
+    The core HELM framework does not natively support computing metrics that depend on
+    per-instance metrics across multiple runs, therefore this special summarizer is needed
+    to compute this robustness metic."""
+
+    def __init__(
+        self,
+        release: Optional[str],
+        suites: Optional[List[str]],
+        suite: Optional[str],
+        schema_path: str,
+        output_path: str,
+        verbose: bool,
+        num_threads: int,
+        allow_unknown_models: bool,
+    ):
+        super().__init__(
+            release,
+            suites,
+            suite,
+            schema_path,
+            output_path,
+            verbose,
+            num_threads,
+            allow_unknown_models,
+        )
+        self.run_group_to_model_name_to_robustness: Dict[str, Dict[str, float]] = {}
+
+    PERFORMANCE_METRIC_GROUP_NAME = "performance_metrics"
+    ROBUSTNESS_METRIC_GROUP_NAME = "robustness_metrics"
+    ROBUSTNESS_METRIC_NAME = "robustness"
+
+    def _get_instance_id_to_performance(
+        self, run: Run, performance_metric_matcher: MetricNameMatcher
+    ) -> Dict[str, float]:
+        with open(os.path.join(run.run_path, "per_instance_stats.json")) as f:
+            per_instance_stats = from_json(f.read(), List[PerInstanceStats])
+        instance_id_to_performance: Dict[str, float] = {}
+        for per_instance_stats_item in per_instance_stats:
+            assert per_instance_stats_item.train_trial_index == 0
+            assert per_instance_stats_item.perturbation is None
+            for stat in per_instance_stats_item.stats:
+                if performance_metric_matcher.matches(stat.name):
+                    assert per_instance_stats_item.instance_id not in instance_id_to_performance
+                    if stat.mean is not None:
+                        instance_id_to_performance[per_instance_stats_item.instance_id] = stat.mean
+
+        return instance_id_to_performance
+
+    def _compute_robustness_for_runs(self, runs: List[Run], performance_metric_matcher: MetricNameMatcher) -> float:
+        instance_id_to_performances: Dict[str, List[float]] = {}
+        for run in runs:
+            for instance_id, performance in self._get_instance_id_to_performance(
+                run, performance_metric_matcher
+            ).items():
+                if instance_id not in instance_id_to_performances:
+                    instance_id_to_performances[instance_id] = []
+                instance_id_to_performances[instance_id].append(performance)
+        instance_id_to_robustness: Dict[str, float] = {}
+        for instance_id, performances in instance_id_to_performances.items():
+            instance_id_to_robustness[instance_id] = 1 - (max(performances) - min(performances))
+        return sum(instance_id_to_robustness.values()) / len(instance_id_to_robustness.values())
+
+    def _compute_robustness_for_run_group(self, run_group: RunGroup) -> Dict[str, float]:
+        performance_metric_group = self.schema.name_to_metric_group[self.PERFORMANCE_METRIC_GROUP_NAME]
+        assert len(performance_metric_group.metrics) == 1
+        performance_metric_matcher = performance_metric_group.metrics[0].substitute(run_group.environment)
+
+        group_runs = [run for run in self.runs if run_group.name in run.run_spec.groups]
+        model_name_to_runs: Dict[str, List[Run]] = {}
+
+        for run in group_runs:
+            model_name = run.run_spec.adapter_spec.model
+            if model_name not in model_name_to_runs:
+                model_name_to_runs[model_name] = []
+            model_name_to_runs[run.run_spec.adapter_spec.model].append(run)
+
+        model_to_robustness: Dict[str, float] = {}
+        for model_name, model_runs in model_name_to_runs.items():
+            model_to_robustness[model_name] = self._compute_robustness_for_runs(model_runs, performance_metric_matcher)
+        return model_to_robustness
+
+    def write_groups(self):
+        for run_group in self.schema.run_groups:
+            if self.ROBUSTNESS_METRIC_GROUP_NAME and self.PERFORMANCE_METRIC_GROUP_NAME in run_group.metric_groups:
+                self.run_group_to_model_name_to_robustness[run_group.name] = self._compute_robustness_for_run_group(
+                    run_group
+                )
+        return super().write_groups()
+
+    def create_cell(
+        self,
+        runs: List[Run],
+        matcher: MetricNameMatcher,
+        additional_info: Optional[str],
+        hide_value: bool = False,
+        is_scenario_table: bool = False,
+    ) -> Cell:
+        """
+        Use the metric name identified by `matcher` to pull out the stats from
+        `runs` and return a representation of the average.
+        There are four cases:
+        1. No matching runs
+        2. Matching runs but no matching stats (maybe stat was named incorrectly)
+        3. Matching runs, matching stats, but stats have count = 0, so mean is undefined
+           (e.g., bias metric ran and computed 0/0)
+        4. Matching runs, matching stats, stats with count > 0
+
+        In the first three cases, the cell value is None, but the description distinguishes between these cases.
+        """
+        if matcher.name != self.ROBUSTNESS_METRIC_NAME:
+            return super().create_cell(runs, matcher, additional_info, hide_value, is_scenario_table)
+
+        if len(runs) == 0:
+            return Cell(value=None, description="No matching runs")
+
+        # Link the runs that this cell was aggregated from, if this is not a scenario table.
+        # Scenario tables link to the runs in the model cells,
+        # whereas non-scenario tables link to the runs in the metrics cells.
+        run_spec_names: Optional[List] = None
+        if not is_scenario_table:
+            # Deduplicate run spec names becuase aggregated_run_specs may have duplicated
+            # run specs if a run spec belongs to multiple groups.
+            run_spec_names = []
+            run_spec_names_set = set()
+            for run in runs:
+                if run.run_spec.name not in run_spec_names_set:
+                    run_spec_names.append(run.run_spec.name)
+                    run_spec_names_set.add(run.run_spec.name)
+
+        run_group_set = set(runs[0].run_spec.groups) & set(self.run_group_to_model_name_to_robustness.keys())
+        assert len(run_group_set) == 1
+        run_group = next(iter(run_group_set))
+
+        model_names_set = set(run.run_spec.adapter_spec.model for run in runs)
+        assert len(model_names_set) == 1
+        model_name = next(iter(model_names_set))
+
+        value = (
+            self.run_group_to_model_name_to_robustness[run_group][model_name]
+            if self.run_group_to_model_name_to_robustness[run_group]
+            and self.run_group_to_model_name_to_robustness[run_group][model_name]
+            else None
+        )
+        description = str(round(value, 3)) if value is not None else ""
+        if hide_value:
+            value = None
+            description = ""
+        if additional_info:
+            description += "\n" + additional_info
+        if self.verbose:
+            description += "\n-- ".join(["\nRun specs:", *(run_spec_names or [])])
+
+        return Cell(
+            value=value,
+            description=description,
+            style={},
+            run_spec_names=run_spec_names,
+        )

helm/benchmark/reeval_run.py

@@ -0,0 +1,202 @@
+import argparse
+from dataclasses import replace
+import re
+from typing import List
+
+from helm.benchmark import model_metadata_registry
+from helm.benchmark.presentation.run_entry import RunEntry, read_run_entries
+from helm.common.general import ensure_directory_exists
+from helm.common.hierarchical_logger import hlog, htrack, hwarn
+from helm.common.authentication import Authentication
+from helm.proxy.services.remote_service import create_authentication, add_service_args
+
+from helm.benchmark.config_registry import (
+    register_configs_from_directory,
+    register_builtin_configs_from_helm_package,
+)
+from helm.benchmark.runner import set_benchmark_output_path
+from helm.common.reeval_parameters import REEvalParameters
+from helm.benchmark.run import (
+    run_benchmarking,
+    validate_args,
+    add_run_args,
+    run_entries_to_run_specs,
+)
+
+
+@htrack(None)
+def main():
+    parser = argparse.ArgumentParser()
+    add_service_args(parser)
+    parser.add_argument(
+        "-c",
+        "--conf-paths",
+        nargs="+",
+        help="Where to read RunSpecs to run from",
+        default=[],
+    )
+    parser.add_argument(
+        "--models-to-run",
+        nargs="+",
+        help="Only RunSpecs with these models specified. If no model is specified, runs with all models.",
+        default=None,
+    )
+    parser.add_argument(
+        "--groups-to-run",
+        nargs="+",
+        help="Only RunSpecs with these (scenario) groups specified. " "If no group is specified, runs with all groups.",
+        default=None,
+    )
+    parser.add_argument(
+        "--exit-on-error",
+        action="store_true",
+        help="Fail and exit immediately if a particular RunSpec fails.",
+    )
+    parser.add_argument(
+        "--skip-completed-runs",
+        action="store_true",
+        help="Skip RunSpecs that have completed i.e. output files exists.",
+    )
+    parser.add_argument(
+        "--priority",
+        type=int,
+        default=None,
+        help="Run RunSpecs with priority less than or equal to this number. "
+        "If a value for --priority is not specified, run on everything",
+    )
+    parser.add_argument(
+        "--run-specs",
+        nargs="*",
+        help="DEPRECATED: Use --run-entries instead. Will be removed in a future release. "
+        "Specifies run entries to run.",
+        default=[],
+    )
+    parser.add_argument("-r", "--run-entries", nargs="*", help="Specifies run entries to run", default=[])
+    parser.add_argument(
+        "--enable-huggingface-models",
+        nargs="+",
+        default=[],
+        help="Experimental: Enable using AutoModelForCausalLM models from Hugging Face Model Hub. "
+        "Format: namespace/model_name[@revision]",
+    )
+    parser.add_argument(
+        "--enable-local-huggingface-models",
+        nargs="+",
+        default=[],
+        help="Experimental: Enable using AutoModelForCausalLM models from a local path.",
+    )
+    # reeval parameters
+    parser.add_argument(
+        "--model-ability",
+        type=float,
+        default=0.0,
+        help="The initial ability of the model for reeval evaluation.",
+    )
+    add_run_args(parser)
+    args = parser.parse_args()
+    validate_args(args)
+
+    register_builtin_configs_from_helm_package()
+    register_configs_from_directory(args.local_path)
+
+    if args.enable_huggingface_models:
+        from helm.benchmark.huggingface_registration import register_huggingface_hub_model_from_flag_value
+
+        for huggingface_model_name in args.enable_huggingface_models:
+            register_huggingface_hub_model_from_flag_value(huggingface_model_name)
+
+    if args.enable_local_huggingface_models:
+        from helm.benchmark.huggingface_registration import register_huggingface_local_model_from_flag_value
+
+        for huggingface_model_path in args.enable_local_huggingface_models:
+            register_huggingface_local_model_from_flag_value(huggingface_model_path)
+
+    run_entries: List[RunEntry] = []
+    if args.conf_paths:
+        run_entries.extend(read_run_entries(args.conf_paths).entries)
+    if args.run_entries:
+        run_entries.extend(
+            [RunEntry(description=description, priority=1, groups=None) for description in args.run_entries]
+        )
+    # TODO: Remove this eventually.
+    if args.run_specs:
+        run_entries.extend(
+            [RunEntry(description=description, priority=1, groups=None) for description in args.run_specs]
+        )
+
+    # Must set benchmark output path before getting RunSpecs,
+    # because run spec functions can use the benchmark output directory for caching.
+    ensure_directory_exists(args.output_path)
+    set_benchmark_output_path(args.output_path)
+
+    # Validate the --models-to-run flag
+    if args.models_to_run:
+        all_models = set(model_metadata_registry.get_all_models())
+        for model_to_run in args.models_to_run:
+            if model_to_run not in all_models:
+                raise Exception(f"Unknown model '{model_to_run}' passed to --models-to-run")
+    else:
+        model_expander_pattern = re.compile(
+            r"\bmodel=(?:all|text_code|text|code|instruction_following|full_functionality_text|limited_functionality_text)\b"  # noqa: E501
+        )
+        if any(model_expander_pattern.search(run_entry.description) for run_entry in run_entries):
+            raise Exception("--models-to-run must be set if the `models=` run expander expands to multiple models")
+
+    run_specs = run_entries_to_run_specs(
+        run_entries=run_entries,
+        max_eval_instances=args.max_eval_instances,
+        num_train_trials=args.num_train_trials,
+        models_to_run=args.models_to_run,
+        groups_to_run=args.groups_to_run,
+        priority=args.priority,
+    )
+    hlog(f"{len(run_entries)} entries produced {len(run_specs)} run specs")
+
+    if len(run_specs) == 0:
+        hlog("There were no RunSpecs or they got filtered out.")
+        return
+
+    # Add reeval_parameters
+    run_specs = [
+        replace(
+            run_spec,
+            adapter_spec=replace(
+                run_spec.adapter_spec, reeval_parameters=REEvalParameters(model_ability=args.model_ability)
+            ),
+        )
+        for run_spec in run_specs
+    ]
+
+    auth: Authentication = (
+        Authentication("") if args.skip_instances or not args.server_url else create_authentication(args)
+    )
+
+    run_benchmarking(
+        run_specs=run_specs,
+        auth=auth,
+        url=args.server_url,
+        local_path=args.local_path,
+        num_threads=args.num_threads,
+        output_path=args.output_path,
+        suite=args.suite,
+        dry_run=args.dry_run,
+        skip_instances=args.skip_instances,
+        cache_instances=args.cache_instances,
+        cache_instances_only=args.cache_instances_only,
+        skip_completed_runs=args.skip_completed_runs,
+        exit_on_error=args.exit_on_error,
+        runner_class_name="helm.benchmark.reeval_runner.REEvalRunner",
+        mongo_uri=args.mongo_uri,
+        disable_cache=args.disable_cache,
+    )
+
+    if args.run_specs:
+        hwarn(
+            "The --run-specs flag is deprecated and will be removed in a future release. " "Use --run-entries instead."
+        )
+
+    hlog("Done.")
+
+
+if __name__ == "__main__":
+    main()