crfm-helm 0.4.0__py3-none-any.whl → 0.5.10__py3-none-any.whl

helm/benchmark/reeval_runner.py

@@ -0,0 +1,355 @@
+import dacite
+import json
+import os
+import typing
+from collections import Counter
+from typing import Any, Dict, List, Optional
+import torch
+
+from tqdm import tqdm
+from dataclasses import replace
+from datasets import load_dataset
+
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.common.general import ensure_directory_exists, write, asdict_without_nones
+from helm.common.hierarchical_logger import hlog, htrack_block, hwarn
+from helm.common.cache import cache_stats
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    create_scenario,
+    Instance,
+    get_scenario_cache_path,
+    with_instance_ids,
+)
+from helm.benchmark.adaptation.adapters.adapter import Adapter
+from helm.benchmark.adaptation.adapters.adapter_factory import AdapterFactory
+from helm.benchmark.adaptation.scenario_state import ScenarioState
+from helm.benchmark.run_spec import RunSpec
+from helm.benchmark.data_preprocessor import DataPreprocessor
+from helm.benchmark.executor import ExecutionSpec
+from helm.benchmark.metrics.metric_name import MetricName
+from helm.benchmark.metrics.dry_run_metrics import DryRunMetric
+from helm.benchmark.metrics.metric import MetricInterface, MetricResult, PerInstanceStats, create_metric, Stat
+from helm.benchmark.runner import (
+    Runner,
+    remove_stats_nans,
+    remove_per_instance_stats_nans,
+)
+
+scenario_to_metric_name = {
+    "air_bench_2024": "air_score",
+    "babi_qa": "quasi_exact_match",
+    "bbq": "quasi_exact_match",
+    "blimp": "exact_match",
+    "boolq": "quasi_exact_match",
+    "civil_comments": "quasi_exact_match",
+    "dyck_language": "exact_match_indicator",
+    "entity_data_imputation": "quasi_exact_match",
+    "entity_matching": "quasi_exact_match",
+    "imdb": "quasi_exact_match",
+    "legal_support": "quasi_exact_match",
+    "raft": "quasi_exact_match",
+    "synthetic_reasoning": "quasi_exact_match",
+    "truthful_qa": "exact_match",
+    "wikifact": "quasi_exact_match",
+    "mmlu": "exact_match",
+    "commonsense": "exact_match",
+    "gsm": "final_number_exact_match",
+    # "gsm": ["exact_match_indicator", "final_number_exact_match"],
+    "legalbench": "quasi_exact_match",
+    "math": "math_equiv_chain_of_thought",
+    "med_qa": "quasi_exact_match",
+    "thai_exam": "exact_match",
+}
+
+
+class REEvalRunner(Runner):
+    """
+    This runner implements the basic (non-amortized) method described in the paper
+    `Reliable and Efficient Amortized Model-Based Evaluation`. This approach, which is
+    also known as Computerized Adaptive Testing (CAT) within the framework of Item Response
+    Theory (IRT), leverages adaptive testing to evaluate model performance.
+
+    The difficulties of the questions are provided in a HuggingFace repository. In addition,
+    the authors of the paper will supply a Python package for calculating these difficulties.
+    At each iteration, the runner estimates the model's ability based on all previously
+    administered questions and their corresponding responses. It then selects the next question
+    whose difficulty is closest to the estimated ability, thereby reliably and efficiently
+    eliciting the model's ability.
+    """
+
+    def __init__(
+        self,
+        execution_spec: ExecutionSpec,
+        output_path: str,
+        suite: str,
+        skip_instances: bool,
+        cache_instances: bool,
+        cache_instances_only: bool,
+        skip_completed_runs: bool,
+        exit_on_error: bool,
+    ):
+        super().__init__(
+            execution_spec=execution_spec,
+            output_path=output_path,
+            suite=suite,
+            skip_instances=skip_instances,
+            cache_instances=cache_instances,
+            cache_instances_only=cache_instances_only,
+            skip_completed_runs=skip_completed_runs,
+            exit_on_error=exit_on_error,
+        )
+
+    def _estimate_model_ability(
+        self,
+        old_ability: float,
+        response_correctness: List[float],
+        instance_difficulties: List[float],
+    ) -> float:
+        def closure():
+            optim.zero_grad()
+            probs = torch.sigmoid(ability + difficulties)
+            loss = -torch.distributions.Bernoulli(probs=probs).log_prob(responses).mean()
+            loss.backward()
+            return loss
+
+        device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+        responses = torch.tensor(response_correctness, device=device)
+        difficulties = torch.tensor(instance_difficulties, device=device)
+        ability = torch.tensor([old_ability], requires_grad=True, device=device)
+        optim = torch.optim.LBFGS([ability], lr=0.1, max_iter=20, history_size=10, line_search_fn="strong_wolfe")
+
+        for iteration in range(100):
+            loss = optim.step(closure)
+
+            if iteration > 0:
+                prev_ability = ability.clone()
+                prev_loss = loss
+                d_loss = prev_loss - loss
+                d_theta = torch.norm(prev_ability - ability, p=2)
+                grad_norm = torch.norm(optim.param_groups[0]["params"][0].grad, p=2)
+                if d_loss < 1e-5 and d_theta < 1e-5 and grad_norm < 1e-5:
+                    break
+
+        return ability.item()
+
+    def run_one(self, run_spec: RunSpec):
+        run_path: str = self._get_run_path(run_spec)
+        if self.skip_completed_runs and self._is_run_completed(run_path):
+            hlog(f"Skipping run {run_spec.name} because run is completed and all output files exist.")
+            return
+        ensure_directory_exists(run_path)
+
+        # Load the scenario
+        scenario: Scenario = create_scenario(run_spec.scenario_spec)
+
+        # This 'output_path' will be used when the model's input instances are saved.
+        args_str = ",".join([f"{k}={v}" for k, v in sorted(run_spec.scenario_spec.args.items())])
+        scenario_name_with_args = f"{scenario.name}:{args_str}" if args_str else f"{scenario.name}"
+        input_instances_output_path = os.path.join(self.instances_path, scenario_name_with_args)
+        input_instances_file_path = os.path.join(input_instances_output_path, "input_instances.json")
+
+        instances: List[Instance]
+        if self.skip_instances:
+            instances = []
+        else:
+            if self.cache_instances and os.path.exists(input_instances_file_path):
+                with open(input_instances_file_path) as f:
+                    json_instances: List[Dict[str, Any]] = json.load(f)
+                instances = [dacite.from_dict(Instance, instance) for instance in json_instances]
+            else:
+                # Create the instances of the scenario
+                scenario_output_path = get_scenario_cache_path(self.output_path, scenario.name)
+                with htrack_block("scenario.get_instances"):
+                    instances = scenario.get_instances(scenario_output_path)
+        if self.cache_instances and not os.path.exists(input_instances_file_path):
+            # Save instances to file
+            ensure_directory_exists(input_instances_output_path)
+            write(
+                os.path.join(input_instances_file_path),
+                json.dumps([asdict_without_nones(instance) for instance in instances], indent=2),
+            )
+        if self.cache_instances_only:
+            return  # Exit after saving the instances.
+
+        # Give each instance a unique ID
+        if any([instance.id is None for instance in instances]):
+            instances = with_instance_ids(instances)
+
+        # Data preprocessing
+        instances = DataPreprocessor(run_spec.data_augmenter_spec).preprocess(
+            instances, self.executor.execution_spec.parallelism
+        )
+
+        # Adapt (convert to requests)
+        adapter: Adapter = AdapterFactory.get_adapter(run_spec.adapter_spec, self.tokenizer_service)
+        unasked_request_states_without_difficulty: List[RequestState] = adapter.adapt(
+            instances, self.executor.execution_spec.parallelism
+        )
+
+        # load difficulty
+        split_name = "dyck_language_np_3" if scenario.name == "dyck_language" else scenario.name
+        try:
+            difficulty_dataset = load_dataset("stair-lab/reeval-difficulty", split=split_name)
+            prompt_to_difficulty: dict[str, float] = {row["request.prompt"]: row["z"] for row in difficulty_dataset}
+        except ValueError:
+            hwarn(f"no available difficulty for {split_name}, skipping")
+            return
+
+        unasked_request_states: List[RequestState] = []
+        for request_state in unasked_request_states_without_difficulty:
+            prompt = request_state.request.prompt
+            if prompt in prompt_to_difficulty:
+                difficulty = prompt_to_difficulty[prompt]
+                current_extra_data = request_state.instance.extra_data or {}
+                if "difficulty" in current_extra_data:
+                    raise Exception("Extra_data already contains a 'difficulty' key.")
+                new_extra_data = current_extra_data.copy()
+                new_extra_data["difficulty"] = difficulty
+                new_instance = replace(request_state.instance, extra_data=new_extra_data)
+                new_request_state = replace(request_state, instance=new_instance)
+                unasked_request_states.append(new_request_state)
+        assert unasked_request_states
+
+        # Execute the requests in an reeval manner
+        assert run_spec.adapter_spec.reeval_parameters is not None
+        model_ability: float = run_spec.adapter_spec.reeval_parameters.model_ability or 0.0
+        scenario_metric_name: str = scenario_to_metric_name[scenario.name]
+
+        asked_request_states: List[RequestState] = []
+        reeval_trajectory: Dict[str, List[float]] = {
+            "model_ability": [],
+            "response_correctness": [],
+            "instance_difficulties": [],
+        }
+
+        assert run_spec.adapter_spec.max_eval_instances is not None
+        for _ in tqdm(range(run_spec.adapter_spec.max_eval_instances), desc="REEval Execution"):
+            if not unasked_request_states:
+                break
+
+            selected_item: Optional[RequestState] = None
+            min_diff = float("inf")
+            for item in unasked_request_states:
+                assert item.instance.extra_data is not None
+                diff = abs(item.instance.extra_data["difficulty"] + model_ability)
+                if diff < min_diff:
+                    min_diff = diff
+                    selected_item = item
+            assert selected_item is not None
+            unasked_request_states.remove(selected_item)
+
+            # Execute the request
+            single_scenario_state: ScenarioState = ScenarioState(
+                adapter_spec=run_spec.adapter_spec,
+                request_states=[selected_item],
+                annotator_specs=run_spec.annotators,
+            )
+
+            # Execute (fill up results)
+            single_scenario_state = self.executor.execute(single_scenario_state)
+
+            # Annotate (post-process the results)
+            single_scenario_state = self.annotator_executor.execute(single_scenario_state)
+
+            # Apply the metrics
+            # When performing a dry run, only estimate the number of tokens instead
+            # of calculating the metrics.
+            metrics: List[MetricInterface] = (
+                [DryRunMetric()]
+                if self.dry_run
+                else [create_metric(metric_spec) for metric_spec in run_spec.metric_specs]
+            )
+
+            temp_per_instance_stats: List[PerInstanceStats] = []
+            with htrack_block(f"{len(metrics)} metrics"):
+                for metric in metrics:
+                    with htrack_block(metric):
+                        temp_metric_result: MetricResult = metric.evaluate(
+                            single_scenario_state,
+                            self.metric_service,
+                            self.eval_cache_path,
+                            self.executor.execution_spec.parallelism,
+                        )
+                        temp_per_instance_stats.extend(temp_metric_result.per_instance_stats)
+
+            # Update the reeval request states
+            asked_request_states.extend(single_scenario_state.request_states)
+
+            # Update the reeval trajectory
+            reeval_trajectory["model_ability"].append(model_ability)
+            scenario_metric_value = [
+                s for s in temp_per_instance_stats[0].stats if s.name.name == scenario_metric_name
+            ][0].mean
+
+            assert scenario_metric_value is not None
+            reeval_trajectory["response_correctness"].append(scenario_metric_value)
+            assert selected_item.instance.extra_data is not None
+            reeval_trajectory["instance_difficulties"].append(selected_item.instance.extra_data["difficulty"])
+
+            # Estimate the model ability
+            model_ability = self._estimate_model_ability(
+                old_ability=model_ability,
+                response_correctness=reeval_trajectory["response_correctness"],
+                instance_difficulties=reeval_trajectory["instance_difficulties"],
+            )
+
+        # Create the scenario state
+        scenario_state: ScenarioState = ScenarioState(
+            adapter_spec=run_spec.adapter_spec,
+            request_states=asked_request_states,
+            annotator_specs=run_spec.annotators,
+        )
+
+        stats: List[Stat] = []
+        per_instance_stats: List[PerInstanceStats] = []
+        with htrack_block(f"{len(metrics)} metrics"):
+            for metric in metrics:
+                with htrack_block(metric):
+                    metric_result: MetricResult = metric.evaluate(
+                        scenario_state,
+                        self.metric_service,
+                        self.eval_cache_path,
+                        self.executor.execution_spec.parallelism,
+                    )
+                    stats.extend(metric_result.aggregated_stats)
+                    per_instance_stats.extend(metric_result.per_instance_stats)
+
+        # Check that there aren't duplicate `Stat`s
+        # Note: doesn't catch near misses.
+        metric_counts: typing.Counter[MetricName] = Counter([stat.name for stat in stats])
+        for metric_name, count in metric_counts.items():
+            if count > 1:
+                hwarn(f"duplicate metric name {metric_name}")
+
+        # Print out the number of stats
+        hlog(f"Generated {len(stats)} stats.")
+
+        if self.skip_instances:
+            hlog("skip_instances was True. Skipping writing results out.")
+            return
+
+        # Output benchmarking information and results to files
+        write(os.path.join(run_path, "run_spec.json"), json.dumps(asdict_without_nones(run_spec), indent=2))
+
+        # Write out scenario
+        write(os.path.join(run_path, "scenario.json"), json.dumps(asdict_without_nones(scenario), indent=2))
+
+        # Write scenario state
+        write(os.path.join(run_path, "scenario_state.json"), json.dumps(asdict_without_nones(scenario_state), indent=2))
+
+        write(
+            os.path.join(run_path, "stats.json"),
+            json.dumps([asdict_without_nones(stat) for stat in remove_stats_nans(stats)], indent=2),
+        )
+        write(
+            os.path.join(run_path, "per_instance_stats.json"),
+            json.dumps(list(map(asdict_without_nones, remove_per_instance_stats_nans(per_instance_stats))), indent=2),
+        )
+
+        write(
+            os.path.join(run_path, "reeval_trajectory.json"),
+            json.dumps(reeval_trajectory, indent=2),
+        )
+
+        cache_stats.print_status()

helm/benchmark/run.py

@@ -1,27 +1,28 @@
 import argparse
 from dataclasses import replace
+import os
+import re
 from typing import List, Optional
-from helm.benchmark.huggingface_registration import (
-    register_huggingface_hub_model_from_flag_value,
-    register_huggingface_local_model_from_flag_value,
-)
 
+
+from helm.benchmark import model_metadata_registry
 from helm.benchmark.presentation.run_entry import RunEntry, read_run_entries
+from helm.common.cache_backend_config import MongoCacheBackendConfig, SqliteCacheBackendConfig
 from helm.common.general import ensure_directory_exists
-from helm.common.hierarchical_logger import hlog, htrack, htrack_block
+from helm.common.hierarchical_logger import hlog, htrack, htrack_block, setup_default_logging, hwarn
 from helm.common.authentication import Authentication
 from helm.common.object_spec import parse_object_spec, get_class_by_name
 from helm.proxy.services.remote_service import create_authentication, add_service_args
+from helm.proxy.services.service import CACHE_DIR
 
 from helm.benchmark.config_registry import (
     register_configs_from_directory,
     register_builtin_configs_from_helm_package,
 )
 from helm.benchmark.adaptation.adapter_spec import AdapterSpec
-from helm.benchmark import vlm_run_specs  # noqa
-from .executor import ExecutionSpec
-from .runner import Runner, RunSpec, LATEST_SYMLINK, set_benchmark_output_path
-from .run_specs import construct_run_specs
+from helm.benchmark.executor import ExecutionSpec
+from helm.benchmark.runner import Runner, RunSpec, LATEST_SYMLINK, set_benchmark_output_path
+from helm.benchmark.run_spec_factory import construct_run_specs
 
 
 def run_entries_to_run_specs(
@@ -36,7 +37,7 @@ def run_entries_to_run_specs(
     run_specs: List[RunSpec] = []
     for entry in run_entries:
         # Filter by priority
-        if priority is not None and entry.priority > priority:
+        if priority is not None and entry.priority is not None and entry.priority > priority:
             continue
 
         for run_spec in construct_run_specs(parse_object_spec(entry.description)):
@@ -85,16 +86,29 @@ def run_benchmarking(
     skip_completed_runs: bool,
     exit_on_error: bool,
     runner_class_name: Optional[str],
-    mongo_uri: str = "",
+    mongo_uri: Optional[str] = None,
+    disable_cache: Optional[bool] = None,
 ) -> List[RunSpec]:
     """Runs RunSpecs given a list of RunSpec descriptions."""
+    sqlite_cache_backend_config: Optional[SqliteCacheBackendConfig] = None
+    mongo_cache_backend_config: Optional[MongoCacheBackendConfig] = None
+
+    if not disable_cache:
+        if mongo_uri:
+            mongo_cache_backend_config = MongoCacheBackendConfig(mongo_uri)
+        else:
+            sqlite_cache_path = os.path.join(local_path, CACHE_DIR)
+            ensure_directory_exists(sqlite_cache_path)
+            sqlite_cache_backend_config = SqliteCacheBackendConfig(sqlite_cache_path)
+
     execution_spec = ExecutionSpec(
         auth=auth,
         url=url,
         local_path=local_path,
         parallelism=num_threads,
         dry_run=dry_run,
-        mongo_uri=mongo_uri,
+        sqlite_cache_backend_config=sqlite_cache_backend_config,
+        mongo_cache_backend_config=mongo_cache_backend_config,
     )
     with htrack_block("run_specs"):
         for run_spec in run_specs:
@@ -160,13 +174,6 @@ def add_run_args(parser: argparse.ArgumentParser):
         help="Name of the suite this run belongs to (default is today's date).",
         required=True,
     )
-    parser.add_argument(
-        "--local",
-        action="store_true",
-        help="DEPRECATED: Does nothing. Do not use. Previously enabled local mode. "
-        "Now does nothing and will be removed in the next released version. "
-        "Local mode is enabled by default, and only disabled if the --server_url flag is set.",
-    )
     parser.add_argument(
         "--local-path",
         type=str,
@@ -179,6 +186,11 @@ def add_run_args(parser: argparse.ArgumentParser):
         help="If non-empty, the URL of the MongoDB database that will be used for caching instead of SQLite",
         default="",
     )
+    parser.add_argument(
+        "--disable-cache",
+        action="store_true",
+        help="If true, the request-response cache for model clients and tokenizers will be disabled.",
+    )
 
 
 def validate_args(args):
@@ -188,7 +200,105 @@ def validate_args(args):
 
 
 @htrack(None)
-def main():
+def helm_run(args):
+
+    validate_args(args)
+    register_builtin_configs_from_helm_package()
+    register_configs_from_directory(args.local_path)
+
+    if args.enable_huggingface_models:
+        from helm.benchmark.huggingface_registration import register_huggingface_hub_model_from_flag_value
+
+        for huggingface_model_name in args.enable_huggingface_models:
+            register_huggingface_hub_model_from_flag_value(huggingface_model_name)
+
+    if args.enable_local_huggingface_models:
+        from helm.benchmark.huggingface_registration import register_huggingface_local_model_from_flag_value
+
+        for huggingface_model_path in args.enable_local_huggingface_models:
+            register_huggingface_local_model_from_flag_value(huggingface_model_path)
+
+    run_entries: List[RunEntry] = []
+    if args.conf_paths:
+        run_entries.extend(read_run_entries(args.conf_paths).entries)
+    if args.run_entries:
+        run_entries.extend(
+            [RunEntry(description=description, priority=1, groups=None) for description in args.run_entries]
+        )
+    # TODO: Remove this eventually.
+    if args.run_specs:
+        run_entries.extend(
+            [RunEntry(description=description, priority=1, groups=None) for description in args.run_specs]
+        )
+
+    # Must set benchmark output path before getting RunSpecs,
+    # because run spec functions can use the benchmark output directory for caching.
+    ensure_directory_exists(args.output_path)
+    set_benchmark_output_path(args.output_path)
+
+    # Validate the --models-to-run flag
+    if args.models_to_run:
+        all_models = set(model_metadata_registry.get_all_models())
+        for model_to_run in args.models_to_run:
+            if model_to_run not in all_models:
+                raise Exception(f"Unknown model '{model_to_run}' passed to --models-to-run")
+    else:
+        model_expander_wildcard_pattern = re.compile(
+            r"\bmodel=(?:all|text_code|text|code|instruction_following|full_functionality_text|limited_functionality_text)\b"  # noqa: E501
+        )
+        if any(model_expander_wildcard_pattern.search(run_entry.description) for run_entry in run_entries):
+            raise Exception("--models-to-run must be set if the `models=` run expander expands to multiple models")
+
+        model_expander_pattern = re.compile(r"\bmodel=\b")
+        if not any(model_expander_pattern.search(run_entry.description) for run_entry in run_entries):
+            raise Exception("--models-to-run must be set if the `models=` run expander is omitted")
+
+    run_specs = run_entries_to_run_specs(
+        run_entries=run_entries,
+        max_eval_instances=args.max_eval_instances,
+        num_train_trials=args.num_train_trials,
+        models_to_run=args.models_to_run,
+        groups_to_run=args.groups_to_run,
+        priority=args.priority,
+    )
+    hlog(f"{len(run_entries)} entries produced {len(run_specs)} run specs")
+
+    if len(run_specs) == 0:
+        hlog("There were no RunSpecs or they got filtered out.")
+        return
+
+    auth: Authentication = (
+        Authentication("") if args.skip_instances or not args.server_url else create_authentication(args)
+    )
+
+    run_benchmarking(
+        run_specs=run_specs,
+        auth=auth,
+        url=args.server_url,
+        local_path=args.local_path,
+        num_threads=args.num_threads,
+        output_path=args.output_path,
+        suite=args.suite,
+        dry_run=args.dry_run,
+        skip_instances=args.skip_instances,
+        cache_instances=args.cache_instances,
+        cache_instances_only=args.cache_instances_only,
+        skip_completed_runs=args.skip_completed_runs,
+        exit_on_error=args.exit_on_error,
+        runner_class_name=args.runner_class_name,
+        mongo_uri=args.mongo_uri,
+        disable_cache=args.disable_cache,
+    )
+
+    if args.run_specs:
+        hwarn(
+            "The --run-specs flag is deprecated and will be removed in a future release. " "Use --run-entries instead."
+        )
+
+    hlog("Done.")
+
+
+def build_parser():
     parser = argparse.ArgumentParser()
     add_service_args(parser)
     parser.add_argument(
@@ -227,7 +337,14 @@ def main():
         help="Run RunSpecs with priority less than or equal to this number. "
         "If a value for --priority is not specified, run on everything",
     )
-    parser.add_argument("-r", "--run-specs", nargs="*", help="Specifies what to run", default=[])
+    parser.add_argument(
+        "--run-specs",
+        nargs="*",
+        help="DEPRECATED: Use --run-entries instead. Will be removed in a future release. "
+        "Specifies run entries to run.",
+        default=[],
+    )
+    parser.add_argument("-r", "--run-entries", nargs="*", help="Specifies run entries to run", default=[])
     parser.add_argument(
         "--enable-huggingface-models",
         nargs="+",
@@ -247,75 +364,22 @@ def main():
         default=None,
         help="Full class name of the Runner class to use. If unset, uses the default Runner.",
     )
-    add_run_args(parser)
-    args = parser.parse_args()
-    validate_args(args)
-
-    register_builtin_configs_from_helm_package()
-    register_configs_from_directory(args.local_path)
-
-    for huggingface_model_name in args.enable_huggingface_models:
-        register_huggingface_hub_model_from_flag_value(huggingface_model_name)
-    for huggingface_model_path in args.enable_local_huggingface_models:
-        register_huggingface_local_model_from_flag_value(huggingface_model_path)
-
-    run_entries: List[RunEntry] = []
-    if args.conf_paths:
-        run_entries.extend(read_run_entries(args.conf_paths).entries)
-    if args.run_specs:
-        run_entries.extend(
-            [RunEntry(description=description, priority=1, groups=None) for description in args.run_specs]
-        )
-
-    # Must set benchmark output path before getting RunSpecs,
-    # because run spec functions can use the benchmark output directory for caching.
-    ensure_directory_exists(args.output_path)
-    set_benchmark_output_path(args.output_path)
-
-    run_specs = run_entries_to_run_specs(
-        run_entries=run_entries,
-        max_eval_instances=args.max_eval_instances,
-        num_train_trials=args.num_train_trials,
-        models_to_run=args.models_to_run,
-        groups_to_run=args.groups_to_run,
-        priority=args.priority,
-    )
-    hlog(f"{len(run_entries)} entries produced {len(run_specs)} run specs")
-
-    if len(run_specs) == 0:
-        hlog("There were no RunSpecs or they got filtered out.")
-        return
-
-    auth: Authentication = (
-        Authentication("") if args.skip_instances or not args.server_url else create_authentication(args)
-    )
-
-    run_benchmarking(
-        run_specs=run_specs,
-        auth=auth,
-        url=args.server_url,
-        local_path=args.local_path,
-        num_threads=args.num_threads,
-        output_path=args.output_path,
-        suite=args.suite,
-        dry_run=args.dry_run,
-        skip_instances=args.skip_instances,
-        cache_instances=args.cache_instances,
-        cache_instances_only=args.cache_instances_only,
-        skip_completed_runs=args.skip_completed_runs,
-        exit_on_error=args.exit_on_error,
-        runner_class_name=args.runner_class_name,
-        mongo_uri=args.mongo_uri,
+    parser.add_argument(
+        "--log-config",
+        type=str,
+        default=None,
+        help="PATH to a YAML file to customize logging",
     )
+    add_run_args(parser)
+    return parser
 
-    if args.local:
-        hlog(
-            "WARNING: The --local flag is deprecated. It now does nothing and will be removed in "
-            "the next released version. Local mode is enabled by default, and only disabled if the "
-            "--server_url flag is set. Please remove --local from your command."
-        )
 
-    hlog("Done.")
+# Separate parsing from starting HELM so we can setup logging
+def main():
+    parser = build_parser()
+    args = parser.parse_args()
+    setup_default_logging(args.log_config)
+    return helm_run(args)
 
 
 if __name__ == "__main__":