crfm-helm 0.5.2__py3-none-any.whl → 0.5.3__py3-none-any.whl

helm/benchmark/run_specs/call_center_run_specs.py

@@ -0,0 +1,152 @@
+"""Run specs for experiments only.
+
+These run specs are not intended for use with public leaderboards."""
+
+from helm.benchmark.adaptation.adapter_spec import ADAPT_GENERATION, AdapterSpec
+from helm.benchmark.annotation.annotator import AnnotatorSpec
+from helm.benchmark.metrics.common_metric_specs import get_basic_metric_specs
+from helm.benchmark.metrics.metric import MetricSpec
+from helm.benchmark.run_spec import RunSpec, run_spec_function
+from helm.benchmark.scenarios.scenario import ScenarioSpec
+
+
+@run_spec_function("call_center_summarization")
+def get_call_center_summarization_spec(subset: str = "summarization") -> RunSpec:
+    from helm.benchmark.annotation.call_center_annotator import CallCenterSummarizationAnnotator
+
+    scenario_spec = ScenarioSpec(
+        class_name="helm.benchmark.scenarios.call_center_scenario.CallCenterSummarizationScenario",
+        args={"subset": subset},
+    )
+
+    instructions = "Summarize the call transcript in under 10 sentences."
+
+    adapter_spec = AdapterSpec(
+        method=ADAPT_GENERATION,
+        instructions=instructions,
+        input_prefix="### Call Transcript\n",
+        input_suffix="",
+        output_prefix="",
+        output_suffix="",
+        max_train_instances=0,
+        temperature=0.0,
+        max_tokens=512,
+        num_outputs=1,
+    )
+
+    annotator_specs = annotator_specs = [
+        AnnotatorSpec(class_name="helm.benchmark.annotation.call_center_annotator.CallCenterSummarizationAnnotator")
+    ]
+    annotation_metric_specs = [
+        MetricSpec(
+            class_name="helm.benchmark.metrics.annotation_metrics.AnnotationLikertScaleMetric",
+            args={
+                "annotator_name": CallCenterSummarizationAnnotator.name,
+                "key": criterion,
+                "min_score": 1,
+                "max_score": 5,
+            },
+        )
+        for criterion in CallCenterSummarizationAnnotator.CRITERIA
+    ]
+
+    metric_specs = get_basic_metric_specs([]) + annotation_metric_specs
+
+    group = "call_center_summarization" if subset == "summarization" else f"call_center_summarization_{subset}"
+
+    return RunSpec(
+        name="call_center_summarization",
+        scenario_spec=scenario_spec,
+        adapter_spec=adapter_spec,
+        metric_specs=metric_specs,
+        annotators=annotator_specs,
+        groups=[group],
+    )
+
+
+@run_spec_function("call_center_summarization_pairwise_comparison")
+def get_call_center_summarization_pairwise_comparison_spec() -> RunSpec:
+    scenario_spec = ScenarioSpec(
+        class_name="helm.benchmark.scenarios.call_center_scenario.CallCenterSummarizationPairwiseComparisonScenario",
+    )
+
+    instructions = "Summarize the call transcript in under 10 sentences."
+
+    adapter_spec = AdapterSpec(
+        method=ADAPT_GENERATION,
+        instructions=instructions,
+        input_prefix="### Call Transcript\n",
+        input_suffix="",
+        output_prefix="",
+        output_suffix="",
+        max_train_instances=0,
+        temperature=0.0,
+        max_tokens=512,
+        num_outputs=1,
+    )
+
+    annotator_specs = annotator_specs = [
+        AnnotatorSpec(
+            class_name="helm.benchmark.annotation.call_center_annotator.CallCenterSummarizationPairwiseComparisonAnnotator"  # noqa: E501
+        )
+    ]
+
+    metric_specs = get_basic_metric_specs([]) + [
+        MetricSpec(
+            class_name="helm.benchmark.metrics.annotation_metrics.AnnotationNumericMetric",
+            args={"annotator_name": "call_center_summarization_pairwise_comparison", "key": "score"},
+        )
+    ]
+
+    return RunSpec(
+        name="call_center_summarization_pairwise_comparison",
+        scenario_spec=scenario_spec,
+        adapter_spec=adapter_spec,
+        metric_specs=metric_specs,
+        annotators=annotator_specs,
+        groups=["call_center_summarization_pairwise_comparison"],
+    )
+
+
+@run_spec_function("call_center_summarization_key_points_recall")
+def get_call_center_summarization_key_points_recall_spec() -> RunSpec:
+    scenario_spec = ScenarioSpec(
+        class_name="helm.benchmark.scenarios.call_center_scenario.CallCenterSummarizationKeyPointsRecallScenario",
+    )
+
+    instructions = "Summarize the call transcript in under 10 sentences."
+
+    adapter_spec = AdapterSpec(
+        method=ADAPT_GENERATION,
+        instructions=instructions,
+        input_prefix="### Call Transcript\n",
+        input_suffix="",
+        output_prefix="",
+        output_suffix="",
+        max_train_instances=0,
+        temperature=0.0,
+        max_tokens=512,
+        num_outputs=1,
+    )
+
+    annotator_specs = annotator_specs = [
+        AnnotatorSpec(
+            class_name="helm.benchmark.annotation.call_center_annotator.CallCenterSummarizationKeyPointsRecallAnnotator"
+        )
+    ]
+
+    metric_specs = get_basic_metric_specs([]) + [
+        MetricSpec(
+            class_name="helm.benchmark.metrics.annotation_metrics.AnnotationNumericMetric",
+            args={"annotator_name": "call_center_summarization_key_points_recall", "key": "score"},
+        )
+    ]
+
+    return RunSpec(
+        name="call_center_summarization_key_points_recall",
+        scenario_spec=scenario_spec,
+        adapter_spec=adapter_spec,
+        metric_specs=metric_specs,
+        annotators=annotator_specs,
+        groups=["call_center_summarization_key_points_recall"],
+    )

helm/benchmark/run_specs/decodingtrust_run_specs.py

@@ -56,7 +56,7 @@ def get_decodingtrust_stereotype_bias_spec(task: str) -> RunSpec:
         scenario_spec=scenario_spec,
         adapter_spec=adapter_spec,
         metric_specs=get_stereotype_bias_metric_specs(),
-        groups=["decodingtrust", "stereotype_bias"],
+        groups=["decodingtrust_stereotype_bias", "stereotype_bias"],
     )
 
 
@@ -74,7 +74,7 @@ def get_decodingtrust_adv_robustness_spec(task: str) -> RunSpec:
         scenario_spec=scenario_spec,
         adapter_spec=adapter_spec,
         metric_specs=get_exact_match_metric_specs(),
-        groups=["decodingtrust", "adv_robustness"],
+        groups=["decodingtrust_adv_robustness", "adv_robustness"],
     )
 
 
@@ -92,7 +92,7 @@ def get_decodingtrust_adv_demonstration_spec(perspective: str, data: str, demo_n
         scenario_spec=scenario_spec,
         adapter_spec=adapter_spec,
         metric_specs=get_exact_match_metric_specs(),
-        groups=["decodingtrust", "adv_demonstration"],
+        groups=["decodingtrust_adv_demonstration", "adv_demonstration"],
     )
 
 
@@ -137,7 +137,7 @@ def get_decodingtrust_ood_robustness_spec(
         scenario_spec=scenario_spec,
         adapter_spec=adapter_spec,
         metric_specs=get_exact_match_metric_specs() if ood_type == "style" else get_ood_knowledge_metric_specs(),
-        groups=["decodingtrust", "ood_robustness"],
+        groups=["decodingtrust_ood_robustness", "ood_robustness"],
     )
 
 
@@ -163,7 +163,7 @@ def get_decodingtrust_fairness_spec(
         scenario_spec=scenario_spec,
         adapter_spec=adapter_spec,
         metric_specs=get_fairness_metric_specs() + get_exact_match_metric_specs(),
-        groups=["decodingtrust", "fairness"],
+        groups=["decodingtrust_fairness", "fairness"],
     )
 
 
@@ -192,7 +192,7 @@ def get_decodingtrust_privacy_spec(
         scenario_spec=scenario_spec,
         adapter_spec=adapter_spec,
         metric_specs=get_privacy_metric_specs(),
-        groups=["decodingtrust", "privacy"],
+        groups=["decodingtrust_privacy", "privacy"],
     )
 
 
@@ -280,7 +280,7 @@ def get_decodingtrust_machine_ethics_spec(
         scenario_spec=scenario_spec,
         adapter_spec=adapter_spec,
         metric_specs=get_exact_match_metric_specs(),
-        groups=["decodingtrust", "machine_ethics"],
+        groups=["decodingtrust_machine_ethics", "machine_ethics"],
     )
 
 
@@ -312,5 +312,5 @@ def get_decodingtrust_toxicity_prompts_spec(subject) -> RunSpec:
         metric_specs=get_generative_harms_metric_specs(
             include_basic_metrics=True, include_generative_harms_metrics=True
         ),
-        groups=["decodingtrust", "toxicity_prompts"],
+        groups=["decodingtrust_toxicity_prompts", "toxicity_prompts"],
     )

helm/benchmark/run_specs/experimental_run_specs.py

@@ -2,6 +2,7 @@
 
 These run specs are not intended for use with public leaderboards."""
 
+from helm.benchmark.adaptation.adapter_spec import AdapterSpec
 from helm.benchmark.adaptation.adapters.adapter_factory import ADAPT_MULTIPLE_CHOICE_JOINT
 from helm.benchmark.adaptation.common_adapter_specs import get_multiple_choice_adapter_spec
 from helm.benchmark.metrics.common_metric_specs import get_exact_match_metric_specs
@@ -31,3 +32,54 @@ def get_ci_mcqa_spec() -> RunSpec:
         metric_specs=get_exact_match_metric_specs(),
         groups=["CIMCQA"],
     )
+
+
+@run_spec_function("ewok")
+def get_ewok_spec(domain: str = "all") -> RunSpec:
+    scenario_spec = ScenarioSpec(
+        class_name="helm.benchmark.scenarios.ewok_scenario.EWoKScenario", args={"domain": domain}
+    )
+
+    instructions = """# INSTRUCTIONS
+
+In this study, you will see multiple examples. In each example, you will be given two contexts and a scenario. Your task is to read the two contexts and the subsequent scenario, and pick the context that makes more sense considering the scenario that follows. The contexts will be numbered "1" or "2". You must answer using "1" or "2" in your response.
+"""  # noqa: E501
+    input_prefix = """# TEST EXAMPLE
+
+## Scenario
+\""""
+    input_suffix = """\"
+
+## Contexts
+"""
+    output_prefix = """
+## Task
+Which context makes more sense given the scenario? Please answer using either "1" or "2".
+
+## Response
+"""
+
+    adapter_spec = AdapterSpec(
+        method=ADAPT_MULTIPLE_CHOICE_JOINT,
+        instructions=instructions,
+        input_prefix=input_prefix,
+        input_suffix=input_suffix,
+        reference_prefix='1. "',
+        reference_suffix='"\n',
+        output_prefix=output_prefix,
+        output_suffix="\n",
+        max_train_instances=2,
+        num_outputs=1,
+        max_tokens=2,
+        temperature=0.0,
+        stop_sequences=["\n\n"],
+        sample_train=True,
+    )
+
+    return RunSpec(
+        name=f"ewok:domain={domain}",
+        scenario_spec=scenario_spec,
+        adapter_spec=adapter_spec,
+        metric_specs=get_exact_match_metric_specs(),
+        groups=["ewok", f"ewok_{domain}"],
+    )

helm/benchmark/run_specs/finance_run_specs.py

@@ -2,15 +2,19 @@
 
 Website: https://crfm.stanford.edu/helm/finance/"""
 
+from helm.benchmark.adaptation.adapter_spec import ADAPT_GENERATION, AdapterSpec
 from helm.benchmark.adaptation.common_adapter_specs import (
     get_generation_adapter_spec,
 )
+from helm.benchmark.annotation.annotator import AnnotatorSpec
 from helm.benchmark.metrics.common_metric_specs import (
     get_basic_metric_specs,
+    get_exact_match_metric_specs,
 )
 from helm.benchmark.metrics.metric import MetricSpec
 from helm.benchmark.run_spec import RunSpec, run_spec_function
-from helm.benchmark.scenarios.scenario import ScenarioSpec
+from helm.benchmark.runner import get_benchmark_output_path
+from helm.benchmark.scenarios.scenario import ScenarioSpec, get_scenario_cache_path
 
 
 @run_spec_function("fin_qa")
@@ -31,3 +35,76 @@ def get_fin_qa_spec() -> RunSpec:
         metric_specs=metric_specs,
         groups=["fin_qa"],
     )
+
+
+@run_spec_function("financebench")
+def get_financebench_spec() -> RunSpec:
+    instructions = (
+        "Answer only the last question using the given evidence. "
+        "Respond with only a single paragraph, sentence or sentence fragment.\n"
+    )
+    scenario_spec = ScenarioSpec(
+        class_name="helm.benchmark.scenarios.financebench_scenario.FinanceBenchScenario", args={}
+    )
+    adapter_spec = AdapterSpec(
+        method=ADAPT_GENERATION,
+        instructions=instructions,
+        input_prefix="\n",
+        input_suffix="\n",
+        output_prefix="\nAnswer: ",
+        output_suffix="\n",
+        instance_prefix="\n###\n",
+        num_outputs=1,
+        max_tokens=300,
+        temperature=0.0,
+        stop_sequences=["###"],
+    )
+    annotator_specs = [
+        AnnotatorSpec(class_name="helm.benchmark.annotation.financebench_annotator.FinanceBenchAnnotator")
+    ]
+    metric_specs = get_basic_metric_specs([]) + [
+        MetricSpec(
+            class_name="helm.benchmark.metrics.annotation_metrics.AnnotationLabelMetric",
+            args={
+                "annotator_name": "financebench",
+                "key": "label",
+                "labels": ["correct_answer", "incorrect_answer", "failure_to_answer"],
+            },
+        )
+    ]
+    return RunSpec(
+        name="financebench",
+        scenario_spec=scenario_spec,
+        annotators=annotator_specs,
+        adapter_spec=adapter_spec,
+        metric_specs=metric_specs,
+        groups=["financebench"],
+    )
+
+
+@run_spec_function("banking77")
+def get_banking77_spec() -> RunSpec:
+    from helm.benchmark.scenarios.raft_scenario import get_raft_instructions
+    from helm.benchmark.scenarios.banking77_scenario import Banking77Scenario
+
+    scenario_spec = ScenarioSpec(class_name="helm.benchmark.scenarios.banking77_scenario.Banking77Scenario", args={})
+
+    # Use same AdapterSpec and instruction prompts as the RAFT implementation of BANKING77
+    scenario_cache_path = get_scenario_cache_path(get_benchmark_output_path(), Banking77Scenario.name)
+    adapter_spec = get_generation_adapter_spec(
+        instructions=get_raft_instructions("banking_77", scenario_cache_path),
+        input_noun=None,
+        output_noun="Label",
+        max_tokens=30,  # at most ~50 characters per label
+    )
+
+    # Not using get_classification_metric_specs() / ClassificationMetric because BANKING77 has too many classes,
+    # so F1 scores don't make sense. The original paper uses accuracy instead.
+    metric_specs = get_exact_match_metric_specs()
+    return RunSpec(
+        name="banking77",
+        scenario_spec=scenario_spec,
+        adapter_spec=adapter_spec,
+        metric_specs=metric_specs,
+        groups=["banking77"],
+    )

helm/benchmark/run_specs/safety_run_specs.py

@@ -0,0 +1,154 @@
+from helm.benchmark.adaptation.adapter_spec import ADAPT_GENERATION, AdapterSpec
+from helm.benchmark.annotation.annotator import AnnotatorSpec
+from helm.benchmark.run_spec import RunSpec, run_spec_function
+from helm.benchmark.scenarios.scenario import ScenarioSpec
+
+from helm.benchmark.metrics.metric import MetricSpec
+
+
+@run_spec_function("harm_bench")
+def get_harm_bench_spec() -> RunSpec:
+    adapter_spec = AdapterSpec(
+        method=ADAPT_GENERATION,
+        global_prefix="",
+        global_suffix="",
+        instructions="",
+        input_prefix="",
+        input_suffix="",
+        output_prefix="",
+        output_suffix="",
+        instance_prefix="",
+        max_train_instances=0,
+        num_outputs=1,
+        max_tokens=512,
+        temperature=0.0,
+        stop_sequences=[],
+    )
+    scenario_spec = ScenarioSpec(class_name="helm.benchmark.scenarios.harm_bench_scenario.HarmBenchScenario")
+    annotator_specs = [AnnotatorSpec(class_name="helm.benchmark.annotation.harm_bench_annotator.HarmBenchAnnotator")]
+    metric_specs = [
+        MetricSpec(class_name="helm.benchmark.metrics.safety_metrics.SafetyScoreMetric"),
+        MetricSpec(class_name="helm.benchmark.metrics.safety_metrics.SafetyBasicGenerationMetric"),
+        MetricSpec(class_name="helm.benchmark.metrics.basic_metrics.InstancesPerSplitMetric"),
+    ]
+    return RunSpec(
+        name="harm_bench",
+        scenario_spec=scenario_spec,
+        adapter_spec=adapter_spec,
+        metric_specs=metric_specs,
+        annotators=annotator_specs,
+        groups=["harm_bench"],
+    )
+
+
+@run_spec_function("simple_safety_tests")
+def get_simple_safety_tests_spec() -> RunSpec:
+    adapter_spec = AdapterSpec(
+        method=ADAPT_GENERATION,
+        global_prefix="",
+        global_suffix="",
+        instructions="",
+        input_prefix="",
+        input_suffix="",
+        output_prefix="",
+        output_suffix="",
+        instance_prefix="",
+        max_train_instances=0,
+        num_outputs=1,
+        max_tokens=512,
+        temperature=0.0,
+        stop_sequences=[],
+    )
+    scenario_spec = ScenarioSpec(
+        class_name="helm.benchmark.scenarios.simple_safety_tests_scenario.SimpleSafetyTestsScenario"
+    )
+    annotator_specs = [
+        AnnotatorSpec(class_name="helm.benchmark.annotation.simple_safety_tests_annotator.SimpleSafetyTestsAnnotator")
+    ]
+    metric_specs = [
+        MetricSpec(class_name="helm.benchmark.metrics.safety_metrics.SafetyScoreMetric"),
+        MetricSpec(class_name="helm.benchmark.metrics.safety_metrics.SafetyBasicGenerationMetric"),
+        MetricSpec(class_name="helm.benchmark.metrics.basic_metrics.InstancesPerSplitMetric"),
+    ]
+    return RunSpec(
+        name="simple_safety_tests",
+        scenario_spec=scenario_spec,
+        adapter_spec=adapter_spec,
+        metric_specs=metric_specs,
+        annotators=annotator_specs,
+        groups=["simple_safety_tests"],
+    )
+
+
+@run_spec_function("xstest")
+def get_xstest_spec() -> RunSpec:
+    adapter_spec = AdapterSpec(
+        method=ADAPT_GENERATION,
+        global_prefix="",
+        global_suffix="",
+        instructions="",
+        input_prefix="",
+        input_suffix="",
+        output_prefix="",
+        output_suffix="",
+        instance_prefix="",
+        max_train_instances=0,
+        num_outputs=1,
+        max_tokens=512,
+        temperature=0.0,
+        stop_sequences=[],
+    )
+    scenario_spec = ScenarioSpec(class_name="helm.benchmark.scenarios.xstest_scenario.XSTestScenario")
+    annotator_specs = [AnnotatorSpec(class_name="helm.benchmark.annotation.xstest_annotator.XSTestAnnotator")]
+    metric_specs = [
+        MetricSpec(class_name="helm.benchmark.metrics.safety_metrics.SafetyScoreMetric"),
+        MetricSpec(class_name="helm.benchmark.metrics.safety_metrics.SafetyBasicGenerationMetric"),
+        MetricSpec(class_name="helm.benchmark.metrics.basic_metrics.InstancesPerSplitMetric"),
+    ]
+    return RunSpec(
+        name="xstest",
+        scenario_spec=scenario_spec,
+        adapter_spec=adapter_spec,
+        metric_specs=metric_specs,
+        annotators=annotator_specs,
+        groups=["xstest"],
+    )
+
+
+@run_spec_function("anthropic_red_team")
+def get_anthropic_red_team_spec() -> RunSpec:
+    adapter_spec = AdapterSpec(
+        method=ADAPT_GENERATION,
+        global_prefix="",
+        global_suffix="",
+        instructions="",
+        input_prefix="",
+        input_suffix="",
+        output_prefix="",
+        output_suffix="",
+        instance_prefix="",
+        max_train_instances=0,
+        num_outputs=1,
+        max_tokens=512,
+        temperature=0.0,
+        stop_sequences=[],
+    )
+    scenario_spec = ScenarioSpec(
+        class_name="helm.benchmark.scenarios.anthropic_red_team_scenario.AnthropicRedTeamScenario"
+    )
+    annotator_specs = [
+        AnnotatorSpec(class_name="helm.benchmark.annotation.anthropic_red_team_annotator.AnthropicRedTeamAnnotator")
+    ]
+    metric_specs = [
+        MetricSpec(class_name="helm.benchmark.metrics.safety_metrics.SafetyScoreMetric"),
+        MetricSpec(class_name="helm.benchmark.metrics.safety_metrics.SafetyBasicGenerationMetric"),
+        MetricSpec(class_name="helm.benchmark.metrics.basic_metrics.InstancesPerSplitMetric"),
+    ]
+    return RunSpec(
+        name="anthropic_red_team",
+        scenario_spec=scenario_spec,
+        adapter_spec=adapter_spec,
+        metric_specs=metric_specs,
+        annotators=annotator_specs,
+        groups=["anthropic_red_team"],
+    )