crfm-helm 0.5.3__py3-none-any.whl → 0.5.5__py3-none-any.whl

helm/benchmark/data_overlap/data_overlap_spec.py

@@ -1,86 +0,0 @@
-from dataclasses import dataclass
-from typing import List, Tuple
-
-try:
-    from light_scenario import LightScenarioKey
-except Exception:
-    from helm.benchmark.data_overlap.light_scenario import LightScenarioKey
-
-
-@dataclass(frozen=True)
-class GroupOverlapStats:
-    """
-    Dataclass that represents group data overlap stats
-    e.g.
-    {
-        "group": "natural_qa_closedbook",
-        "num_instances": 2144,
-        "num_overlapping_inputs": 1,
-        "num_overlapping_references": 100
-    }
-    """
-
-    group: str
-
-    num_instances: int
-
-    num_overlapping_inputs: int
-
-    num_overlapping_references: int
-
-    @property
-    def overlapping_input_ratio(self):
-        return self.num_overlapping_inputs / self.num_instances
-
-    @property
-    def overlapping_reference_ratio(self):
-        return self.num_overlapping_references / self.num_instances
-
-
-@dataclass(frozen=True)
-class OverlapProtocolSpec:
-    """Specification for how we compute overlap"""
-
-    # the N of the n_grams we're running
-    n: int
-
-
-@dataclass(frozen=True)
-class DataOverlapStatsKey:
-    """Dataclass that represents output data overlap stats"""
-
-    light_scenario_key: LightScenarioKey
-
-    overlap_protocol_spec: OverlapProtocolSpec
-
-
-@dataclass(frozen=True)
-class DataOverlapStats:
-    """Dataclass that represents output data overlap stats"""
-
-    data_overlap_stats_key: DataOverlapStatsKey
-
-    num_instances: int
-
-    instance_ids_with_overlapping_input: List[str]
-
-    instance_ids_with_overlapping_reference: List[str]
-
-
-@dataclass(frozen=True)
-class EntryDataOverlapKey:
-    """Unique key representing either the input or references of a single instance in a scenario."""
-
-    stats_key: DataOverlapStatsKey
-    part: str
-    """Either PART_INPUT or PART_REF"""
-    instance_id: str
-
-
-@dataclass(frozen=True)
-class EntryOverlapNgrams:
-    """Dataclass that represents output data overlap stats"""
-
-    entry_data_overlap_key: EntryDataOverlapKey
-
-    overlapping_ngram_counts: List[Tuple[str, int]]

helm/benchmark/data_overlap/export_scenario_text.py

@@ -1,119 +0,0 @@
-import json
-import os
-import argparse
-from typing import List, DefaultDict, Set
-from collections import defaultdict
-
-from helm.common.general import asdict_without_nones, ensure_directory_exists
-from helm.common.hierarchical_logger import hlog, htrack_block
-
-from helm.benchmark.scenarios.scenario import (
-    Scenario,
-    Instance,
-    create_scenario,
-    TRAIN_SPLIT,
-    VALID_SPLIT,
-    TEST_SPLIT,
-    ScenarioSpec,
-    with_instance_ids,
-)
-from helm.benchmark.presentation.run_entry import read_run_entries
-from helm.benchmark.run import run_entries_to_run_specs
-from helm.benchmark.data_overlap.light_scenario import LightInstance, LightScenario, LightScenarioKey
-
-
-def create_light_instance_from_instance(instance: Instance) -> LightInstance:
-    """Create a LightInstance given an Instance. Only keep the text attributes."""
-    input_text: str = instance.input.text
-    reference_texts: List[str] = [reference.output.text for reference in instance.references]
-    return LightInstance(input=input_text, references=reference_texts, id=instance.id)
-
-
-def get_light_scenarios_from_scenario_spec(
-    scenario_spec: ScenarioSpec, scenario_download_path: str = "exported_scenarios"
-) -> List[LightScenario]:
-    """
-    Create a list of LightInstances given a ScenarioSpec. Only keep the text of the input and references.
-    Note that one LightScenario object is created for each split of the Scenario for simplification.
-    """
-
-    scenario: Scenario = create_scenario(scenario_spec)
-
-    ensure_directory_exists(scenario_download_path)
-    scenario_output_path = os.path.join(scenario_download_path, scenario.name)
-    ensure_directory_exists(scenario_output_path)
-
-    # Load instances
-    instances: List[Instance]
-    with htrack_block("scenario.get_instances"):
-        instances = scenario.get_instances(scenario_output_path)
-
-    # Get instance ids
-    instances = with_instance_ids(instances)
-
-    # Classify instances into splits
-    splits: List[str] = [TRAIN_SPLIT, VALID_SPLIT, TEST_SPLIT]
-    split_mapping: DefaultDict[str, list] = defaultdict(list)
-    for instance in instances:
-        if instance.split is None or instance.split not in splits:
-            raise ValueError(
-                f"split should be one of {TRAIN_SPLIT}, {VALID_SPLIT}, or {TEST_SPLIT}, but got {instance.split}"
-            )
-        split_mapping[instance.split].append(instance)
-
-    # Convert Scenarios to LightScenarios
-    light_scenarios: List[LightScenario] = []
-    for split, instances in split_mapping.items():
-        light_instances: List[LightInstance] = [create_light_instance_from_instance(instance) for instance in instances]
-        light_scenario_key: LightScenarioKey = LightScenarioKey(
-            scenario_spec=scenario_spec,
-            split=split,
-        )
-        light_scenario = LightScenario(
-            scenario_key=light_scenario_key,
-            instances=light_instances,
-        )
-        light_scenarios.append(light_scenario)
-    return light_scenarios
-
-
-def save_scenarios_to_jsonl(light_scenarios: List[LightScenario], filename: str):
-    """
-    Save a list of LightInstance to a jsonl file where each line represents a LightScenario object.
-    """
-    with open(filename, "a") as f:
-        for light_scenario in light_scenarios:
-            f.write(json.dumps(asdict_without_nones(light_scenario), ensure_ascii=False) + "\n")
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--run-specs", nargs="+", required=True, help="Specifies what to export")
-    parser.add_argument("--output-data", type=str, required=True, help="The path to the output file")
-    args = parser.parse_args()
-
-    hlog("Loading run_specs")
-    run_entries = read_run_entries(args.run_specs).entries
-    run_specs = run_entries_to_run_specs(
-        run_entries=run_entries,
-        priority=4,
-    )
-
-    try:
-        os.remove(args.output_data)
-    except OSError:
-        pass
-
-    scenario_specs: Set = set()
-    for run_spec in run_specs:
-        scenario_spec = run_spec.scenario_spec
-        if (
-            scenario_spec.class_name
-            != "helm.benchmark.scenarios.synthetic_efficiency_scenario.SyntheticEfficiencyScenario"
-        ):
-            scenario_specs.add(scenario_spec)
-
-    hlog("Generating light scenarios from scenarios")
-    for scenario_spec in scenario_specs:
-        light_scenarios: List[LightScenario] = get_light_scenarios_from_scenario_spec(scenario_spec)
-        save_scenarios_to_jsonl(light_scenarios, args.output_data)

helm/benchmark/data_overlap/light_scenario.py

@@ -1,60 +0,0 @@
-from dataclasses import dataclass
-from typing import List, Optional
-
-try:
-    from scenarios.scenario import ScenarioSpec
-except Exception:
-    from helm.benchmark.scenarios.scenario import ScenarioSpec
-
-
-@dataclass(frozen=True)
-class LightInstance:
-    """
-    A lighter `Instance` with only text fields.
-    """
-
-    input: str
-    """The input"""
-
-    references: List[str]
-    """References that help us evaluate"""
-
-    id: Optional[str] = None
-    """Helm instance id"""
-
-
-@dataclass(frozen=True)
-class LightScenarioKey:
-    """
-    Key for LightScenario
-    """
-
-    scenario_spec: ScenarioSpec
-
-    split: str
-
-    def __hash__(self):
-        return hash((self.scenario_spec, self.split))
-
-
-@dataclass(frozen=True)
-class LightScenario:
-    """
-    A lighter `Scenario`.
-    """
-
-    scenario_key: LightScenarioKey
-
-    instances: List[LightInstance]
-    """Instances of this scenario"""
-
-
-@dataclass(frozen=True)
-class ScenarioSpecInstanceIds:
-    """
-    Instance ids associated with a scenario
-    """
-
-    scenario_spec: ScenarioSpec
-
-    instance_ids: List[str]

helm/benchmark/metrics/bhasa_metrics_specs.py

@@ -1,10 +0,0 @@
-from typing import Any, Dict, List
-from helm.benchmark.metrics.metric import MetricSpec
-
-
-def get_bhasa_machine_translation_metric_specs() -> List[MetricSpec]:
-    return [MetricSpec(class_name="helm.benchmark.metrics.bhasa_metrics.BhasaMachineTranslationMetric")]
-
-
-def get_bhasa_qa_metric_specs(args: Dict[str, Any]) -> List[MetricSpec]:
-    return [MetricSpec(class_name="helm.benchmark.metrics.bhasa_metrics.BhasaQAMetric", args=args)]