crfm-helm 0.5.2__py3-none-any.whl → 0.5.4__py3-none-any.whl

helm/tokenizers/test_ai21_tokenizer.py

@@ -0,0 +1,48 @@
+import pytest
+
+from helm.common.cache import BlackHoleCacheConfig
+from helm.common.tokenization_request import (
+    DecodeRequest,
+    TokenizationRequest,
+    TokenizationToken,
+)
+
+
+@pytest.mark.models
+def test_tokenize():
+    from helm.tokenizers.ai21_tokenizer import AI21LocalTokenizer
+
+    tokenizer = AI21LocalTokenizer(cache_config=BlackHoleCacheConfig())
+    request = TokenizationRequest(tokenizer="ai21/jamba-instruct-tokenizer", text="otter 🦦")
+    result = tokenizer.tokenize(request)
+    assert result.success
+    assert not result.cached
+    assert result.tokens == [
+        TokenizationToken(token) for token in ["ot", "ter", "▁", "<0xF0>", "<0x9F>", "<0xA6>", "<0xA6>"]
+    ]
+
+
+@pytest.mark.models
+def test_encode():
+    from helm.tokenizers.ai21_tokenizer import AI21LocalTokenizer
+
+    tokenizer = AI21LocalTokenizer(cache_config=BlackHoleCacheConfig())
+    request = TokenizationRequest(tokenizer="ai21/jamba-instruct-tokenizer", text="otter 🦦", encode=True)
+    result = tokenizer.tokenize(request)
+    assert result.success
+    assert not result.cached
+    assert result.tokens == [TokenizationToken(token) for token in [1860, 1901, 62934, 1784, 1703, 1710, 1710]]
+
+
+@pytest.mark.models
+def test_decode():
+    from helm.tokenizers.ai21_tokenizer import AI21LocalTokenizer
+
+    tokenizer = AI21LocalTokenizer(cache_config=BlackHoleCacheConfig())
+    request = DecodeRequest(
+        tokenizer="ai21/jamba-instruct-tokenizer", tokens=[1860, 1901, 62934, 1784, 1703, 1710, 1710]
+    )
+    result = tokenizer.decode(request)
+    assert result.success
+    assert not result.cached
+    assert result.text == "otter 🦦"

helm/benchmark/data_overlap/data_overlap_spec.py

@@ -1,86 +0,0 @@
-from dataclasses import dataclass
-from typing import List, Tuple
-
-try:
-    from light_scenario import LightScenarioKey
-except Exception:
-    from helm.benchmark.data_overlap.light_scenario import LightScenarioKey
-
-
-@dataclass(frozen=True)
-class GroupOverlapStats:
-    """
-    Dataclass that represents group data overlap stats
-    e.g.
-    {
-        "group": "natural_qa_closedbook",
-        "num_instances": 2144,
-        "num_overlapping_inputs": 1,
-        "num_overlapping_references": 100
-    }
-    """
-
-    group: str
-
-    num_instances: int
-
-    num_overlapping_inputs: int
-
-    num_overlapping_references: int
-
-    @property
-    def overlapping_input_ratio(self):
-        return self.num_overlapping_inputs / self.num_instances
-
-    @property
-    def overlapping_reference_ratio(self):
-        return self.num_overlapping_references / self.num_instances
-
-
-@dataclass(frozen=True)
-class OverlapProtocolSpec:
-    """Specification for how we compute overlap"""
-
-    # the N of the n_grams we're running
-    n: int
-
-
-@dataclass(frozen=True)
-class DataOverlapStatsKey:
-    """Dataclass that represents output data overlap stats"""
-
-    light_scenario_key: LightScenarioKey
-
-    overlap_protocol_spec: OverlapProtocolSpec
-
-
-@dataclass(frozen=True)
-class DataOverlapStats:
-    """Dataclass that represents output data overlap stats"""
-
-    data_overlap_stats_key: DataOverlapStatsKey
-
-    num_instances: int
-
-    instance_ids_with_overlapping_input: List[str]
-
-    instance_ids_with_overlapping_reference: List[str]
-
-
-@dataclass(frozen=True)
-class EntryDataOverlapKey:
-    """Unique key representing either the input or references of a single instance in a scenario."""
-
-    stats_key: DataOverlapStatsKey
-    part: str
-    """Either PART_INPUT or PART_REF"""
-    instance_id: str
-
-
-@dataclass(frozen=True)
-class EntryOverlapNgrams:
-    """Dataclass that represents output data overlap stats"""
-
-    entry_data_overlap_key: EntryDataOverlapKey
-
-    overlapping_ngram_counts: List[Tuple[str, int]]

helm/benchmark/data_overlap/export_scenario_text.py

@@ -1,119 +0,0 @@
-import json
-import os
-import argparse
-from typing import List, DefaultDict, Set
-from collections import defaultdict
-
-from helm.common.general import asdict_without_nones, ensure_directory_exists
-from helm.common.hierarchical_logger import hlog, htrack_block
-
-from helm.benchmark.scenarios.scenario import (
-    Scenario,
-    Instance,
-    create_scenario,
-    TRAIN_SPLIT,
-    VALID_SPLIT,
-    TEST_SPLIT,
-    ScenarioSpec,
-    with_instance_ids,
-)
-from helm.benchmark.presentation.run_entry import read_run_entries
-from helm.benchmark.run import run_entries_to_run_specs
-from helm.benchmark.data_overlap.light_scenario import LightInstance, LightScenario, LightScenarioKey
-
-
-def create_light_instance_from_instance(instance: Instance) -> LightInstance:
-    """Create a LightInstance given an Instance. Only keep the text attributes."""
-    input_text: str = instance.input.text
-    reference_texts: List[str] = [reference.output.text for reference in instance.references]
-    return LightInstance(input=input_text, references=reference_texts, id=instance.id)
-
-
-def get_light_scenarios_from_scenario_spec(
-    scenario_spec: ScenarioSpec, scenario_download_path: str = "exported_scenarios"
-) -> List[LightScenario]:
-    """
-    Create a list of LightInstances given a ScenarioSpec. Only keep the text of the input and references.
-    Note that one LightScenario object is created for each split of the Scenario for simplification.
-    """
-
-    scenario: Scenario = create_scenario(scenario_spec)
-
-    ensure_directory_exists(scenario_download_path)
-    scenario_output_path = os.path.join(scenario_download_path, scenario.name)
-    ensure_directory_exists(scenario_output_path)
-
-    # Load instances
-    instances: List[Instance]
-    with htrack_block("scenario.get_instances"):
-        instances = scenario.get_instances(scenario_output_path)
-
-    # Get instance ids
-    instances = with_instance_ids(instances)
-
-    # Classify instances into splits
-    splits: List[str] = [TRAIN_SPLIT, VALID_SPLIT, TEST_SPLIT]
-    split_mapping: DefaultDict[str, list] = defaultdict(list)
-    for instance in instances:
-        if instance.split is None or instance.split not in splits:
-            raise ValueError(
-                f"split should be one of {TRAIN_SPLIT}, {VALID_SPLIT}, or {TEST_SPLIT}, but got {instance.split}"
-            )
-        split_mapping[instance.split].append(instance)
-
-    # Convert Scenarios to LightScenarios
-    light_scenarios: List[LightScenario] = []
-    for split, instances in split_mapping.items():
-        light_instances: List[LightInstance] = [create_light_instance_from_instance(instance) for instance in instances]
-        light_scenario_key: LightScenarioKey = LightScenarioKey(
-            scenario_spec=scenario_spec,
-            split=split,
-        )
-        light_scenario = LightScenario(
-            scenario_key=light_scenario_key,
-            instances=light_instances,
-        )
-        light_scenarios.append(light_scenario)
-    return light_scenarios
-
-
-def save_scenarios_to_jsonl(light_scenarios: List[LightScenario], filename: str):
-    """
-    Save a list of LightInstance to a jsonl file where each line represents a LightScenario object.
-    """
-    with open(filename, "a") as f:
-        for light_scenario in light_scenarios:
-            f.write(json.dumps(asdict_without_nones(light_scenario), ensure_ascii=False) + "\n")
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--run-specs", nargs="+", required=True, help="Specifies what to export")
-    parser.add_argument("--output-data", type=str, required=True, help="The path to the output file")
-    args = parser.parse_args()
-
-    hlog("Loading run_specs")
-    run_entries = read_run_entries(args.run_specs).entries
-    run_specs = run_entries_to_run_specs(
-        run_entries=run_entries,
-        priority=4,
-    )
-
-    try:
-        os.remove(args.output_data)
-    except OSError:
-        pass
-
-    scenario_specs: Set = set()
-    for run_spec in run_specs:
-        scenario_spec = run_spec.scenario_spec
-        if (
-            scenario_spec.class_name
-            != "helm.benchmark.scenarios.synthetic_efficiency_scenario.SyntheticEfficiencyScenario"
-        ):
-            scenario_specs.add(scenario_spec)
-
-    hlog("Generating light scenarios from scenarios")
-    for scenario_spec in scenario_specs:
-        light_scenarios: List[LightScenario] = get_light_scenarios_from_scenario_spec(scenario_spec)
-        save_scenarios_to_jsonl(light_scenarios, args.output_data)

helm/benchmark/data_overlap/light_scenario.py

@@ -1,60 +0,0 @@
-from dataclasses import dataclass
-from typing import List, Optional
-
-try:
-    from scenarios.scenario import ScenarioSpec
-except Exception:
-    from helm.benchmark.scenarios.scenario import ScenarioSpec
-
-
-@dataclass(frozen=True)
-class LightInstance:
-    """
-    A lighter `Instance` with only text fields.
-    """
-
-    input: str
-    """The input"""
-
-    references: List[str]
-    """References that help us evaluate"""
-
-    id: Optional[str] = None
-    """Helm instance id"""
-
-
-@dataclass(frozen=True)
-class LightScenarioKey:
-    """
-    Key for LightScenario
-    """
-
-    scenario_spec: ScenarioSpec
-
-    split: str
-
-    def __hash__(self):
-        return hash((self.scenario_spec, self.split))
-
-
-@dataclass(frozen=True)
-class LightScenario:
-    """
-    A lighter `Scenario`.
-    """
-
-    scenario_key: LightScenarioKey
-
-    instances: List[LightInstance]
-    """Instances of this scenario"""
-
-
-@dataclass(frozen=True)
-class ScenarioSpecInstanceIds:
-    """
-    Instance ids associated with a scenario
-    """
-
-    scenario_spec: ScenarioSpec
-
-    instance_ids: List[str]

helm/benchmark/static/benchmarking.css

@@ -1,156 +0,0 @@
-.correct {
-  background-color: #dfffdf;
-}
-
-.wrong {
-  background-color: #ffdfdf;
-}
-
-.scenario-info {
-  margin-top: 30px;
-  margin-bottom: 30px;
-}
-
-td {
-  padding-left: 15px;
-  padding-right: 15px;
-  padding-top: 5px;
-  padding-bottom: 5px;
-  word-wrap: break-word;
-  max-width: 900px;
-  vertical-align: top;
-}
-
-.results-table {
-}
-
-.table-container {
-  margin-top: 30px;
-  margin-bottom: 30px;
-}
-
-tr {
-  border: solid;
-  border-color: #f0f0f0;
-  border-width: 1px 0;
-}
-
-.results-table thead tr {
-  background-color: #f9f9f9;
-}
-
-.logprob {
-  font-size: 8pt;
-  font-style: italic;
-  color: gray;
-}
-
-.list-header {
-  font-size: 24px;
-  font-weight: bold;
-}
-.list-item {
-  color: black;
-  font-size: 14px;
-  white-space: nowrap;
-}
-.list-item-todo {
-  color: lightgray;
-}
-.list-item:hover {
-  color: black;
-  text-decoration: none;
-  background-color: lightgray;
-}
-
-.main-link {
-  color: white;
-  background-color: #53A0C0;
-}
-.main-link:hover {
-  color: lightgray;
-}
-
-.access-open {
-  background-color: lightgreen;
-  width: 100px;
-}
-.access-limited {
-  background-color: yellow;
-  width: 100px;
-}
-.access-restricted {
-  background-color: orange;
-  width: 100px;
-}
-.access-closed {
-  background-color: lightgray;
-  width: 100px;
-}
-
-.technical-details {
-  font-size: 10px;
-  font-style: italic;
-  color: gray;
-}
-
-.logo-container {
-  display: flex;
-  flex-flow: row wrap;
-  justify-content: space-between;
-  padding: 20px;
-}
-
-.logo-item {
-  margin: auto;
-  padding: 10px;
-}
-
-.instance-input {
-  font-style: italic;
-  background-color: #f5f5f5;
-  margin-left: 20px;
-  white-space: pre-wrap;
-}
-
-.instance-reference {
-  font-style: italic;
-  background-color: #f5f5f5;
-  white-space: pre-wrap;
-}
-
-.taxonomy-table {
-  margin: 10px;
-}
-
-thead .table-sort-column {
-  background-color: #ffe599;
-}
-
-tbody .table-sort-column {
-  background-color: #fff2cc;
-}
-
-.prompt {
-  font-style: italic;
-  background-color: #f5f5f5;
-  white-space: pre-wrap;
-}
-
-.plot {
-  margin: 15px;
-}
-
-.plot img {
-  margin: 10px;
-}
-
-.plot-caption {
-  color: #555;
-  font-style: italic;
-  margin: 5px;
-}
-
-.prediction-text {
-  white-space: pre-wrap;
-}