crfm-helm 0.5.3__py3-none-any.whl → 0.5.4__py3-none-any.whl

helm/config/tokenizer_configs.yaml

@@ -317,6 +317,14 @@ tokenizer_configs:
     prefix_token: "<|begin_of_text|>"
     end_of_text_token: "<|end_of_text|>"
 
+  - name: meta/llama-3-8b-instruct
+    tokenizer_spec:
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+      args:
+        pretrained_model_name_or_path: meta-llama/Meta-Llama-3.1-8B-Instruct
+    prefix_token: "<|begin_of_text|>"
+    end_of_text_token: "<|eot_id|>"
+
   - name: meta/llama-3.1-8b
     tokenizer_spec:
       class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
@@ -325,6 +333,22 @@ tokenizer_configs:
     prefix_token: "<|begin_of_text|>"
     end_of_text_token: "<|end_of_text|>"
 
+  - name: meta/llama-3.2-3b-instruct
+    tokenizer_spec:
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+      args:
+        pretrained_model_name_or_path: meta-llama/Llama-3.2-3B-Instruct
+    prefix_token: "<|begin_of_text|>"
+    end_of_text_token: "<|eot_id|>"
+
+  - name: meta/llama-3.2-11b-vision-instruct
+    tokenizer_spec:
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+      args:
+        pretrained_model_name_or_path: meta-llama/Llama-3.2-11B-Vision-Instruct
+    prefix_token: "<|begin_of_text|>"
+    end_of_text_token: "<|eot_id|>"
+
   # 01-ai
   - name: 01-ai/Yi-6B
     tokenizer_spec:

helm/proxy/server.py

@@ -106,15 +106,6 @@ def handle_get_general_info():
     return safe_call(perform)
 
 
-@app.get("/api/window_service_info")
-def handle_get_window_service_info():
-    def perform(args):
-        global service
-        return dataclasses.asdict(service.get_window_service_info(args["model_name"]))
-
-    return safe_call(perform)
-
-
 @app.post("/api/account")
 def handle_create_account():
     def perform(args):

helm/proxy/services/remote_service.py

@@ -15,7 +15,6 @@ from helm.common.file_upload_request import FileUploadRequest, FileUploadResult
 from helm.common.perspective_api_request import PerspectiveAPIRequest, PerspectiveAPIRequestResult
 from helm.common.clip_score_request import CLIPScoreRequest, CLIPScoreResult
 from helm.common.tokenization_request import (
-    WindowServiceInfo,
     TokenizationRequest,
     TokenizationRequestResult,
     DecodeRequestResult,
@@ -51,11 +50,6 @@ class RemoteService(Service):
         response = requests.get(f"{self.base_url}/api/general_info").json()
         return from_dict(GeneralInfo, response)
 
-    def get_window_service_info(self, model_name) -> WindowServiceInfo:
-        params = {"model_name": model_name}
-        response = requests.get(f"{self.base_url}/api/window_service_info?{urllib.parse.urlencode(params)}").json()
-        return from_dict(WindowServiceInfo, response)
-
     def expand_query(self, query: Query) -> QueryResult:
         params = asdict(query)
         response = requests.get(f"{self.base_url}/api/query?{urllib.parse.urlencode(params)}").json()

helm/proxy/services/server_service.py

@@ -14,7 +14,6 @@ from helm.common.file_upload_request import FileUploadRequest, FileUploadResult
 from helm.common.general import ensure_directory_exists, parse_hocon, get_credentials
 from helm.common.perspective_api_request import PerspectiveAPIRequest, PerspectiveAPIRequestResult
 from helm.common.tokenization_request import (
-    WindowServiceInfo,
     TokenizationRequest,
     TokenizationRequestResult,
     DecodeRequest,
@@ -85,22 +84,6 @@ class ServerService(Service):
         all_models = [dataclasses.replace(model_metadata, release_date=None) for model_metadata in ALL_MODELS_METADATA]
         return GeneralInfo(version=VERSION, example_queries=example_queries, all_models=all_models)
 
-    def get_window_service_info(self, model_name) -> WindowServiceInfo:
-        # The import statement is placed here to avoid two problems, please refer to the link for details
-        # https://github.com/stanford-crfm/helm/pull/1430#discussion_r1156686624
-        from helm.benchmark.window_services.tokenizer_service import TokenizerService
-        from helm.benchmark.window_services.window_service_factory import WindowServiceFactory
-
-        token_service = TokenizerService(self, Authentication(""))
-        window_service = WindowServiceFactory.get_window_service(model_name, token_service)
-        return WindowServiceInfo(
-            tokenizer_name=window_service.tokenizer_name,
-            max_sequence_length=window_service.max_sequence_length,
-            max_request_length=window_service.max_request_length,
-            end_of_text_token=window_service.end_of_text_token,
-            prefix_token=window_service.prefix_token,
-        )
-
     def expand_query(self, query: Query) -> QueryResult:
         """Turn the `query` into requests."""
         prompt = query.prompt
@@ -120,8 +103,12 @@ class ServerService(Service):
                 return "dall_e"
             elif model_deployment.startswith("openai/gpt-4"):
                 return "gpt4"
-            else:
+            elif model_deployment.startswith("openai/gpt-3"):
                 return "gpt3"
+            elif model_deployment.startswith("openai/o1"):
+                return "o1"
+            else:
+                return "openai"
         elif model_deployment.startswith("ai21/"):
             return "jurassic"
         else:

helm/proxy/services/service.py

@@ -11,7 +11,6 @@ from helm.common.nudity_check_request import NudityCheckRequest, NudityCheckResu
 from helm.common.perspective_api_request import PerspectiveAPIRequestResult, PerspectiveAPIRequest
 from helm.common.moderations_api_request import ModerationAPIRequest, ModerationAPIRequestResult
 from helm.common.tokenization_request import (
-    WindowServiceInfo,
     TokenizationRequest,
     TokenizationRequestResult,
     DecodeRequest,
@@ -85,11 +84,6 @@ class Service(ABC):
         """Get general info."""
         pass
 
-    @abstractmethod
-    def get_window_service_info(self, model_name: str) -> WindowServiceInfo:
-        """Get window service info."""
-        pass
-
     @abstractmethod
     def expand_query(self, query: Query) -> QueryResult:
         """Turn the `query` into requests."""

helm/benchmark/data_overlap/data_overlap_spec.py

@@ -1,86 +0,0 @@
-from dataclasses import dataclass
-from typing import List, Tuple
-
-try:
-    from light_scenario import LightScenarioKey
-except Exception:
-    from helm.benchmark.data_overlap.light_scenario import LightScenarioKey
-
-
-@dataclass(frozen=True)
-class GroupOverlapStats:
-    """
-    Dataclass that represents group data overlap stats
-    e.g.
-    {
-        "group": "natural_qa_closedbook",
-        "num_instances": 2144,
-        "num_overlapping_inputs": 1,
-        "num_overlapping_references": 100
-    }
-    """
-
-    group: str
-
-    num_instances: int
-
-    num_overlapping_inputs: int
-
-    num_overlapping_references: int
-
-    @property
-    def overlapping_input_ratio(self):
-        return self.num_overlapping_inputs / self.num_instances
-
-    @property
-    def overlapping_reference_ratio(self):
-        return self.num_overlapping_references / self.num_instances
-
-
-@dataclass(frozen=True)
-class OverlapProtocolSpec:
-    """Specification for how we compute overlap"""
-
-    # the N of the n_grams we're running
-    n: int
-
-
-@dataclass(frozen=True)
-class DataOverlapStatsKey:
-    """Dataclass that represents output data overlap stats"""
-
-    light_scenario_key: LightScenarioKey
-
-    overlap_protocol_spec: OverlapProtocolSpec
-
-
-@dataclass(frozen=True)
-class DataOverlapStats:
-    """Dataclass that represents output data overlap stats"""
-
-    data_overlap_stats_key: DataOverlapStatsKey
-
-    num_instances: int
-
-    instance_ids_with_overlapping_input: List[str]
-
-    instance_ids_with_overlapping_reference: List[str]
-
-
-@dataclass(frozen=True)
-class EntryDataOverlapKey:
-    """Unique key representing either the input or references of a single instance in a scenario."""
-
-    stats_key: DataOverlapStatsKey
-    part: str
-    """Either PART_INPUT or PART_REF"""
-    instance_id: str
-
-
-@dataclass(frozen=True)
-class EntryOverlapNgrams:
-    """Dataclass that represents output data overlap stats"""
-
-    entry_data_overlap_key: EntryDataOverlapKey
-
-    overlapping_ngram_counts: List[Tuple[str, int]]

helm/benchmark/data_overlap/export_scenario_text.py

@@ -1,119 +0,0 @@
-import json
-import os
-import argparse
-from typing import List, DefaultDict, Set
-from collections import defaultdict
-
-from helm.common.general import asdict_without_nones, ensure_directory_exists
-from helm.common.hierarchical_logger import hlog, htrack_block
-
-from helm.benchmark.scenarios.scenario import (
-    Scenario,
-    Instance,
-    create_scenario,
-    TRAIN_SPLIT,
-    VALID_SPLIT,
-    TEST_SPLIT,
-    ScenarioSpec,
-    with_instance_ids,
-)
-from helm.benchmark.presentation.run_entry import read_run_entries
-from helm.benchmark.run import run_entries_to_run_specs
-from helm.benchmark.data_overlap.light_scenario import LightInstance, LightScenario, LightScenarioKey
-
-
-def create_light_instance_from_instance(instance: Instance) -> LightInstance:
-    """Create a LightInstance given an Instance. Only keep the text attributes."""
-    input_text: str = instance.input.text
-    reference_texts: List[str] = [reference.output.text for reference in instance.references]
-    return LightInstance(input=input_text, references=reference_texts, id=instance.id)
-
-
-def get_light_scenarios_from_scenario_spec(
-    scenario_spec: ScenarioSpec, scenario_download_path: str = "exported_scenarios"
-) -> List[LightScenario]:
-    """
-    Create a list of LightInstances given a ScenarioSpec. Only keep the text of the input and references.
-    Note that one LightScenario object is created for each split of the Scenario for simplification.
-    """
-
-    scenario: Scenario = create_scenario(scenario_spec)
-
-    ensure_directory_exists(scenario_download_path)
-    scenario_output_path = os.path.join(scenario_download_path, scenario.name)
-    ensure_directory_exists(scenario_output_path)
-
-    # Load instances
-    instances: List[Instance]
-    with htrack_block("scenario.get_instances"):
-        instances = scenario.get_instances(scenario_output_path)
-
-    # Get instance ids
-    instances = with_instance_ids(instances)
-
-    # Classify instances into splits
-    splits: List[str] = [TRAIN_SPLIT, VALID_SPLIT, TEST_SPLIT]
-    split_mapping: DefaultDict[str, list] = defaultdict(list)
-    for instance in instances:
-        if instance.split is None or instance.split not in splits:
-            raise ValueError(
-                f"split should be one of {TRAIN_SPLIT}, {VALID_SPLIT}, or {TEST_SPLIT}, but got {instance.split}"
-            )
-        split_mapping[instance.split].append(instance)
-
-    # Convert Scenarios to LightScenarios
-    light_scenarios: List[LightScenario] = []
-    for split, instances in split_mapping.items():
-        light_instances: List[LightInstance] = [create_light_instance_from_instance(instance) for instance in instances]
-        light_scenario_key: LightScenarioKey = LightScenarioKey(
-            scenario_spec=scenario_spec,
-            split=split,
-        )
-        light_scenario = LightScenario(
-            scenario_key=light_scenario_key,
-            instances=light_instances,
-        )
-        light_scenarios.append(light_scenario)
-    return light_scenarios
-
-
-def save_scenarios_to_jsonl(light_scenarios: List[LightScenario], filename: str):
-    """
-    Save a list of LightInstance to a jsonl file where each line represents a LightScenario object.
-    """
-    with open(filename, "a") as f:
-        for light_scenario in light_scenarios:
-            f.write(json.dumps(asdict_without_nones(light_scenario), ensure_ascii=False) + "\n")
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--run-specs", nargs="+", required=True, help="Specifies what to export")
-    parser.add_argument("--output-data", type=str, required=True, help="The path to the output file")
-    args = parser.parse_args()
-
-    hlog("Loading run_specs")
-    run_entries = read_run_entries(args.run_specs).entries
-    run_specs = run_entries_to_run_specs(
-        run_entries=run_entries,
-        priority=4,
-    )
-
-    try:
-        os.remove(args.output_data)
-    except OSError:
-        pass
-
-    scenario_specs: Set = set()
-    for run_spec in run_specs:
-        scenario_spec = run_spec.scenario_spec
-        if (
-            scenario_spec.class_name
-            != "helm.benchmark.scenarios.synthetic_efficiency_scenario.SyntheticEfficiencyScenario"
-        ):
-            scenario_specs.add(scenario_spec)
-
-    hlog("Generating light scenarios from scenarios")
-    for scenario_spec in scenario_specs:
-        light_scenarios: List[LightScenario] = get_light_scenarios_from_scenario_spec(scenario_spec)
-        save_scenarios_to_jsonl(light_scenarios, args.output_data)

helm/benchmark/data_overlap/light_scenario.py

@@ -1,60 +0,0 @@
-from dataclasses import dataclass
-from typing import List, Optional
-
-try:
-    from scenarios.scenario import ScenarioSpec
-except Exception:
-    from helm.benchmark.scenarios.scenario import ScenarioSpec
-
-
-@dataclass(frozen=True)
-class LightInstance:
-    """
-    A lighter `Instance` with only text fields.
-    """
-
-    input: str
-    """The input"""
-
-    references: List[str]
-    """References that help us evaluate"""
-
-    id: Optional[str] = None
-    """Helm instance id"""
-
-
-@dataclass(frozen=True)
-class LightScenarioKey:
-    """
-    Key for LightScenario
-    """
-
-    scenario_spec: ScenarioSpec
-
-    split: str
-
-    def __hash__(self):
-        return hash((self.scenario_spec, self.split))
-
-
-@dataclass(frozen=True)
-class LightScenario:
-    """
-    A lighter `Scenario`.
-    """
-
-    scenario_key: LightScenarioKey
-
-    instances: List[LightInstance]
-    """Instances of this scenario"""
-
-
-@dataclass(frozen=True)
-class ScenarioSpecInstanceIds:
-    """
-    Instance ids associated with a scenario
-    """
-
-    scenario_spec: ScenarioSpec
-
-    instance_ids: List[str]