crfm-helm 0.3.0__py3-none-any.whl → 0.5.0__py3-none-any.whl

helm/benchmark/runner.py

@@ -6,70 +6,69 @@ import traceback
 import typing
 from collections import Counter
 import dataclasses
-from dataclasses import dataclass, field
 from typing import Any, Dict, List
+import numpy as np
 
 from tqdm import tqdm
 
+from helm.benchmark.adaptation.request_state import RequestState
 from helm.common.general import ensure_directory_exists, write, asdict_without_nones
 from helm.common.hierarchical_logger import hlog, htrack_block
 from helm.common.cache import cache_stats
-from .augmentations.data_augmenter import DataAugmenterSpec
-from .scenarios.scenario import Scenario, ScenarioSpec, create_scenario, Instance, with_instance_ids
-from .adaptation.adapters.adapter import Adapter
-from .adaptation.adapters.adapter_factory import AdapterFactory
-from .adaptation.scenario_state import ScenarioState
-from .adaptation.adapter_spec import AdapterSpec
-from .data_preprocessor import DataPreprocessor
-from .executor import ExecutionSpec, Executor
-from .metrics.dry_run_metrics import DryRunMetric
-from .metrics.metric_name import MetricName
-from .metrics.metric_service import MetricService
-from .metrics.metric import Metric, MetricSpec, MetricResult, PerInstanceStats, create_metric, Stat
-from .window_services.tokenizer_service import TokenizerService
+from helm.benchmark.scenarios.scenario import (
+    EVAL_SPLITS,
+    TRAIN_SPLIT,
+    Scenario,
+    create_scenario,
+    Instance,
+    get_scenario_cache_path,
+    with_instance_ids,
+)
+from helm.benchmark.adaptation.adapters.adapter import Adapter
+from helm.benchmark.adaptation.adapters.adapter_factory import AdapterFactory
+from helm.benchmark.adaptation.scenario_state import ScenarioState
+from helm.benchmark.run_spec import RunSpec
+from helm.benchmark.data_preprocessor import DataPreprocessor
+from helm.benchmark.executor import ExecutionSpec, Executor
+from helm.benchmark.annotation_executor import AnnotationExecutionSpec, AnnotationExecutor
+from helm.benchmark.metrics.dry_run_metrics import DryRunMetric
+from helm.benchmark.metrics.metric_name import MetricName
+from helm.benchmark.metrics.metric_service import MetricService
+from helm.benchmark.metrics.metric import MetricInterface, MetricResult, PerInstanceStats, create_metric, Stat
+from helm.benchmark.window_services.tokenizer_service import TokenizerService
 
 
 LATEST_SYMLINK: str = "latest"
+_BENCHMARK_OUTPUT_PATH: str = "benchmark_output"
+_CACHED_MODELS_FOLDER: str = "models"
 
 
-class RunnerError(Exception):
-    """Error that happens in the Runner."""
-
-    pass
-
+def get_benchmark_output_path() -> str:
+    """Get the benchmark output path.
 
-@dataclass(frozen=True)
-class RunSpec:
-    """
-    Specifies how to do a single run, which gets a scenario, adapts it, and
-    computes a list of stats based on the defined metrics.
-    """
+    Many run spec functions need to know the benchmark output path,
+    but there is no way to pass it via  the run spec function,
+    so instead the run spec function should read this global variable."""
+    return _BENCHMARK_OUTPUT_PATH
 
-    # Unique identifier of the RunSpec
-    name: str
 
-    # Which scenario
-    scenario_spec: ScenarioSpec
+def get_cached_models_path() -> str:
+    """Get the cached models pat within the benchmark output path."""
+    path: str = os.path.join(get_benchmark_output_path(), _CACHED_MODELS_FOLDER)
+    ensure_directory_exists(path)
+    return path
 
-    # Specifies how to adapt an instance into a set of requests
-    adapter_spec: AdapterSpec
 
-    # What to evaluate on
-    metric_specs: List[MetricSpec]
+def set_benchmark_output_path(benchmark_output_path: str) -> None:
+    """Set the benchmark output path."""
+    global _BENCHMARK_OUTPUT_PATH
+    _BENCHMARK_OUTPUT_PATH = benchmark_output_path
 
-    # Data augmenter. The default `DataAugmenterSpec` does nothing.
-    data_augmenter_spec: DataAugmenterSpec = DataAugmenterSpec()
 
-    # Groups that this run spec belongs to (for aggregation)
-    groups: List[str] = field(default_factory=list)
+class RunnerError(Exception):
+    """Error that happens in the Runner."""
 
-    def __post_init__(self):
-        """
-        `self.name` is used as the name of the output folder for the `RunSpec`.
-        Clean up `self.name` by replacing any "/"'s with "_".
-        """
-        # TODO: Don't mutate name! clean this up before passing it into the constructor here
-        object.__setattr__(self, "name", self.name.replace(os.path.sep, "_"))
+    pass
 
 
 def remove_stats_nans(stats: List[Stat]) -> List[Stat]:
@@ -103,6 +102,40 @@ def remove_per_instance_stats_nans(per_instance_stats_list: List[PerInstanceStat
     return result
 
 
+def downsample_eval_instances(
+    instances: List[Instance], max_eval_instances: int, eval_splits: List[str]
+) -> List[Instance]:
+    """
+    Get the instances necessary for this run:
+    Train instances (split=train): keep all (if any) for in-context learning
+    Eval instances (split=valid or test): keep at most `max_eval_instances` specified in `AdapterSpec` by sampling
+    Return the resulting train and eval instances.
+    """
+    all_train_instances: List[Instance] = [instance for instance in instances if instance.split == TRAIN_SPLIT]
+
+    all_eval_instances: List[Instance] = [instance for instance in instances if instance.split in eval_splits]
+    if len(all_eval_instances) > max_eval_instances:
+        # The random sampling includes instances monotonically.
+        np.random.seed(0)
+        selected_eval_instances = list(
+            np.random.choice(
+                all_eval_instances,  # type: ignore
+                max_eval_instances,
+                replace=False,
+            )
+        )
+    else:
+        selected_eval_instances = all_eval_instances
+
+    hlog(
+        f"{len(instances)} instances, "
+        f"{len(all_train_instances)} train instances, "
+        f"{len(selected_eval_instances)}/{len(all_eval_instances)} eval instances"
+    )
+
+    return all_train_instances + selected_eval_instances
+
+
 class Runner:
     """
     The main entry point for running the entire benchmark.  Mostly just
@@ -121,6 +154,15 @@ class Runner:
         exit_on_error: bool,
     ):
         self.executor = Executor(execution_spec)
+        self.annotator_executor = AnnotationExecutor(
+            AnnotationExecutionSpec(
+                local_path=execution_spec.local_path if execution_spec.local_path is not None else "",
+                parallelism=execution_spec.parallelism,
+                dry_run=execution_spec.dry_run,
+                sqlite_cache_backend_config=execution_spec.sqlite_cache_backend_config,
+                mongo_cache_backend_config=execution_spec.mongo_cache_backend_config,
+            )
+        )
         self.dry_run: bool = execution_spec.dry_run
         self.tokenizer_service = TokenizerService(self.executor.service, execution_spec.auth)
         self.metric_service = MetricService(self.executor.service, execution_spec.auth)
@@ -131,9 +173,8 @@ class Runner:
         self.exit_on_error: bool = exit_on_error
 
         ensure_directory_exists(output_path)
-        # Decide where to save the raw data (e.g., "output/scenarios/mmlu").
-        self.scenarios_path: str = os.path.join(output_path, "scenarios")
-        ensure_directory_exists(self.scenarios_path)
+        self.output_path = output_path
+
         # Decide where to save input instances
         self.instances_path: str = os.path.join(output_path, "scenario_instances")
         ensure_directory_exists(self.instances_path)
@@ -145,11 +186,13 @@ class Runner:
         self.eval_cache_path: str = os.path.join(self.runs_path, "eval_cache")
         ensure_directory_exists(self.eval_cache_path)
 
-    def _is_run_completed(self, run_spec: RunSpec):
+    def _get_run_path(self, run_spec: RunSpec) -> str:
+        return os.path.join(self.runs_path, run_spec.name)
+
+    def _is_run_completed(self, run_path: str):
         """Return whether the run was previously completed.
 
         A run is completed if all of the expected output files exist."""
-        run_path: str = os.path.join(self.runs_path, run_spec.name)
         if not os.path.isdir(run_path):
             return False
         output_paths = [
@@ -182,31 +225,21 @@ class Runner:
             raise RunnerError(f"Failed runs: [{failed_runs_str}]")
 
     def run_one(self, run_spec: RunSpec):
+        run_path: str = self._get_run_path(run_spec)
+        if self.skip_completed_runs and self._is_run_completed(run_path):
+            hlog(f"Skipping run {run_spec.name} because run is completed and all output files exist.")
+            return
+        ensure_directory_exists(run_path)
+
         # Load the scenario
         scenario: Scenario = create_scenario(run_spec.scenario_spec)
 
-        # This `output_path` will be used when `Adapter` calls `Scenario.get_instances`.
-        scenario_output_path = os.path.join(self.scenarios_path, scenario.name)
-        ensure_directory_exists(scenario_output_path)
-
         # This 'output_path' will be used when the model's input instances are saved.
         args_str = ",".join([f"{k}={v}" for k, v in sorted(run_spec.scenario_spec.args.items())])
         scenario_name_with_args = f"{scenario.name}:{args_str}" if args_str else f"{scenario.name}"
         input_instances_output_path = os.path.join(self.instances_path, scenario_name_with_args)
         input_instances_file_path = os.path.join(input_instances_output_path, "input_instances.json")
 
-        run_path: str = os.path.join(self.runs_path, run_spec.name)
-        ensure_directory_exists(run_path)
-
-        if self.skip_completed_runs and self._is_run_completed(run_spec):
-            # If scenario_state.json exists, assume that all other output files exist
-            # because scenario_state.json is the last output file to be written.
-            hlog(f"Skipping run {run_spec.name} because run is completed and all output files exist.")
-            return
-
-        # Fetch and initialize the Adapter based on the `AdapterSpec`.
-        adapter: Adapter = AdapterFactory.get_adapter(run_spec.adapter_spec, self.tokenizer_service)
-
         instances: List[Instance]
         if self.skip_instances:
             instances = []
@@ -217,6 +250,7 @@ class Runner:
                 instances = [dacite.from_dict(Instance, instance) for instance in json_instances]
             else:
                 # Create the instances of the scenario
+                scenario_output_path = get_scenario_cache_path(self.output_path, scenario.name)
                 with htrack_block("scenario.get_instances"):
                     instances = scenario.get_instances(scenario_output_path)
         if self.cache_instances and not os.path.exists(input_instances_file_path):
@@ -230,10 +264,14 @@ class Runner:
             return  # Exit after saving the instances.
 
         # Give each instance a unique ID
-        instances = with_instance_ids(instances)
+        if any([instance.id is None for instance in instances]):
+            instances = with_instance_ids(instances)
 
         # Get the instances necessary for this run.
-        instances = adapter.get_run_instances(instances)
+        max_eval_instances = run_spec.adapter_spec.max_eval_instances
+        eval_splits = run_spec.adapter_spec.eval_splits or EVAL_SPLITS
+        if max_eval_instances is not None:
+            instances = downsample_eval_instances(instances, max_eval_instances, eval_splits)
 
         # Data preprocessing
         instances = DataPreprocessor(run_spec.data_augmenter_spec).preprocess(
@@ -241,15 +279,24 @@ class Runner:
         )
 
         # Adapt (convert to requests)
-        scenario_state: ScenarioState = adapter.adapt(instances, self.executor.execution_spec.parallelism)
+        adapter: Adapter = AdapterFactory.get_adapter(run_spec.adapter_spec, self.tokenizer_service)
+        request_states: List[RequestState] = adapter.adapt(instances, self.executor.execution_spec.parallelism)
+        scenario_state: ScenarioState = ScenarioState(
+            adapter_spec=run_spec.adapter_spec,
+            request_states=request_states,
+            annotator_specs=run_spec.annotators,
+        )
 
         # Execute (fill up results)
         scenario_state = self.executor.execute(scenario_state)
 
+        # Annotate (post-process the results)
+        scenario_state = self.annotator_executor.execute(scenario_state)
+
         # Apply the metrics
         # When performing a dry run, only estimate the number of tokens instead
         # of calculating the metrics.
-        metrics: List[Metric] = (
+        metrics: List[MetricInterface] = (
             [DryRunMetric()] if self.dry_run else [create_metric(metric_spec) for metric_spec in run_spec.metric_specs]
         )
         stats: List[Stat] = []

helm/benchmark/runner_config_registry.py

@@ -0,0 +1,21 @@
+from dataclasses import dataclass
+from typing import Any, Dict, Optional
+import dacite
+import yaml
+
+
+@dataclass
+class RunnerConfigSpec:
+    helm_max_concurrent_workers: int = -1
+    slurm_monitor_interval: int = 60
+    slurm_args: Optional[Dict[str, Any]] = None
+
+
+RUNNER_CONFIG = RunnerConfigSpec()
+
+
+def register_runner_config_from_path(dir_path: str) -> None:
+    global RUNNER_CONFIG
+    with open(dir_path, "r") as f:
+        raw = yaml.safe_load(f)
+    RUNNER_CONFIG = dacite.from_dict(RunnerConfigSpec, raw)

helm/benchmark/scenarios/bbq_scenario.py

@@ -123,7 +123,7 @@ class BBQScenario(Scenario):
         # trying to book a cab on Uber.", "question": "Who was not comfortable using the phone?", "ans0":
         # "The grandfather", "ans1": "The grandson", "ans2": "Can't be determined", "label": 2}
 
-        for (idx, (loaded_line, category)) in enumerate(loaded_lines):
+        for idx, (loaded_line, category) in enumerate(loaded_lines):
             is_negative: bool = loaded_line["question_polarity"] == "neg"
             is_ambiguous: bool = loaded_line["context_condition"] == "ambig"
             context: str = loaded_line["context"]

helm/benchmark/scenarios/bold_scenario.py

@@ -98,7 +98,7 @@ class BOLDScenario(Scenario):
                     # with https://raw.githubusercontent.com/amazon-
                     # research/bold/main/prompts/political_ideology_prompt.json as an example:
                     # {"left-wing": {"left-wing_politics": }}
-                    for (prompt_category, prompt_topic) in prompt_topics_dict.items():
+                    for prompt_category, prompt_topic in prompt_topics_dict.items():
                         for prompt_text in prompt_topic.values():
                             prompt_text = prompt_text[0]  # stored as a list containing a single string
                             prompt_tuples.append((prompt_category, prompt_text))
@@ -108,7 +108,7 @@ class BOLDScenario(Scenario):
 
         split_sizes = {"train": len(instances) - DEFAULT_TEST_SIZE, "test": DEFAULT_TEST_SIZE}
 
-        for (idx, prompt_tuple) in enumerate(prompt_tuples):
+        for idx, prompt_tuple in enumerate(prompt_tuples):
             prompt_category, prompt_text = prompt_tuple
             curr_split = TRAIN_SPLIT
 

helm/benchmark/scenarios/cleva_scenario.py

@@ -10,14 +10,29 @@ from helm.benchmark.adaptation.adapters.adapter_factory import (
     ADAPT_MULTIPLE_CHOICE_SEPARATE_ORIGINAL,
     ADAPT_GENERATION,
 )
-from helm.common.general import ensure_file_downloaded, ensure_directory_exists
+from helm.benchmark.runner import get_benchmark_output_path
+from helm.common.general import (
+    assert_is_str,
+    assert_is_str_list,
+    ensure_file_downloaded,
+    ensure_directory_exists,
+)
 from helm.common.hierarchical_logger import hlog
-from .scenario import Scenario, Instance, Reference, TRAIN_SPLIT, TEST_SPLIT, CORRECT_TAG, Input, Output
+from .scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TRAIN_SPLIT,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+    get_scenario_cache_path,
+)
 from .code_scenario import CodeReference, CodeInstance
 
 
 CLEVA_DATA_URL = "http://39.108.215.175/data"
-CLEVA_DATA_PATH = "benchmark_output/scenarios/cleva"
 
 
 @dataclass(frozen=True)
@@ -69,26 +84,17 @@ class Converter:
         """Convert a data point in CLEVA format to a HELM instance according to a given CLEVA prompt template."""
         transformed_data = self._apply_all(copy.deepcopy(data), templates)
 
-        prompt: str = transformed_data["input"]  # type: ignore
-        assert isinstance(prompt, str)
+        prompt = assert_is_str(transformed_data["input"])
         if "choices" in transformed_data:
             # This is a multiple-choice task
-            choices: List[str] = transformed_data["choices"]  # type: ignore
-            # Gurantee `choices` must be `List[str]`
-            assert isinstance(choices, list)
-            for c in choices:
-                assert isinstance(c, str)
+            choices = assert_is_str_list(transformed_data["choices"])
             references: List[Reference] = [
                 Reference(Output(text=text), tags=[CORRECT_TAG] if idx in transformed_data["label"] else [])
                 for idx, text in enumerate(choices)
             ]
         else:
             # This is a generation task
-            correct_answer: List[str] = transformed_data["label"]  # type: ignore
-            # Gurantee `label` must be `List[str]`
-            assert isinstance(correct_answer, list)
-            for a in correct_answer:
-                assert isinstance(a, str)
+            correct_answer = assert_is_str_list(transformed_data["label"])
             references = [Reference(Output(text=answer), tags=[CORRECT_TAG]) for answer in correct_answer]
 
         instance = Instance(
@@ -109,15 +115,12 @@ class Converter:
         to a HELM CodeInstance according to a given CLEVA prompt template.
         """
 
-        assert isinstance(templates["input"], str)
-        data["prompt"] = templates["input"].format(**data)
-        assert isinstance(data["prompt"], str)
-        assert isinstance(data["canonical_solution"], str)
+        data["prompt"] = assert_is_str(templates["input"]).format(**data)
         instance = CodeInstance(
-            input=Input(text=data["prompt"]),
+            input=Input(text=assert_is_str(data["prompt"])),
             references=[
                 CodeReference(
-                    output=Output(text=data["canonical_solution"]),
+                    output=Output(text=assert_is_str(data["canonical_solution"])),
                     test_cases=data,
                     tags=[CORRECT_TAG],
                 )
@@ -211,27 +214,18 @@ class Converter:
                 transformed_data[k] = self._apply(data[k], template, **data)
 
         # We then merge all other fields into the `input`
-        assert isinstance(templates["input"], str), "The input field of a template should be a string"
-        data["input"] = templates["input"].format(**transformed_data)
+        data["input"] = assert_is_str(templates["input"]).format(**transformed_data)
         if "choices" in data:
             # We take the corresponding choices and apply the `label` template
             # Note: we do not allow `label` template to access other fields in multi-choice tasks
             # Overwrite `choices` to the actual continuations
-            choices: List[str] = data["choices"]  # type: ignore
-            # Gurantee `choices` must be `List[str]`
-            assert isinstance(choices, list)
-            for c in choices:
-                assert isinstance(c, str)
+            choices = assert_is_str_list(data["choices"])
             data["choices"] = [self._apply(c, templates.get("label", None), label=c) for c in choices]
         else:
             # For generation tasks, we allow it to access to other stringified fields
             kwargs = transformed_data
             del kwargs["label"]
-            labels: List[str] = data["label"]  # type: ignore
-            # Gurantee `label` must be `List[str]`
-            assert isinstance(labels, list)
-            for label in labels:
-                assert isinstance(label, str)
+            labels = assert_is_str_list(data["label"])
             data["label"] = [self._apply(x, templates.get("label", None), **kwargs, label=x) for x in labels]
         return data
 
@@ -402,7 +396,10 @@ class CLEVAScenario(Scenario):
         self.subtask = subtask
         self.version = version
         self.converter = Converter()
-        self.prompt_template, _ = CLEVAScenario.get_prompt_setting(self.task, subtask, version, prompt_id)
+        scenario_cache_path = get_scenario_cache_path(get_benchmark_output_path(), CLEVAScenario.name)
+        self.prompt_template, _ = CLEVAScenario.get_prompt_setting(
+            self.task, subtask, version, prompt_id, scenario_cache_path
+        )
 
     @property
     @abstractmethod
@@ -410,14 +407,14 @@ class CLEVAScenario(Scenario):
         pass
 
     @classmethod
-    def download_dataset(cls, task: str, version: str):
+    def download_dataset(cls, task: str, version: str, cache_dir: str):
         source_url: str = CLEVA_DATA_URL + f"/{version}/{task}.zip"
-        target_dir: str = os.path.join(CLEVA_DATA_PATH, "data", version)
+        target_dir: str = os.path.join(cache_dir, "data", version)
         ensure_directory_exists(target_dir)
         ensure_file_downloaded(source_url=source_url, target_path=os.path.join(target_dir, task), unpack=True)
 
-    def load_dataset(self) -> Dict[str, List[Dict[str, Any]]]:
-        data_dir: str = os.path.join(CLEVA_DATA_PATH, "data", self.version, self.task)
+    def load_dataset(self, cache_dir: str) -> Dict[str, List[Dict[str, Any]]]:
+        data_dir: str = os.path.join(cache_dir, "data", self.version, self.task)
         if self.subtask:
             data_dir = os.path.join(data_dir, self.subtask)
 
@@ -434,8 +431,8 @@ class CLEVAScenario(Scenario):
         return dataset
 
     @staticmethod
-    def load_prompt_templates(task: str, subtask: Optional[str], version: str) -> List[Dict[str, Any]]:
-        prompt_dir: str = os.path.join(CLEVA_DATA_PATH, "data", version, task)
+    def load_prompt_templates(task: str, subtask: Optional[str], version: str, cache_dir: str) -> List[Dict[str, Any]]:
+        prompt_dir: str = os.path.join(cache_dir, "data", version, task)
         if subtask:
             prompt_dir = os.path.join(prompt_dir, subtask)
         file_path = os.path.join(prompt_dir, "prompts.json")
@@ -448,7 +445,7 @@ class CLEVAScenario(Scenario):
 
     def get_instances(self, output_path: str) -> List[Instance]:
         # Download the raw data
-        dataset = self.load_dataset()
+        dataset = self.load_dataset(output_path)
 
         # Read all the instances
         instances: List[Instance] = []
@@ -465,9 +462,9 @@ class CLEVAScenario(Scenario):
 
     @classmethod
     def get_prompt_setting(
-        cls, task: str, subtask: Optional[str], version: str, prompt_id: int
+        cls, task: str, subtask: Optional[str], version: str, prompt_id: int, output_path: str
     ) -> Tuple[Dict[str, Any], PromptSetting]:
-        prompt_templates = cls.load_prompt_templates(task, subtask, version)
+        prompt_templates = cls.load_prompt_templates(task, subtask, version, output_path)
         if prompt_id >= len(prompt_templates):
             raise ValueError(
                 f"You want to use prompt template with prompt_id {prompt_id}, but there is only"
@@ -519,10 +516,10 @@ class CLEVAScenario(Scenario):
 
     @classmethod
     def load_inference_parameters(
-        cls, task: str, subtask: Optional[str], version: str, prompt_id: int
+        cls, task: str, subtask: Optional[str], version: str, prompt_id: int, cache_dir: str
     ) -> Dict[str, Any]:
         # We use a dict instead of dataclass to store hyperparameters such that we can set different default values
-        params_dir: str = os.path.join(CLEVA_DATA_PATH, "data", version, task)
+        params_dir: str = os.path.join(cache_dir, "data", version, task)
         if subtask:
             params_dir = os.path.join(params_dir, subtask)
         file_path = os.path.join(params_dir, "infer_params.json")
@@ -932,7 +929,7 @@ class CLEVADialogueGenerationScenario(CLEVAScenario):
 
     def get_instances(self, output_path: str) -> List[Instance]:
         # Download the raw data
-        dataset = self.load_dataset()
+        dataset = self.load_dataset(output_path)
 
         # Read all the instances
         instances: List[Instance] = []

helm/benchmark/scenarios/code_scenario.py

@@ -48,6 +48,7 @@ APPS is a benchmark for code generation from natural language specifications.
 Each instance has 1) a problem description with examples (as what you get in
 programming competitions), 2) coding solutions, 3) test cases.
 """
+
 import io
 import json
 import os
@@ -139,8 +140,8 @@ def _read_and_preprocess_apps(target_path: str) -> List[CodeInstance]:
     # only if the version of Python has a default limit.
     #
     # See: https://docs.python.org/3/library/stdtypes.html#int-max-str-digits
-    if hasattr(sys, "set_int_max_str_digits"):  # type: ignore
-        sys.set_int_max_str_digits(100000)  # type: ignore
+    if hasattr(sys, "set_int_max_str_digits"):
+        sys.set_int_max_str_digits(100000)
 
     SINGLE_STR_LIMIT = 150000  # From original codebase.