crfm-helm 0.3.0__py3-none-any.whl → 0.5.0__py3-none-any.whl

helm/benchmark/window_services/test_anthropic_window_service.py

@@ -2,6 +2,7 @@ import shutil
 import tempfile
 from typing import List
 
+from helm.common.cache_backend_config import BlackHoleCacheBackendConfig
 from .tokenizer_service import TokenizerService
 from .window_service_factory import WindowServiceFactory
 from .test_utils import get_tokenizer_service, TEST_PROMPT
@@ -120,7 +121,7 @@ class TestAnthropicWindowService:
 
     def setup_method(self):
         self.path: str = tempfile.mkdtemp()
-        service: TokenizerService = get_tokenizer_service(self.path)
+        service: TokenizerService = get_tokenizer_service(self.path, BlackHoleCacheBackendConfig())
         self.window_service = WindowServiceFactory.get_window_service("anthropic/claude-v1.3", service)
 
     def teardown_method(self, method):

helm/benchmark/window_services/test_bloom_window_service.py

@@ -2,6 +2,7 @@ import shutil
 import tempfile
 from typing import List
 
+from helm.common.cache_backend_config import BlackHoleCacheBackendConfig
 from .tokenizer_service import TokenizerService
 from .window_service_factory import WindowServiceFactory
 from .test_utils import get_tokenizer_service, TEST_PROMPT
@@ -64,7 +65,7 @@ class TestBloomWindowService:
 
     def setup_method(self):
         self.path: str = tempfile.mkdtemp()
-        service: TokenizerService = get_tokenizer_service(self.path)
+        service: TokenizerService = get_tokenizer_service(self.path, BlackHoleCacheBackendConfig())
         self.window_service = WindowServiceFactory.get_window_service("together/bloom", service)
 
     def teardown_method(self, method):

helm/benchmark/window_services/test_cohere_window_service.py

@@ -6,6 +6,7 @@ from typing import List
 
 from sqlitedict import SqliteDict
 
+from helm.common.cache_backend_config import SqliteCacheBackendConfig
 from helm.common.general import ensure_directory_exists
 from .test_cohere_window_service_utils import REQUESTS_TO_RESPONSES, TEST_PROMPT, TOKENIZED_PROMPT
 from .tokenizer_service import TokenizerService
@@ -30,7 +31,7 @@ class TestCohereWindowService:
         with open(os.path.join(cls.path, "credentials.conf"), "w") as f:
             f.write("cohereApiKey: secret")
 
-        service: TokenizerService = get_tokenizer_service(cls.path)
+        service: TokenizerService = get_tokenizer_service(cls.path, SqliteCacheBackendConfig(cache_path))
         cls.window_service = WindowServiceFactory.get_window_service("cohere/xlarge-20220609", service)
         cls.prompt: str = TEST_PROMPT
         cls.tokenized_prompt: List[str] = TOKENIZED_PROMPT

helm/benchmark/window_services/test_flan_t5_window_service.py

@@ -1,5 +1,6 @@
 import tempfile
 
+from helm.common.cache_backend_config import BlackHoleCacheBackendConfig
 from helm.benchmark.window_services.test_t511b_window_service import TestT511bWindowService
 from helm.benchmark.window_services.window_service_factory import TokenizerService, WindowServiceFactory
 from helm.benchmark.window_services.test_utils import get_tokenizer_service
@@ -8,5 +9,5 @@ from helm.benchmark.window_services.test_utils import get_tokenizer_service
 class TestFlanT5WindowService(TestT511bWindowService):
     def setup_method(self):
         self.path: str = tempfile.mkdtemp()
-        service: TokenizerService = get_tokenizer_service(self.path)
+        service: TokenizerService = get_tokenizer_service(self.path, BlackHoleCacheBackendConfig())
         self.window_service = WindowServiceFactory.get_window_service("together/flan-t5-xxl", service)

helm/benchmark/window_services/test_gpt2_window_service.py

@@ -2,7 +2,7 @@ import shutil
 import tempfile
 
 from helm.benchmark.window_services.tokenizer_service import TokenizerService
-
+from helm.common.cache_backend_config import BlackHoleCacheBackendConfig
 from .test_utils import get_tokenizer_service, TEST_PROMPT, GPT2_TEST_TOKENS, GPT2_TEST_TOKEN_IDS
 from .window_service_factory import WindowServiceFactory
 
@@ -10,7 +10,7 @@ from .window_service_factory import WindowServiceFactory
 class TestGPT2WindowService:
     def setup_method(self):
         self.path: str = tempfile.mkdtemp()
-        service: TokenizerService = get_tokenizer_service(self.path)
+        service: TokenizerService = get_tokenizer_service(self.path, BlackHoleCacheBackendConfig())
         self.window_service = WindowServiceFactory.get_window_service("huggingface/gpt2", service)
 
     def teardown_method(self, method):

helm/benchmark/window_services/test_gpt4_window_service.py

@@ -1,6 +1,7 @@
 import shutil
 import tempfile
 
+from helm.common.cache_backend_config import BlackHoleCacheBackendConfig
 from .test_utils import get_tokenizer_service, TEST_PROMPT, GPT4_TEST_TOKEN_IDS, GPT4_TEST_TOKENS
 from .tokenizer_service import TokenizerService
 from .window_service_factory import WindowServiceFactory
@@ -9,7 +10,7 @@ from .window_service_factory import WindowServiceFactory
 class TestOpenAIWindowService:
     def setup_method(self):
         self.path: str = tempfile.mkdtemp()
-        service: TokenizerService = get_tokenizer_service(self.path)
+        service: TokenizerService = get_tokenizer_service(self.path, BlackHoleCacheBackendConfig())
         self.window_service = WindowServiceFactory.get_window_service("openai/gpt-3.5-turbo-0301", service)
 
     def teardown_method(self, method):

helm/benchmark/window_services/test_gptj_window_service.py

@@ -1,6 +1,7 @@
 import shutil
 import tempfile
 
+from helm.common.cache_backend_config import BlackHoleCacheBackendConfig
 from .tokenizer_service import TokenizerService
 from .window_service_factory import WindowServiceFactory
 from .test_utils import get_tokenizer_service, GPT2_TEST_TOKENS, GPT2_TEST_TOKEN_IDS, TEST_PROMPT
@@ -9,8 +10,8 @@ from .test_utils import get_tokenizer_service, GPT2_TEST_TOKENS, GPT2_TEST_TOKEN
 class TestGPTJWindowService:
     def setup_method(self):
         self.path: str = tempfile.mkdtemp()
-        service: TokenizerService = get_tokenizer_service(self.path)
-        self.window_service = WindowServiceFactory.get_window_service("together/gpt-j-6b", service)
+        service: TokenizerService = get_tokenizer_service(self.path, BlackHoleCacheBackendConfig())
+        self.window_service = WindowServiceFactory.get_window_service("huggingface/gpt-j-6b", service)
 
     def teardown_method(self, method):
         shutil.rmtree(self.path)

helm/benchmark/window_services/test_gptneox_window_service.py

@@ -2,6 +2,7 @@ import shutil
 import tempfile
 from typing import List
 
+from helm.common.cache_backend_config import BlackHoleCacheBackendConfig
 from .tokenizer_service import TokenizerService
 from .window_service_factory import WindowServiceFactory
 from .test_utils import get_tokenizer_service, TEST_PROMPT
@@ -65,8 +66,8 @@ class TestGPTNeoXWindowService:
 
     def setup_method(self):
         self.path: str = tempfile.mkdtemp()
-        service: TokenizerService = get_tokenizer_service(self.path)
-        self.window_service = WindowServiceFactory.get_window_service("together/gpt-neox-20b", service)
+        service: TokenizerService = get_tokenizer_service(self.path, BlackHoleCacheBackendConfig())
+        self.window_service = WindowServiceFactory.get_window_service("huggingface/gpt-neox-20b", service)
 
     def teardown_method(self, method):
         shutil.rmtree(self.path)

helm/benchmark/window_services/test_ice_window_service.py

@@ -2,6 +2,7 @@ import shutil
 import tempfile
 from typing import List
 
+from helm.common.cache_backend_config import BlackHoleCacheBackendConfig
 from .tokenizer_service import TokenizerService
 from .window_service_factory import WindowServiceFactory
 from .test_utils import get_tokenizer_service, TEST_PROMPT
@@ -64,7 +65,7 @@ class TestICEWindowService:
 
     def setup_method(self):
         self.path: str = tempfile.mkdtemp()
-        service: TokenizerService = get_tokenizer_service(self.path)
+        service: TokenizerService = get_tokenizer_service(self.path, BlackHoleCacheBackendConfig())
         self.window_service = WindowServiceFactory.get_window_service("together/glm", service)
 
     def teardown_method(self, method):

helm/benchmark/window_services/test_openai_window_service.py

@@ -1,6 +1,7 @@
 import shutil
 import tempfile
 
+from helm.common.cache_backend_config import BlackHoleCacheBackendConfig
 from .test_utils import get_tokenizer_service, TEST_PROMPT, GPT2_TEST_TOKENS, GPT2_TEST_TOKEN_IDS
 from .tokenizer_service import TokenizerService
 from .window_service_factory import WindowServiceFactory
@@ -9,7 +10,7 @@ from .window_service_factory import WindowServiceFactory
 class TestOpenAIWindowService:
     def setup_method(self):
         self.path: str = tempfile.mkdtemp()
-        service: TokenizerService = get_tokenizer_service(self.path)
+        service: TokenizerService = get_tokenizer_service(self.path, BlackHoleCacheBackendConfig())
         self.window_service = WindowServiceFactory.get_window_service("openai/davinci", service)
 
     def teardown_method(self, method):

helm/benchmark/window_services/test_opt_window_service.py

@@ -1,6 +1,7 @@
 import shutil
 import tempfile
 
+from helm.common.cache_backend_config import BlackHoleCacheBackendConfig
 from .test_utils import get_tokenizer_service, TEST_PROMPT
 from .tokenizer_service import TokenizerService
 from .window_service_factory import WindowServiceFactory
@@ -9,8 +10,8 @@ from .window_service_factory import WindowServiceFactory
 class TestOPTWindowService:
     def setup_method(self):
         self.path: str = tempfile.mkdtemp()
-        service: TokenizerService = get_tokenizer_service(self.path)
-        self.window_service = WindowServiceFactory.get_window_service("together/opt-175b", service)
+        service: TokenizerService = get_tokenizer_service(self.path, BlackHoleCacheBackendConfig())
+        self.window_service = WindowServiceFactory.get_window_service("huggingface/opt-175b", service)
 
     def teardown_method(self, method):
         shutil.rmtree(self.path)

helm/benchmark/window_services/test_palmyra_window_service.py

@@ -1,6 +1,7 @@
 from tempfile import TemporaryDirectory
 from typing import List
 
+from helm.common.cache_backend_config import BlackHoleCacheBackendConfig
 from .tokenizer_service import TokenizerService
 from .window_service_factory import WindowServiceFactory
 from .test_utils import get_tokenizer_service, TEST_PROMPT
@@ -117,7 +118,7 @@ class TestPalmyraWindowService:
 
     def setup_method(self):
         self.temporary_directory = TemporaryDirectory()
-        service: TokenizerService = get_tokenizer_service(self.temporary_directory.name)
+        service: TokenizerService = get_tokenizer_service(self.temporary_directory.name, BlackHoleCacheBackendConfig())
         self.window_service = WindowServiceFactory.get_window_service("writer/palmyra-large", service)
 
     def teardown_method(self, method):

helm/benchmark/window_services/test_t0pp_window_service.py

@@ -2,6 +2,7 @@ import shutil
 import tempfile
 from typing import List
 
+from helm.common.cache_backend_config import BlackHoleCacheBackendConfig
 from .tokenizer_service import TokenizerService
 from .window_service_factory import WindowServiceFactory
 from .test_utils import get_tokenizer_service, TEST_PROMPT
@@ -70,7 +71,7 @@ class TestT0ppWindowService:
 
     def setup_method(self):
         self.path: str = tempfile.mkdtemp()
-        service: TokenizerService = get_tokenizer_service(self.path)
+        service: TokenizerService = get_tokenizer_service(self.path, BlackHoleCacheBackendConfig())
         self.window_service = WindowServiceFactory.get_window_service("together/t0pp", service)
 
     def teardown_method(self, method):

helm/benchmark/window_services/test_t511b_window_service.py

@@ -2,6 +2,7 @@ import shutil
 import tempfile
 from typing import List
 
+from helm.common.cache_backend_config import BlackHoleCacheBackendConfig
 from .tokenizer_service import TokenizerService
 from .window_service_factory import WindowServiceFactory
 from .test_utils import get_tokenizer_service, TEST_PROMPT
@@ -70,7 +71,7 @@ class TestT511bWindowService:
 
     def setup_method(self):
         self.path: str = tempfile.mkdtemp()
-        service: TokenizerService = get_tokenizer_service(self.path)
+        service: TokenizerService = get_tokenizer_service(self.path, BlackHoleCacheBackendConfig())
         self.window_service = WindowServiceFactory.get_window_service("together/t5-11b", service)
 
     def teardown_method(self, method):

helm/benchmark/window_services/test_ul2_window_service.py

@@ -2,6 +2,7 @@ import shutil
 import tempfile
 from typing import List
 
+from helm.common.cache_backend_config import BlackHoleCacheBackendConfig
 from .tokenizer_service import TokenizerService
 from .window_service_factory import WindowServiceFactory
 from .test_utils import get_tokenizer_service, TEST_PROMPT
@@ -70,7 +71,7 @@ class TestUL2WindowService:
 
     def setup_method(self):
         self.path: str = tempfile.mkdtemp()
-        service: TokenizerService = get_tokenizer_service(self.path)
+        service: TokenizerService = get_tokenizer_service(self.path, BlackHoleCacheBackendConfig())
         self.window_service = WindowServiceFactory.get_window_service("together/ul2", service)
 
     def teardown_method(self, method):

helm/benchmark/window_services/test_utils.py

@@ -1,6 +1,7 @@
 from typing import List
 
 from helm.common.authentication import Authentication
+from helm.common.cache_backend_config import CacheBackendConfig
 from helm.proxy.services.server_service import ServerService
 from helm.benchmark.metrics.metric_service import MetricService
 from .tokenizer_service import TokenizerService
@@ -227,6 +228,6 @@ GPT4_TEST_TOKENS: List[str] = [
 ]
 
 
-def get_tokenizer_service(local_path: str) -> TokenizerService:
-    service = ServerService(base_path=local_path, root_mode=True)
+def get_tokenizer_service(local_path: str, cache_backend_config: CacheBackendConfig) -> TokenizerService:
+    service = ServerService(base_path=local_path, root_mode=True, cache_backend_config=cache_backend_config)
     return MetricService(service, Authentication("test"))

helm/benchmark/window_services/test_yalm_window_service.py

@@ -1,6 +1,7 @@
 import shutil
 import tempfile
 
+from helm.common.cache_backend_config import BlackHoleCacheBackendConfig
 from .test_utils import get_tokenizer_service, TEST_PROMPT
 from .tokenizer_service import TokenizerService
 from .window_service_factory import WindowServiceFactory
@@ -9,7 +10,7 @@ from .window_service_factory import WindowServiceFactory
 class TestYaLMWindowService:
     def setup_method(self):
         self.path: str = tempfile.mkdtemp()
-        service: TokenizerService = get_tokenizer_service(self.path)
+        service: TokenizerService = get_tokenizer_service(self.path, BlackHoleCacheBackendConfig())
         self.window_service = WindowServiceFactory.get_window_service("together/yalm", service)
 
     def teardown_method(self, method):

helm/benchmark/window_services/window_service.py

@@ -110,3 +110,45 @@ class WindowService(ABC):
         minus the expected completion length (defaults to 0).
         """
         pass
+
+
+class ConfigurableWindowService(WindowService, ABC):
+    def __init__(
+        self,
+        tokenizer_name: str,
+        max_sequence_length: int,
+        max_request_length: Optional[int] = None,
+        max_sequence_and_generated_tokens_length: Optional[int] = None,
+        end_of_text_token: Optional[str] = None,
+        prefix_token: Optional[str] = None,
+    ):
+        self._tokenizer_name = tokenizer_name
+        self._max_sequence_length = max_sequence_length
+        self._max_request_length = max_request_length or max_sequence_length
+        self._max_sequence_and_generated_tokens_length = max_sequence_and_generated_tokens_length or INT_MAX
+        self._end_of_text_token = end_of_text_token or ""
+        self._prefix_token = prefix_token or ""
+
+    @property
+    def tokenizer_name(self) -> str:
+        return self._tokenizer_name
+
+    @property
+    def max_sequence_length(self) -> int:
+        return self._max_sequence_length
+
+    @property
+    def max_request_length(self) -> int:
+        return self._max_request_length
+
+    @property
+    def max_sequence_and_generated_tokens_length(self) -> int:
+        return self._max_sequence_and_generated_tokens_length
+
+    @property
+    def end_of_text_token(self) -> str:
+        return self._end_of_text_token
+
+    @property
+    def prefix_token(self) -> str:
+        return self._prefix_token