edsl 0.1.33.dev2__py3-none-any.whl → 0.1.34__py3-none-any.whl

edsl/inference_services/TestService.py

@@ -1,4 +1,4 @@
-from typing import Any, List
+from typing import Any, List, Optional
 import os
 import asyncio
 from edsl.inference_services.InferenceServiceABC import InferenceServiceABC
@@ -7,14 +7,25 @@ from edsl.inference_services.rate_limits_cache import rate_limits
 from edsl.utilities.utilities import fix_partial_correct_response
 
 from edsl.enums import InferenceServiceType
+import random
 
 
 class TestService(InferenceServiceABC):
     """OpenAI service class."""
 
+    _inference_service_ = "test"
+    _env_key_name_ = None
+    _base_url_ = None
+
+    _sync_client_ = None
+    _async_client_ = None
+
+    _sync_client_instance = None
+    _async_client_instance = None
+
     key_sequence = None
+    usage_sequence = None
     model_exclude_list = []
-    _inference_service_ = "test"
     input_token_name = "prompt_tokens"
     output_token_name = "completion_tokens"
 
@@ -45,27 +56,34 @@ class TestService(InferenceServiceABC):
                     return "Hello, world"
 
             async def async_execute_model_call(
-                self, user_prompt: str, system_prompt: str
+                self,
+                user_prompt: str,
+                system_prompt: str,
+                # func: Optional[callable] = None,
+                files_list: Optional[List["File"]] = None,
             ) -> dict[str, Any]:
                 await asyncio.sleep(0.1)
                 # return {"message": """{"answer": "Hello, world"}"""}
+
+                if hasattr(self, "func"):
+                    return {
+                        "message": [
+                            {"text": self.func(user_prompt, system_prompt, files_list)}
+                        ],
+                        "usage": {"prompt_tokens": 1, "completion_tokens": 1},
+                    }
+
                 if hasattr(self, "throw_exception") and self.throw_exception:
-                    raise Exception("This is a test error")
+                    if hasattr(self, "exception_probability"):
+                        p = self.exception_probability
+                    else:
+                        p = 1
+
+                    if random.random() < p:
+                        raise Exception("This is a test error")
                 return {
                     "message": [{"text": f"{self._canned_response}"}],
                     "usage": {"prompt_tokens": 1, "completion_tokens": 1},
                 }
 
         return TestServiceLanguageModel
-
-    # _inference_service_ = "openai"
-    # _env_key_name_ = "OPENAI_API_KEY"
-    # _base_url_ = None
-
-    # _sync_client_ = openai.OpenAI
-    # _async_client_ = openai.AsyncOpenAI
-
-    # _sync_client_instance = None
-    # _async_client_instance = None
-
-    # key_sequence = ["choices", 0, "message", "content"]

edsl/inference_services/TogetherAIService.py

@@ -0,0 +1,170 @@
+import aiohttp
+import json
+import requests
+from typing import Any, List, Optional
+
+# from edsl.inference_services.InferenceServiceABC import InferenceServiceABC
+from edsl.language_models import LanguageModel
+
+from edsl.inference_services.OpenAIService import OpenAIService
+import openai
+
+
+class TogetherAIService(OpenAIService):
+    """DeepInfra service class."""
+
+    _inference_service_ = "together"
+    _env_key_name_ = "TOGETHER_API_KEY"
+    _base_url_ = "https://api.together.xyz/v1"
+    _models_list_cache: List[str] = []
+
+    # These are non-serverless models. There was no api param to filter them
+    model_exclude_list = [
+        "EleutherAI/llemma_7b",
+        "HuggingFaceH4/zephyr-7b-beta",
+        "Nexusflow/NexusRaven-V2-13B",
+        "NousResearch/Hermes-2-Theta-Llama-3-70B",
+        "NousResearch/Nous-Capybara-7B-V1p9",
+        "NousResearch/Nous-Hermes-13b",
+        "NousResearch/Nous-Hermes-2-Mistral-7B-DPO",
+        "NousResearch/Nous-Hermes-2-Mixtral-8x7B-SFT",
+        "NousResearch/Nous-Hermes-Llama2-13b",
+        "NousResearch/Nous-Hermes-Llama2-70b",
+        "NousResearch/Nous-Hermes-llama-2-7b",
+        "NumbersStation/nsql-llama-2-7B",
+        "Open-Orca/Mistral-7B-OpenOrca",
+        "Phind/Phind-CodeLlama-34B-Python-v1",
+        "Phind/Phind-CodeLlama-34B-v2",
+        "Qwen/Qwen1.5-0.5B",
+        "Qwen/Qwen1.5-0.5B-Chat",
+        "Qwen/Qwen1.5-1.8B",
+        "Qwen/Qwen1.5-1.8B-Chat",
+        "Qwen/Qwen1.5-14B",
+        "Qwen/Qwen1.5-14B-Chat",
+        "Qwen/Qwen1.5-32B",
+        "Qwen/Qwen1.5-32B-Chat",
+        "Qwen/Qwen1.5-4B",
+        "Qwen/Qwen1.5-4B-Chat",
+        "Qwen/Qwen1.5-72B",
+        "Qwen/Qwen1.5-7B",
+        "Qwen/Qwen1.5-7B-Chat",
+        "Qwen/Qwen2-1.5B",
+        "Qwen/Qwen2-1.5B-Instruct",
+        "Qwen/Qwen2-72B",
+        "Qwen/Qwen2-7B",
+        "Qwen/Qwen2-7B-Instruct",
+        "SG161222/Realistic_Vision_V3.0_VAE",
+        "Snowflake/snowflake-arctic-instruct",
+        "Undi95/ReMM-SLERP-L2-13B",
+        "Undi95/Toppy-M-7B",
+        "WizardLM/WizardCoder-Python-34B-V1.0",
+        "WizardLM/WizardLM-13B-V1.2",
+        "WizardLM/WizardLM-70B-V1.0",
+        "allenai/OLMo-7B",
+        "allenai/OLMo-7B-Instruct",
+        "bert-base-uncased",
+        "codellama/CodeLlama-13b-Instruct-hf",
+        "codellama/CodeLlama-13b-Python-hf",
+        "codellama/CodeLlama-13b-hf",
+        "codellama/CodeLlama-34b-Python-hf",
+        "codellama/CodeLlama-34b-hf",
+        "codellama/CodeLlama-70b-Instruct-hf",
+        "codellama/CodeLlama-70b-Python-hf",
+        "codellama/CodeLlama-70b-hf",
+        "codellama/CodeLlama-7b-Instruct-hf",
+        "codellama/CodeLlama-7b-Python-hf",
+        "codellama/CodeLlama-7b-hf",
+        "cognitivecomputations/dolphin-2.5-mixtral-8x7b",
+        "deepseek-ai/deepseek-coder-33b-instruct",
+        "garage-bAInd/Platypus2-70B-instruct",
+        "google/gemma-2b",
+        "google/gemma-7b",
+        "google/gemma-7b-it",
+        "gradientai/Llama-3-70B-Instruct-Gradient-1048k",
+        "hazyresearch/M2-BERT-2k-Retrieval-Encoder-V1",
+        "huggyllama/llama-13b",
+        "huggyllama/llama-30b",
+        "huggyllama/llama-65b",
+        "huggyllama/llama-7b",
+        "lmsys/vicuna-13b-v1.3",
+        "lmsys/vicuna-13b-v1.5",
+        "lmsys/vicuna-13b-v1.5-16k",
+        "lmsys/vicuna-7b-v1.3",
+        "lmsys/vicuna-7b-v1.5",
+        "meta-llama/Llama-2-13b-hf",
+        "meta-llama/Llama-2-70b-chat-hf",
+        "meta-llama/Llama-2-7b-hf",
+        "meta-llama/Llama-3-70b-hf",
+        "meta-llama/Llama-3-8b-hf",
+        "meta-llama/Meta-Llama-3-70B",
+        "meta-llama/Meta-Llama-3-70B-Instruct",
+        "meta-llama/Meta-Llama-3-8B-Instruct",
+        "meta-llama/Meta-Llama-3.1-70B-Instruct-Reference",
+        "meta-llama/Meta-Llama-3.1-70B-Reference",
+        "meta-llama/Meta-Llama-3.1-8B-Reference",
+        "microsoft/phi-2",
+        "mistralai/Mixtral-8x22B",
+        "openchat/openchat-3.5-1210",
+        "prompthero/openjourney",
+        "runwayml/stable-diffusion-v1-5",
+        "sentence-transformers/msmarco-bert-base-dot-v5",
+        "snorkelai/Snorkel-Mistral-PairRM-DPO",
+        "stabilityai/stable-diffusion-2-1",
+        "teknium/OpenHermes-2-Mistral-7B",
+        "teknium/OpenHermes-2p5-Mistral-7B",
+        "togethercomputer/CodeLlama-13b-Instruct",
+        "togethercomputer/CodeLlama-13b-Python",
+        "togethercomputer/CodeLlama-34b",
+        "togethercomputer/CodeLlama-34b-Python",
+        "togethercomputer/CodeLlama-7b-Instruct",
+        "togethercomputer/CodeLlama-7b-Python",
+        "togethercomputer/Koala-13B",
+        "togethercomputer/Koala-7B",
+        "togethercomputer/LLaMA-2-7B-32K",
+        "togethercomputer/SOLAR-10.7B-Instruct-v1.0-int4",
+        "togethercomputer/StripedHyena-Hessian-7B",
+        "togethercomputer/alpaca-7b",
+        "togethercomputer/evo-1-131k-base",
+        "togethercomputer/evo-1-8k-base",
+        "togethercomputer/guanaco-13b",
+        "togethercomputer/guanaco-33b",
+        "togethercomputer/guanaco-65b",
+        "togethercomputer/guanaco-7b",
+        "togethercomputer/llama-2-13b",
+        "togethercomputer/llama-2-70b-chat",
+        "togethercomputer/llama-2-7b",
+        "wavymulder/Analog-Diffusion",
+        "zero-one-ai/Yi-34B",
+        "zero-one-ai/Yi-34B-Chat",
+        "zero-one-ai/Yi-6B",
+    ]
+
+    _sync_client_ = openai.OpenAI
+    _async_client_ = openai.AsyncOpenAI
+
+    @classmethod
+    def get_model_list(cls):
+        # Togheter.ai has a different response in model list then openai
+        # and the OpenAI class returns an error when calling .models.list()
+        import requests
+        import os
+
+        url = "https://api.together.xyz/v1/models?filter=serverless"
+        token = os.getenv(cls._env_key_name_)
+        headers = {"accept": "application/json", "authorization": f"Bearer {token}"}
+
+        response = requests.get(url, headers=headers)
+        return response.json()
+
+    @classmethod
+    def available(cls) -> List[str]:
+        if not cls._models_list_cache:
+            try:
+                cls._models_list_cache = [
+                    m["id"]
+                    for m in cls.get_model_list()
+                    if m["id"] not in cls.model_exclude_list
+                ]
+            except Exception as e:
+                raise
+        return cls._models_list_cache

edsl/inference_services/registry.py

@@ -12,6 +12,7 @@ from edsl.inference_services.AzureAI import AzureAIService
 from edsl.inference_services.OllamaService import OllamaService
 from edsl.inference_services.TestService import TestService
 from edsl.inference_services.MistralAIService import MistralAIService
+from edsl.inference_services.TogetherAIService import TogetherAIService
 
 default = InferenceServicesCollection(
     [
@@ -25,5 +26,6 @@ default = InferenceServicesCollection(
         OllamaService,
         TestService,
         MistralAIService,
+        TogetherAIService,
     ]
 )

edsl/jobs/Jobs.py

@@ -145,14 +145,21 @@ class Jobs(Base):
         >>> Jobs.example().prompts()
         Dataset(...)
         """
+        from edsl import Coop
+
+        c = Coop()
+        price_lookup = c.fetch_prices()
 
         interviews = self.interviews()
         # data = []
         interview_indices = []
-        question_indices = []
+        question_names = []
         user_prompts = []
         system_prompts = []
         scenario_indices = []
+        agent_indices = []
+        models = []
+        costs = []
         from edsl.results.Dataset import Dataset
 
         for interview_index, interview in enumerate(interviews):
@@ -160,23 +167,97 @@ class Jobs(Base):
                 interview._get_invigilator(question)
                 for question in self.survey.questions
             ]
-            # list(interview._build_invigilators(debug=False))
             for _, invigilator in enumerate(invigilators):
                 prompts = invigilator.get_prompts()
-                user_prompts.append(prompts["user_prompt"])
-                system_prompts.append(prompts["system_prompt"])
+                user_prompt = prompts["user_prompt"]
+                system_prompt = prompts["system_prompt"]
+                user_prompts.append(user_prompt)
+                system_prompts.append(system_prompt)
+                agent_index = self.agents.index(invigilator.agent)
+                agent_indices.append(agent_index)
                 interview_indices.append(interview_index)
-                scenario_indices.append(invigilator.scenario)
-                question_indices.append(invigilator.question.question_name)
-        return Dataset(
+                scenario_index = self.scenarios.index(invigilator.scenario)
+                scenario_indices.append(scenario_index)
+                models.append(invigilator.model.model)
+                question_names.append(invigilator.question.question_name)
+                # cost calculation
+                key = (invigilator.model._inference_service_, invigilator.model.model)
+                relevant_prices = price_lookup[key]
+                inverse_output_price = relevant_prices["output"]["one_usd_buys"]
+                inverse_input_price = relevant_prices["input"]["one_usd_buys"]
+                input_tokens = len(str(user_prompt) + str(system_prompt)) // 4
+                output_tokens = len(str(user_prompt) + str(system_prompt)) // 4
+                cost = input_tokens / float(
+                    inverse_input_price
+                ) + output_tokens / float(inverse_output_price)
+                costs.append(cost)
+
+        d = Dataset(
             [
-                {"interview_index": interview_indices},
-                {"question_index": question_indices},
                 {"user_prompt": user_prompts},
-                {"scenario_index": scenario_indices},
                 {"system_prompt": system_prompts},
+                {"interview_index": interview_indices},
+                {"question_name": question_names},
+                {"scenario_index": scenario_indices},
+                {"agent_index": agent_indices},
+                {"model": models},
+                {"estimated_cost": costs},
             ]
         )
+        return d
+        # if table:
+        #     d.to_scenario_list().print(format="rich")
+        # else:
+        #     return d
+
+    def show_prompts(self) -> None:
+        """Print the prompts."""
+        self.prompts().to_scenario_list().print(format="rich")
+
+    def estimate_job_cost(self):
+        from edsl import Coop
+
+        c = Coop()
+        price_lookup = c.fetch_prices()
+
+        prompts = self.prompts()
+
+        text_len = 0
+        for prompt in prompts:
+            text_len += len(str(prompt))
+
+        input_token_aproximations = text_len // 4
+
+        aproximation_cost = {}
+        total_cost = 0
+        for model in self.models:
+            key = (model._inference_service_, model.model)
+            relevant_prices = price_lookup[key]
+            inverse_output_price = relevant_prices["output"]["one_usd_buys"]
+            inverse_input_price = relevant_prices["input"]["one_usd_buys"]
+
+            aproximation_cost[key] = {
+                "input": input_token_aproximations / float(inverse_input_price),
+                "output": input_token_aproximations / float(inverse_output_price),
+            }
+            ##TODO curenlty we approximate the number of output tokens with the number
+            # of input tokens. A better solution will be to compute the quesiton answer options length and sum them
+            # to compute the output tokens
+
+            total_cost += input_token_aproximations / float(inverse_input_price)
+            total_cost += input_token_aproximations / float(inverse_output_price)
+
+        # multiply_factor = len(self.agents or [1]) * len(self.scenarios or [1])
+        multiply_factor = 1
+        out = {
+            "input_token_aproximations": input_token_aproximations,
+            "models_costs": aproximation_cost,
+            "estimated_total_cost": total_cost * multiply_factor,
+            "multiply_factor": multiply_factor,
+            "single_config_cost": total_cost,
+        }
+
+        return out
 
     @staticmethod
     def _get_container_class(object):
@@ -460,6 +541,12 @@ class Jobs(Base):
                 if warn:
                     warnings.warn(message)
 
+        if self.scenarios.has_jinja_braces:
+            warnings.warn(
+                "The scenarios have Jinja braces ({{ and }}). Converting to '<<' and '>>'. If you want a different conversion, use the convert_jinja_braces method first to modify the scenario."
+            )
+            self.scenarios = self.scenarios.convert_jinja_braces()
+
     @property
     def skip_retry(self):
         if not hasattr(self, "_skip_retry"):
@@ -486,6 +573,7 @@ class Jobs(Base):
         remote_inference_description: Optional[str] = None,
         skip_retry: bool = False,
         raise_validation_errors: bool = False,
+        disable_remote_inference: bool = False,
     ) -> Results:
         """
         Runs the Job: conducts Interviews and returns their results.
@@ -508,14 +596,17 @@ class Jobs(Base):
 
         self.verbose = verbose
 
-        try:
-            coop = Coop()
-            user_edsl_settings = coop.edsl_settings
-            remote_cache = user_edsl_settings["remote_caching"]
-            remote_inference = user_edsl_settings["remote_inference"]
-        except Exception:
-            remote_cache = False
-            remote_inference = False
+        remote_cache = False
+        remote_inference = False
+
+        if not disable_remote_inference:
+            try:
+                coop = Coop()
+                user_edsl_settings = Coop().edsl_settings
+                remote_cache = user_edsl_settings.get("remote_caching", False)
+                remote_inference = user_edsl_settings.get("remote_inference", False)
+            except Exception:
+                pass
 
         if remote_inference:
             import time

edsl/jobs/buckets/BucketCollection.py

@@ -13,6 +13,8 @@ class BucketCollection(UserDict):
     def __init__(self, infinity_buckets=False):
         super().__init__()
         self.infinity_buckets = infinity_buckets
+        self.models_to_services = {}
+        self.services_to_buckets = {}
 
     def __repr__(self):
         return f"BucketCollection({self.data})"
@@ -21,6 +23,7 @@ class BucketCollection(UserDict):
         """Adds a model to the bucket collection.
 
         This will create the token and request buckets for the model."""
+
         # compute the TPS and RPS from the model
         if not self.infinity_buckets:
             TPS = model.TPM / 60.0
@@ -29,22 +32,28 @@ class BucketCollection(UserDict):
             TPS = float("inf")
             RPS = float("inf")
 
-        # create the buckets
-        requests_bucket = TokenBucket(
-            bucket_name=model.model,
-            bucket_type="requests",
-            capacity=RPS,
-            refill_rate=RPS,
-        )
-        tokens_bucket = TokenBucket(
-            bucket_name=model.model, bucket_type="tokens", capacity=TPS, refill_rate=TPS
-        )
-        model_buckets = ModelBuckets(requests_bucket, tokens_bucket)
-        if model in self:
-            # it if already exists, combine the buckets
-            self[model] += model_buckets
+        if model.model not in self.models_to_services:
+            service = model._inference_service_
+            if service not in self.services_to_buckets:
+                requests_bucket = TokenBucket(
+                    bucket_name=service,
+                    bucket_type="requests",
+                    capacity=RPS,
+                    refill_rate=RPS,
+                )
+                tokens_bucket = TokenBucket(
+                    bucket_name=service,
+                    bucket_type="tokens",
+                    capacity=TPS,
+                    refill_rate=TPS,
+                )
+                self.services_to_buckets[service] = ModelBuckets(
+                    requests_bucket, tokens_bucket
+                )
+            self.models_to_services[model.model] = service
+            self[model] = self.services_to_buckets[service]
         else:
-            self[model] = model_buckets
+            self[model] = self.services_to_buckets[self.models_to_services[model.model]]
 
     def visualize(self) -> dict:
         """Visualize the token and request buckets for each model."""

edsl/jobs/buckets/TokenBucket.py

@@ -1,4 +1,4 @@
-from typing import Union, List, Any
+from typing import Union, List, Any, Optional
 import asyncio
 import time
 
@@ -17,6 +17,12 @@ class TokenBucket:
         self.bucket_name = bucket_name
         self.bucket_type = bucket_type
         self.capacity = capacity  # Maximum number of tokens
+        self.added_tokens = 0
+
+        self.target_rate = (
+            capacity * 60
+        )  # set this here because it can change with turbo mode
+
         self._old_capacity = capacity
         self.tokens = capacity  # Current number of available tokens
         self.refill_rate = refill_rate  # Rate at which tokens are refilled
@@ -25,6 +31,12 @@ class TokenBucket:
         self.log: List[Any] = []
         self.turbo_mode = False
 
+        self.creation_time = time.monotonic()
+
+        self.num_requests = 0
+        self.num_released = 0
+        self.tokens_returned = 0
+
     def turbo_mode_on(self):
         """Set the refill rate to infinity."""
         if self.turbo_mode:
@@ -69,6 +81,7 @@ class TokenBucket:
         >>> bucket.tokens
         10
         """
+        self.tokens_returned += tokens
         self.tokens = min(self.capacity, self.tokens + tokens)
         self.log.append((time.monotonic(), self.tokens))
 
@@ -133,15 +146,12 @@ class TokenBucket:
         >>> bucket.capacity
         12.100000000000001
         """
+        self.num_requests += amount
         if amount >= self.capacity:
             if not cheat_bucket_capacity:
                 msg = f"Requested amount exceeds bucket capacity. Bucket capacity: {self.capacity}, requested amount: {amount}. As the bucket never overflows, the requested amount will never be available."
                 raise ValueError(msg)
             else:
-                # self.tokens = 0  # clear the bucket but let it go through
-                # print(
-                #    f"""The requested amount, {amount}, exceeds the current bucket capacity of {self.capacity}.Increasing bucket capacity to {amount} * 1.10 accommodate the requested amount."""
-                # )
                 self.capacity = amount * 1.10
                 self._old_capacity = self.capacity
 
@@ -153,14 +163,10 @@ class TokenBucket:
                 break
 
             wait_time = self.wait_time(amount)
-            # print(f"Waiting for {wait_time:.4f} seconds")
             if wait_time > 0:
-                # print(f"Waiting for {wait_time:.4f} seconds")
                 await asyncio.sleep(wait_time)
 
-        # total_elapsed = time.monotonic() - start_time
-        # print(f"Total time to acquire tokens: {total_elapsed:.4f} seconds")
-
+        self.num_released += amount
         now = time.monotonic()
         self.log.append((now, self.tokens))
         return None
@@ -187,6 +193,54 @@ class TokenBucket:
         plt.tight_layout()
         plt.show()
 
+    def get_throughput(self, time_window: Optional[float] = None) -> float:
+        """
+        Calculate the empirical bucket throughput in tokens per minute for the specified time window.
+
+        :param time_window: The time window in seconds to calculate the throughput for.
+        :return: The throughput in tokens per minute.
+
+        >>> bucket = TokenBucket(bucket_name="test", bucket_type="test", capacity=100, refill_rate=10)
+        >>> asyncio.run(bucket.get_tokens(50))
+        >>> time.sleep(1)  # Wait for 1 second
+        >>> asyncio.run(bucket.get_tokens(30))
+        >>> throughput = bucket.get_throughput(1)
+        >>> 4750 < throughput < 4850
+        True
+        """
+        now = time.monotonic()
+
+        if time_window is None:
+            start_time = self.creation_time
+        else:
+            start_time = now - time_window
+
+        if start_time < self.creation_time:
+            start_time = self.creation_time
+
+        elapsed_time = now - start_time
+
+        return (self.num_released / elapsed_time) * 60
+
+        # # Filter log entries within the time window
+        # relevant_log = [(t, tokens) for t, tokens in self.log if t >= start_time]
+
+        # if len(relevant_log) < 2:
+        #     return 0  # Not enough data points to calculate throughput
+
+        # # Calculate total tokens used
+        # initial_tokens = relevant_log[0][1]
+        # final_tokens = relevant_log[-1][1]
+        # tokens_used = self.num_released - (final_tokens - initial_tokens)
+
+        # # Calculate actual time elapsed
+        # actual_time_elapsed = relevant_log[-1][0] - relevant_log[0][0]
+
+        # # Calculate throughput in tokens per minute
+        # throughput = (tokens_used / actual_time_elapsed) * 60
+
+        # return throughput
+
 
 if __name__ == "__main__":
     import doctest