edsl 0.1.32__py3-none-any.whl → 0.1.33__py3-none-any.whl

edsl/inference_services/TestService.py

@@ -0,0 +1,80 @@
+from typing import Any, List
+import os
+import asyncio
+from edsl.inference_services.InferenceServiceABC import InferenceServiceABC
+from edsl.language_models import LanguageModel
+from edsl.inference_services.rate_limits_cache import rate_limits
+from edsl.utilities.utilities import fix_partial_correct_response
+
+from edsl.enums import InferenceServiceType
+import random
+
+
+class TestService(InferenceServiceABC):
+    """OpenAI service class."""
+
+    _inference_service_ = "test"
+    _env_key_name_ = None
+    _base_url_ = None
+
+    _sync_client_ = None
+    _async_client_ = None
+
+    _sync_client_instance = None
+    _async_client_instance = None
+
+    key_sequence = None
+    usage_sequence = None
+    model_exclude_list = []
+    input_token_name = "prompt_tokens"
+    output_token_name = "completion_tokens"
+
+    @classmethod
+    def available(cls) -> list[str]:
+        return ["test"]
+
+    @classmethod
+    def create_model(cls, model_name, model_class_name=None) -> LanguageModel:
+        throw_exception = False
+
+        class TestServiceLanguageModel(LanguageModel):
+            _model_ = "test"
+            _parameters_ = {"temperature": 0.5}
+            _inference_service_ = InferenceServiceType.TEST.value
+            usage_sequence = ["usage"]
+            key_sequence = ["message", 0, "text"]
+            input_token_name = cls.input_token_name
+            output_token_name = cls.output_token_name
+            _rpm = 1000
+            _tpm = 100000
+
+            @property
+            def _canned_response(self):
+                if hasattr(self, "canned_response"):
+                    return self.canned_response
+                else:
+                    return "Hello, world"
+
+            async def async_execute_model_call(
+                self,
+                user_prompt: str,
+                system_prompt: str,
+                encoded_image=None,
+            ) -> dict[str, Any]:
+                await asyncio.sleep(0.1)
+                # return {"message": """{"answer": "Hello, world"}"""}
+
+                if hasattr(self, "throw_exception") and self.throw_exception:
+                    if hasattr(self, "exception_probability"):
+                        p = self.exception_probability
+                    else:
+                        p = 1
+
+                    if random.random() < p:
+                        raise Exception("This is a test error")
+                return {
+                    "message": [{"text": f"{self._canned_response}"}],
+                    "usage": {"prompt_tokens": 1, "completion_tokens": 1},
+                }
+
+        return TestServiceLanguageModel

edsl/inference_services/TogetherAIService.py

@@ -0,0 +1,170 @@
+import aiohttp
+import json
+import requests
+from typing import Any, List
+
+# from edsl.inference_services.InferenceServiceABC import InferenceServiceABC
+from edsl.language_models import LanguageModel
+
+from edsl.inference_services.OpenAIService import OpenAIService
+import openai
+
+
+class TogetherAIService(OpenAIService):
+    """DeepInfra service class."""
+
+    _inference_service_ = "together"
+    _env_key_name_ = "TOGETHER_API_KEY"
+    _base_url_ = "https://api.together.xyz/v1"
+    _models_list_cache: List[str] = []
+
+    # These are non-serverless models. There was no api param to filter them
+    model_exclude_list = [
+        "EleutherAI/llemma_7b",
+        "HuggingFaceH4/zephyr-7b-beta",
+        "Nexusflow/NexusRaven-V2-13B",
+        "NousResearch/Hermes-2-Theta-Llama-3-70B",
+        "NousResearch/Nous-Capybara-7B-V1p9",
+        "NousResearch/Nous-Hermes-13b",
+        "NousResearch/Nous-Hermes-2-Mistral-7B-DPO",
+        "NousResearch/Nous-Hermes-2-Mixtral-8x7B-SFT",
+        "NousResearch/Nous-Hermes-Llama2-13b",
+        "NousResearch/Nous-Hermes-Llama2-70b",
+        "NousResearch/Nous-Hermes-llama-2-7b",
+        "NumbersStation/nsql-llama-2-7B",
+        "Open-Orca/Mistral-7B-OpenOrca",
+        "Phind/Phind-CodeLlama-34B-Python-v1",
+        "Phind/Phind-CodeLlama-34B-v2",
+        "Qwen/Qwen1.5-0.5B",
+        "Qwen/Qwen1.5-0.5B-Chat",
+        "Qwen/Qwen1.5-1.8B",
+        "Qwen/Qwen1.5-1.8B-Chat",
+        "Qwen/Qwen1.5-14B",
+        "Qwen/Qwen1.5-14B-Chat",
+        "Qwen/Qwen1.5-32B",
+        "Qwen/Qwen1.5-32B-Chat",
+        "Qwen/Qwen1.5-4B",
+        "Qwen/Qwen1.5-4B-Chat",
+        "Qwen/Qwen1.5-72B",
+        "Qwen/Qwen1.5-7B",
+        "Qwen/Qwen1.5-7B-Chat",
+        "Qwen/Qwen2-1.5B",
+        "Qwen/Qwen2-1.5B-Instruct",
+        "Qwen/Qwen2-72B",
+        "Qwen/Qwen2-7B",
+        "Qwen/Qwen2-7B-Instruct",
+        "SG161222/Realistic_Vision_V3.0_VAE",
+        "Snowflake/snowflake-arctic-instruct",
+        "Undi95/ReMM-SLERP-L2-13B",
+        "Undi95/Toppy-M-7B",
+        "WizardLM/WizardCoder-Python-34B-V1.0",
+        "WizardLM/WizardLM-13B-V1.2",
+        "WizardLM/WizardLM-70B-V1.0",
+        "allenai/OLMo-7B",
+        "allenai/OLMo-7B-Instruct",
+        "bert-base-uncased",
+        "codellama/CodeLlama-13b-Instruct-hf",
+        "codellama/CodeLlama-13b-Python-hf",
+        "codellama/CodeLlama-13b-hf",
+        "codellama/CodeLlama-34b-Python-hf",
+        "codellama/CodeLlama-34b-hf",
+        "codellama/CodeLlama-70b-Instruct-hf",
+        "codellama/CodeLlama-70b-Python-hf",
+        "codellama/CodeLlama-70b-hf",
+        "codellama/CodeLlama-7b-Instruct-hf",
+        "codellama/CodeLlama-7b-Python-hf",
+        "codellama/CodeLlama-7b-hf",
+        "cognitivecomputations/dolphin-2.5-mixtral-8x7b",
+        "deepseek-ai/deepseek-coder-33b-instruct",
+        "garage-bAInd/Platypus2-70B-instruct",
+        "google/gemma-2b",
+        "google/gemma-7b",
+        "google/gemma-7b-it",
+        "gradientai/Llama-3-70B-Instruct-Gradient-1048k",
+        "hazyresearch/M2-BERT-2k-Retrieval-Encoder-V1",
+        "huggyllama/llama-13b",
+        "huggyllama/llama-30b",
+        "huggyllama/llama-65b",
+        "huggyllama/llama-7b",
+        "lmsys/vicuna-13b-v1.3",
+        "lmsys/vicuna-13b-v1.5",
+        "lmsys/vicuna-13b-v1.5-16k",
+        "lmsys/vicuna-7b-v1.3",
+        "lmsys/vicuna-7b-v1.5",
+        "meta-llama/Llama-2-13b-hf",
+        "meta-llama/Llama-2-70b-chat-hf",
+        "meta-llama/Llama-2-7b-hf",
+        "meta-llama/Llama-3-70b-hf",
+        "meta-llama/Llama-3-8b-hf",
+        "meta-llama/Meta-Llama-3-70B",
+        "meta-llama/Meta-Llama-3-70B-Instruct",
+        "meta-llama/Meta-Llama-3-8B-Instruct",
+        "meta-llama/Meta-Llama-3.1-70B-Instruct-Reference",
+        "meta-llama/Meta-Llama-3.1-70B-Reference",
+        "meta-llama/Meta-Llama-3.1-8B-Reference",
+        "microsoft/phi-2",
+        "mistralai/Mixtral-8x22B",
+        "openchat/openchat-3.5-1210",
+        "prompthero/openjourney",
+        "runwayml/stable-diffusion-v1-5",
+        "sentence-transformers/msmarco-bert-base-dot-v5",
+        "snorkelai/Snorkel-Mistral-PairRM-DPO",
+        "stabilityai/stable-diffusion-2-1",
+        "teknium/OpenHermes-2-Mistral-7B",
+        "teknium/OpenHermes-2p5-Mistral-7B",
+        "togethercomputer/CodeLlama-13b-Instruct",
+        "togethercomputer/CodeLlama-13b-Python",
+        "togethercomputer/CodeLlama-34b",
+        "togethercomputer/CodeLlama-34b-Python",
+        "togethercomputer/CodeLlama-7b-Instruct",
+        "togethercomputer/CodeLlama-7b-Python",
+        "togethercomputer/Koala-13B",
+        "togethercomputer/Koala-7B",
+        "togethercomputer/LLaMA-2-7B-32K",
+        "togethercomputer/SOLAR-10.7B-Instruct-v1.0-int4",
+        "togethercomputer/StripedHyena-Hessian-7B",
+        "togethercomputer/alpaca-7b",
+        "togethercomputer/evo-1-131k-base",
+        "togethercomputer/evo-1-8k-base",
+        "togethercomputer/guanaco-13b",
+        "togethercomputer/guanaco-33b",
+        "togethercomputer/guanaco-65b",
+        "togethercomputer/guanaco-7b",
+        "togethercomputer/llama-2-13b",
+        "togethercomputer/llama-2-70b-chat",
+        "togethercomputer/llama-2-7b",
+        "wavymulder/Analog-Diffusion",
+        "zero-one-ai/Yi-34B",
+        "zero-one-ai/Yi-34B-Chat",
+        "zero-one-ai/Yi-6B",
+    ]
+
+    _sync_client_ = openai.OpenAI
+    _async_client_ = openai.AsyncOpenAI
+
+    @classmethod
+    def get_model_list(cls):
+        # Togheter.ai has a different response in model list then openai
+        # and the OpenAI class returns an error when calling .models.list()
+        import requests
+        import os
+
+        url = "https://api.together.xyz/v1/models?filter=serverless"
+        token = os.getenv(cls._env_key_name_)
+        headers = {"accept": "application/json", "authorization": f"Bearer {token}"}
+
+        response = requests.get(url, headers=headers)
+        return response.json()
+
+    @classmethod
+    def available(cls) -> List[str]:
+        if not cls._models_list_cache:
+            try:
+                cls._models_list_cache = [
+                    m["id"]
+                    for m in cls.get_model_list()
+                    if m["id"] not in cls.model_exclude_list
+                ]
+            except Exception as e:
+                raise
+        return cls._models_list_cache

edsl/inference_services/models_available_cache.py

@@ -70,12 +70,6 @@ models_available = {
         "amazon.titan-tg1-large",
         "amazon.titan-text-lite-v1",
         "amazon.titan-text-express-v1",
-        "ai21.j2-grande-instruct",
-        "ai21.j2-jumbo-instruct",
-        "ai21.j2-mid",
-        "ai21.j2-mid-v1",
-        "ai21.j2-ultra",
-        "ai21.j2-ultra-v1",
         "anthropic.claude-instant-v1",
         "anthropic.claude-v2:1",
         "anthropic.claude-v2",

edsl/inference_services/registry.py

@@ -10,6 +10,9 @@ from edsl.inference_services.GroqService import GroqService
 from edsl.inference_services.AwsBedrock import AwsBedrockService
 from edsl.inference_services.AzureAI import AzureAIService
 from edsl.inference_services.OllamaService import OllamaService
+from edsl.inference_services.TestService import TestService
+from edsl.inference_services.MistralAIService import MistralAIService
+from edsl.inference_services.TogetherAIService import TogetherAIService
 
 default = InferenceServicesCollection(
     [
@@ -21,5 +24,8 @@ default = InferenceServicesCollection(
         AwsBedrockService,
         AzureAIService,
         OllamaService,
+        TestService,
+        MistralAIService,
+        TogetherAIService,
     ]
 )

edsl/jobs/Answers.py

@@ -2,24 +2,22 @@
 
 from collections import UserDict
 from rich.table import Table
+from edsl.data_transfer_models import EDSLResultObjectInput
 
 
 class Answers(UserDict):
     """Helper class to hold the answers to a survey."""
 
-    def add_answer(self, response, question) -> None:
-        """Add a response to the answers dictionary.
-
-        >>> from edsl import QuestionFreeText
-        >>> q = QuestionFreeText.example()
-        >>> answers = Answers()
-        >>> answers.add_answer({"answer": "yes"}, q)
-        >>> answers[q.question_name]
-        'yes'
-        """
-        answer = response.get("answer")
-        comment = response.pop("comment", None)
+    def add_answer(
+        self, response: EDSLResultObjectInput, question: "QuestionBase"
+    ) -> None:
+        """Add a response to the answers dictionary."""
+        answer = response.answer
+        comment = response.comment
+        generated_tokens = response.generated_tokens
         # record the answer
+        if generated_tokens:
+            self[question.question_name + "_generated_tokens"] = generated_tokens
         self[question.question_name] = answer
         if comment:
             self[question.question_name + "_comment"] = comment

edsl/jobs/FailedQuestion.py

@@ -0,0 +1,78 @@
+from edsl.questions import QuestionBase
+from edsl import Question, Scenario, Model, Agent
+
+from edsl.language_models.LanguageModel import LanguageModel
+
+
+class FailedQuestion:
+    # tests/jobs/test_Interview.py::test_handle_model_exceptions
+
+    # (Pdb) dir(self.exception.__traceback__)
+    # ['tb_frame', 'tb_lasti', 'tb_lineno', 'tb_next']
+
+    def __init__(
+        self, question, scenario, model, agent, raw_model_response, exception, prompts
+    ):
+        self.question = question
+        self.scenario = scenario
+        self.model = model
+        self.agent = agent
+        self.raw_model_response = raw_model_response  # JSON
+        self.exception = exception
+        self.prompts = prompts
+
+    def to_dict(self):
+        return {
+            "question": self.question._to_dict(),
+            "scenario": self.scenario._to_dict(),
+            "model": self.model._to_dict(),
+            "agent": self.agent._to_dict(),
+            "raw_model_response": self.raw_model_response,
+            "exception": self.exception.__class__.__name__,  # self.exception,
+            "prompts": self.prompts,
+        }
+
+    @classmethod
+    def from_dict(cls, data):
+        question = QuestionBase.from_dict(data["question"])
+        scenario = Scenario.from_dict(data["scenario"])
+        model = LanguageModel.from_dict(data["model"])
+        agent = Agent.from_dict(data["agent"])
+        raw_model_response = data["raw_model_response"]
+        exception = data["exception"]
+        prompts = data["prompts"]
+        return cls(
+            question, scenario, model, agent, raw_model_response, exception, prompts
+        )
+
+    def __repr__(self):
+        return f"{self.__class__.__name__}(question={repr(self.question)}, scenario={repr(self.scenario)}, model={repr(self.model)}, agent={repr(self.agent)}, raw_model_response={repr(self.raw_model_response)}, exception={repr(self.exception)})"
+
+    @property
+    def jobs(self):
+        return self.question.by(self.scenario).by(self.agent).by(self.model)
+
+    def rerun(self):
+        results = self.jobs.run()
+        return results
+
+    def help(self):
+        pass
+
+    @classmethod
+    def example(cls):
+        from edsl.language_models.utilities import create_language_model
+        from edsl.language_models.utilities import create_survey
+
+        survey = create_survey(2, chained=False, take_scenario=False)
+        fail_at_number = 1
+        model = create_language_model(ValueError, fail_at_number)()
+        from edsl import Survey
+
+        results = survey.by(model).run()
+        return results.failed_questions[0][0]
+
+
+if __name__ == "__main__":
+    fq = FailedQuestion.example()
+    new_fq = FailedQuestion.from_dict(fq.to_dict())

edsl/jobs/Jobs.py

@@ -156,7 +156,11 @@ class Jobs(Base):
         from edsl.results.Dataset import Dataset
 
         for interview_index, interview in enumerate(interviews):
-            invigilators = list(interview._build_invigilators(debug=False))
+            invigilators = [
+                interview._get_invigilator(question)
+                for question in self.survey.questions
+            ]
+            # list(interview._build_invigilators(debug=False))
             for _, invigilator in enumerate(invigilators):
                 prompts = invigilator.get_prompts()
                 user_prompts.append(prompts["user_prompt"])
@@ -344,6 +348,7 @@ class Jobs(Base):
                 scenario=scenario,
                 model=model,
                 skip_retry=self.skip_retry,
+                raise_validation_errors=self.raise_validation_errors,
             )
 
     def create_bucket_collection(self) -> BucketCollection:
@@ -455,33 +460,44 @@ class Jobs(Base):
                 if warn:
                     warnings.warn(message)
 
+        if self.scenarios.has_jinja_braces:
+            warnings.warn(
+                "The scenarios have Jinja braces ({{ and }}). Converting to '<<' and '>>'. If you want a different conversion, use the convert_jinja_braces method first to modify the scenario."
+            )
+            self.scenarios = self.scenarios.convert_jinja_braces()
+
     @property
     def skip_retry(self):
         if not hasattr(self, "_skip_retry"):
             return False
         return self._skip_retry
 
+    @property
+    def raise_validation_errors(self):
+        if not hasattr(self, "_raise_validation_errors"):
+            return False
+        return self._raise_validation_errors
+
     def run(
         self,
         n: int = 1,
-        debug: bool = False,
         progress_bar: bool = False,
         stop_on_exception: bool = False,
         cache: Union[Cache, bool] = None,
         check_api_keys: bool = False,
         sidecar_model: Optional[LanguageModel] = None,
-        batch_mode: Optional[bool] = None,
         verbose: bool = False,
         print_exceptions=True,
         remote_cache_description: Optional[str] = None,
         remote_inference_description: Optional[str] = None,
         skip_retry: bool = False,
+        raise_validation_errors: bool = False,
+        disable_remote_inference: bool = False,
     ) -> Results:
         """
         Runs the Job: conducts Interviews and returns their results.
 
         :param n: how many times to run each interview
-        :param debug: prints debug messages
         :param progress_bar: shows a progress bar
         :param stop_on_exception: stops the job if an exception is raised
         :param cache: a cache object to store results
@@ -495,22 +511,21 @@ class Jobs(Base):
 
         self._check_parameters()
         self._skip_retry = skip_retry
-
-        if batch_mode is not None:
-            raise NotImplementedError(
-                "Batch mode is deprecated. Please update your code to not include 'batch_mode' in the 'run' method."
-            )
+        self._raise_validation_errors = raise_validation_errors
 
         self.verbose = verbose
 
-        try:
-            coop = Coop()
-            user_edsl_settings = coop.edsl_settings
-            remote_cache = user_edsl_settings["remote_caching"]
-            remote_inference = user_edsl_settings["remote_inference"]
-        except Exception:
-            remote_cache = False
-            remote_inference = False
+        remote_cache = False
+        remote_inference = False
+
+        if not disable_remote_inference:
+            try:
+                coop = Coop()
+                user_edsl_settings = Coop().edsl_settings
+                remote_cache = user_edsl_settings.get("remote_caching", False)
+                remote_inference = user_edsl_settings.get("remote_inference", False)
+            except Exception:
+                pass
 
         if remote_inference:
             import time
@@ -562,7 +577,7 @@ class Jobs(Base):
                     )
                     return results
                 else:
-                    duration = 10 if len(self) < 10 else 60
+                    duration = 5
                     time_checked = datetime.now().strftime("%Y-%m-%d %I:%M:%S %p")
                     frames = ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"]
                     start_time = time.time()
@@ -587,7 +602,7 @@ class Jobs(Base):
                         )
 
         # handle cache
-        if cache is None:
+        if cache is None or cache is True:
             from edsl.data.CacheHandler import CacheHandler
 
             cache = CacheHandler().get_cache()
@@ -599,12 +614,12 @@ class Jobs(Base):
         if not remote_cache:
             results = self._run_local(
                 n=n,
-                debug=debug,
                 progress_bar=progress_bar,
                 cache=cache,
                 stop_on_exception=stop_on_exception,
                 sidecar_model=sidecar_model,
                 print_exceptions=print_exceptions,
+                raise_validation_errors=raise_validation_errors,
             )
 
             results.cache = cache.new_entries_cache()
@@ -643,12 +658,12 @@ class Jobs(Base):
             self._output("Running job...")
             results = self._run_local(
                 n=n,
-                debug=debug,
                 progress_bar=progress_bar,
                 cache=cache,
                 stop_on_exception=stop_on_exception,
                 sidecar_model=sidecar_model,
                 print_exceptions=print_exceptions,
+                raise_validation_errors=raise_validation_errors,
             )
             self._output("Job completed!")
 
@@ -883,7 +898,7 @@ def main():
 
     job = Jobs.example()
     len(job) == 8
-    results = job.run(debug=True, cache=Cache())
+    results = job.run(cache=Cache())
     len(results) == 8
     results
 

edsl/jobs/buckets/BucketCollection.py

@@ -13,6 +13,8 @@ class BucketCollection(UserDict):
     def __init__(self, infinity_buckets=False):
         super().__init__()
         self.infinity_buckets = infinity_buckets
+        self.models_to_services = {}
+        self.services_to_buckets = {}
 
     def __repr__(self):
         return f"BucketCollection({self.data})"
@@ -21,6 +23,7 @@ class BucketCollection(UserDict):
         """Adds a model to the bucket collection.
 
         This will create the token and request buckets for the model."""
+
         # compute the TPS and RPS from the model
         if not self.infinity_buckets:
             TPS = model.TPM / 60.0
@@ -29,22 +32,28 @@ class BucketCollection(UserDict):
             TPS = float("inf")
             RPS = float("inf")
 
-        # create the buckets
-        requests_bucket = TokenBucket(
-            bucket_name=model.model,
-            bucket_type="requests",
-            capacity=RPS,
-            refill_rate=RPS,
-        )
-        tokens_bucket = TokenBucket(
-            bucket_name=model.model, bucket_type="tokens", capacity=TPS, refill_rate=TPS
-        )
-        model_buckets = ModelBuckets(requests_bucket, tokens_bucket)
-        if model in self:
-            # it if already exists, combine the buckets
-            self[model] += model_buckets
+        if model.model not in self.models_to_services:
+            service = model._inference_service_
+            if service not in self.services_to_buckets:
+                requests_bucket = TokenBucket(
+                    bucket_name=service,
+                    bucket_type="requests",
+                    capacity=RPS,
+                    refill_rate=RPS,
+                )
+                tokens_bucket = TokenBucket(
+                    bucket_name=service,
+                    bucket_type="tokens",
+                    capacity=TPS,
+                    refill_rate=TPS,
+                )
+                self.services_to_buckets[service] = ModelBuckets(
+                    requests_bucket, tokens_bucket
+                )
+            self.models_to_services[model.model] = service
+            self[model] = self.services_to_buckets[service]
         else:
-            self[model] = model_buckets
+            self[model] = self.services_to_buckets[self.models_to_services[model.model]]
 
     def visualize(self) -> dict:
         """Visualize the token and request buckets for each model."""