edsl 0.1.31.dev4__py3-none-any.whl → 0.1.33__py3-none-any.whl

edsl/jobs/FailedQuestion.py

@@ -0,0 +1,78 @@
+from edsl.questions import QuestionBase
+from edsl import Question, Scenario, Model, Agent
+
+from edsl.language_models.LanguageModel import LanguageModel
+
+
+class FailedQuestion:
+    # tests/jobs/test_Interview.py::test_handle_model_exceptions
+
+    # (Pdb) dir(self.exception.__traceback__)
+    # ['tb_frame', 'tb_lasti', 'tb_lineno', 'tb_next']
+
+    def __init__(
+        self, question, scenario, model, agent, raw_model_response, exception, prompts
+    ):
+        self.question = question
+        self.scenario = scenario
+        self.model = model
+        self.agent = agent
+        self.raw_model_response = raw_model_response  # JSON
+        self.exception = exception
+        self.prompts = prompts
+
+    def to_dict(self):
+        return {
+            "question": self.question._to_dict(),
+            "scenario": self.scenario._to_dict(),
+            "model": self.model._to_dict(),
+            "agent": self.agent._to_dict(),
+            "raw_model_response": self.raw_model_response,
+            "exception": self.exception.__class__.__name__,  # self.exception,
+            "prompts": self.prompts,
+        }
+
+    @classmethod
+    def from_dict(cls, data):
+        question = QuestionBase.from_dict(data["question"])
+        scenario = Scenario.from_dict(data["scenario"])
+        model = LanguageModel.from_dict(data["model"])
+        agent = Agent.from_dict(data["agent"])
+        raw_model_response = data["raw_model_response"]
+        exception = data["exception"]
+        prompts = data["prompts"]
+        return cls(
+            question, scenario, model, agent, raw_model_response, exception, prompts
+        )
+
+    def __repr__(self):
+        return f"{self.__class__.__name__}(question={repr(self.question)}, scenario={repr(self.scenario)}, model={repr(self.model)}, agent={repr(self.agent)}, raw_model_response={repr(self.raw_model_response)}, exception={repr(self.exception)})"
+
+    @property
+    def jobs(self):
+        return self.question.by(self.scenario).by(self.agent).by(self.model)
+
+    def rerun(self):
+        results = self.jobs.run()
+        return results
+
+    def help(self):
+        pass
+
+    @classmethod
+    def example(cls):
+        from edsl.language_models.utilities import create_language_model
+        from edsl.language_models.utilities import create_survey
+
+        survey = create_survey(2, chained=False, take_scenario=False)
+        fail_at_number = 1
+        model = create_language_model(ValueError, fail_at_number)()
+        from edsl import Survey
+
+        results = survey.by(model).run()
+        return results.failed_questions[0][0]
+
+
+if __name__ == "__main__":
+    fq = FailedQuestion.example()
+    new_fq = FailedQuestion.from_dict(fq.to_dict())

edsl/jobs/Jobs.py

@@ -39,6 +39,8 @@ class Jobs(Base):
 
         self.__bucket_collection = None
 
+    # these setters and getters are used to ensure that the agents, models, and scenarios are stored as AgentList, ModelList, and ScenarioList objects
+
     @property
     def models(self):
         return self._models
@@ -119,7 +121,9 @@ class Jobs(Base):
         - scenarios: traits of new scenarios are combined with traits of old existing. New scenarios will overwrite overlapping traits, and do not increase the number of scenarios in the instance
         - models: new models overwrite old models.
         """
-        passed_objects = self._turn_args_to_list(args)
+        passed_objects = self._turn_args_to_list(
+            args
+        )  # objects can also be passed comma-separated
 
         current_objects, objects_key = self._get_current_objects_of_this_type(
             passed_objects[0]
@@ -152,7 +156,11 @@ class Jobs(Base):
         from edsl.results.Dataset import Dataset
 
         for interview_index, interview in enumerate(interviews):
-            invigilators = list(interview._build_invigilators(debug=False))
+            invigilators = [
+                interview._get_invigilator(question)
+                for question in self.survey.questions
+            ]
+            # list(interview._build_invigilators(debug=False))
             for _, invigilator in enumerate(invigilators):
                 prompts = invigilator.get_prompts()
                 user_prompts.append(prompts["user_prompt"])
@@ -176,17 +184,27 @@ class Jobs(Base):
         from edsl.agents.Agent import Agent
         from edsl.scenarios.Scenario import Scenario
         from edsl.scenarios.ScenarioList import ScenarioList
+        from edsl.language_models.ModelList import ModelList
 
         if isinstance(object, Agent):
             return AgentList
         elif isinstance(object, Scenario):
             return ScenarioList
+        elif isinstance(object, ModelList):
+            return ModelList
         else:
             return list
 
     @staticmethod
     def _turn_args_to_list(args):
-        """Return a list of the first argument if it is a sequence, otherwise returns a list of all the arguments."""
+        """Return a list of the first argument if it is a sequence, otherwise returns a list of all the arguments.
+
+        Example:
+
+        >>> Jobs._turn_args_to_list([1,2,3])
+        [1, 2, 3]
+
+        """
 
         def did_user_pass_a_sequence(args):
             """Return True if the user passed a sequence, False otherwise.
@@ -209,7 +227,7 @@ class Jobs(Base):
             return container_class(args)
 
     def _get_current_objects_of_this_type(
-        self, object: Union[Agent, Scenario, LanguageModel]
+        self, object: Union["Agent", "Scenario", "LanguageModel"]
     ) -> tuple[list, str]:
         from edsl.agents.Agent import Agent
         from edsl.scenarios.Scenario import Scenario
@@ -292,7 +310,11 @@ class Jobs(Base):
 
     @classmethod
     def from_interviews(cls, interview_list):
-        """Return a Jobs instance from a list of interviews."""
+        """Return a Jobs instance from a list of interviews.
+
+        This is useful when you have, say, a list of failed interviews and you want to create
+        a new job with only those interviews.
+        """
         survey = interview_list[0].survey
         # get all the models
         models = list(set([interview.model for interview in interview_list]))
@@ -308,6 +330,8 @@ class Jobs(Base):
         Note that this sets the agents, model and scenarios if they have not been set. This is a side effect of the method.
         This is useful because a user can create a job without setting the agents, models, or scenarios, and the job will still run,
         with us filling in defaults.
+
+
         """
         # if no agents, models, or scenarios are set, set them to defaults
         from edsl.agents.Agent import Agent
@@ -319,7 +343,12 @@ class Jobs(Base):
         self.scenarios = self.scenarios or [Scenario()]
         for agent, scenario, model in product(self.agents, self.scenarios, self.models):
             yield Interview(
-                survey=self.survey, agent=agent, scenario=scenario, model=model
+                survey=self.survey,
+                agent=agent,
+                scenario=scenario,
+                model=model,
+                skip_retry=self.skip_retry,
+                raise_validation_errors=self.raise_validation_errors,
             )
 
     def create_bucket_collection(self) -> BucketCollection:
@@ -359,10 +388,16 @@ class Jobs(Base):
         return links
 
     def __hash__(self):
-        """Allow the model to be used as a key in a dictionary."""
+        """Allow the model to be used as a key in a dictionary.
+
+        >>> from edsl.jobs import Jobs
+        >>> hash(Jobs.example())
+        846655441787442972
+
+        """
         from edsl.utilities.utilities import dict_hash
 
-        return dict_hash(self.to_dict())
+        return dict_hash(self._to_dict())
 
     def _output(self, message) -> None:
         """Check if a Job is verbose. If so, print the message."""
@@ -390,11 +425,27 @@ class Jobs(Base):
         Traceback (most recent call last):
         ...
         ValueError: The following parameters are in the scenarios but not in the survey: {'plop'}
+
+        >>> q = QuestionFreeText(question_text = "Hello", question_name = "ugly_question")
+        >>> s = Scenario({'ugly_question': "B"})
+        >>> j = Jobs(survey = Survey(questions=[q])).by(s)
+        >>> j._check_parameters()
+        Traceback (most recent call last):
+        ...
+        ValueError: The following names are in both the survey question_names and the scenario keys: {'ugly_question'}. This will create issues.
         """
         survey_parameters: set = self.survey.parameters
         scenario_parameters: set = self.scenarios.parameters
 
-        msg1, msg2 = None, None
+        msg0, msg1, msg2 = None, None, None
+
+        # look for key issues
+        if intersection := set(self.scenarios.parameters) & set(
+            self.survey.question_names
+        ):
+            msg0 = f"The following names are in both the survey question_names and the scenario keys: {intersection}. This will create issues."
+
+            raise ValueError(msg0)
 
         if in_survey_but_not_in_scenarios := survey_parameters - scenario_parameters:
             msg1 = f"The following parameters are in the survey but not in the scenarios: {in_survey_but_not_in_scenarios}"
@@ -409,26 +460,44 @@ class Jobs(Base):
                 if warn:
                     warnings.warn(message)
 
+        if self.scenarios.has_jinja_braces:
+            warnings.warn(
+                "The scenarios have Jinja braces ({{ and }}). Converting to '<<' and '>>'. If you want a different conversion, use the convert_jinja_braces method first to modify the scenario."
+            )
+            self.scenarios = self.scenarios.convert_jinja_braces()
+
+    @property
+    def skip_retry(self):
+        if not hasattr(self, "_skip_retry"):
+            return False
+        return self._skip_retry
+
+    @property
+    def raise_validation_errors(self):
+        if not hasattr(self, "_raise_validation_errors"):
+            return False
+        return self._raise_validation_errors
+
     def run(
         self,
         n: int = 1,
-        debug: bool = False,
         progress_bar: bool = False,
         stop_on_exception: bool = False,
         cache: Union[Cache, bool] = None,
         check_api_keys: bool = False,
         sidecar_model: Optional[LanguageModel] = None,
-        batch_mode: Optional[bool] = None,
         verbose: bool = False,
         print_exceptions=True,
         remote_cache_description: Optional[str] = None,
         remote_inference_description: Optional[str] = None,
+        skip_retry: bool = False,
+        raise_validation_errors: bool = False,
+        disable_remote_inference: bool = False,
     ) -> Results:
         """
         Runs the Job: conducts Interviews and returns their results.
 
         :param n: how many times to run each interview
-        :param debug: prints debug messages
         :param progress_bar: shows a progress bar
         :param stop_on_exception: stops the job if an exception is raised
         :param cache: a cache object to store results
@@ -441,22 +510,22 @@ class Jobs(Base):
         from edsl.coop.coop import Coop
 
         self._check_parameters()
-
-        if batch_mode is not None:
-            raise NotImplementedError(
-                "Batch mode is deprecated. Please update your code to not include 'batch_mode' in the 'run' method."
-            )
+        self._skip_retry = skip_retry
+        self._raise_validation_errors = raise_validation_errors
 
         self.verbose = verbose
 
-        try:
-            coop = Coop()
-            user_edsl_settings = coop.edsl_settings
-            remote_cache = user_edsl_settings["remote_caching"]
-            remote_inference = user_edsl_settings["remote_inference"]
-        except Exception:
-            remote_cache = False
-            remote_inference = False
+        remote_cache = False
+        remote_inference = False
+
+        if not disable_remote_inference:
+            try:
+                coop = Coop()
+                user_edsl_settings = Coop().edsl_settings
+                remote_cache = user_edsl_settings.get("remote_caching", False)
+                remote_inference = user_edsl_settings.get("remote_inference", False)
+            except Exception:
+                pass
 
         if remote_inference:
             import time
@@ -508,7 +577,7 @@ class Jobs(Base):
                     )
                     return results
                 else:
-                    duration = 10 if len(self) < 10 else 60
+                    duration = 5
                     time_checked = datetime.now().strftime("%Y-%m-%d %I:%M:%S %p")
                     frames = ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"]
                     start_time = time.time()
@@ -533,7 +602,7 @@ class Jobs(Base):
                         )
 
         # handle cache
-        if cache is None:
+        if cache is None or cache is True:
             from edsl.data.CacheHandler import CacheHandler
 
             cache = CacheHandler().get_cache()
@@ -545,12 +614,12 @@ class Jobs(Base):
         if not remote_cache:
             results = self._run_local(
                 n=n,
-                debug=debug,
                 progress_bar=progress_bar,
                 cache=cache,
                 stop_on_exception=stop_on_exception,
                 sidecar_model=sidecar_model,
                 print_exceptions=print_exceptions,
+                raise_validation_errors=raise_validation_errors,
             )
 
             results.cache = cache.new_entries_cache()
@@ -589,12 +658,12 @@ class Jobs(Base):
             self._output("Running job...")
             results = self._run_local(
                 n=n,
-                debug=debug,
                 progress_bar=progress_bar,
                 cache=cache,
                 stop_on_exception=stop_on_exception,
                 sidecar_model=sidecar_model,
                 print_exceptions=print_exceptions,
+                raise_validation_errors=raise_validation_errors,
             )
             self._output("Job completed!")
 
@@ -631,12 +700,16 @@ class Jobs(Base):
         return results
 
     async def run_async(self, cache=None, n=1, **kwargs):
-        """Run the job asynchronously."""
+        """Run asynchronously."""
         results = await JobsRunnerAsyncio(self).run_async(cache=cache, n=n, **kwargs)
         return results
 
     def all_question_parameters(self):
-        """Return all the fields in the questions in the survey."""
+        """Return all the fields in the questions in the survey.
+        >>> from edsl.jobs import Jobs
+        >>> Jobs.example().all_question_parameters()
+        {'period'}
+        """
         return set.union(*[question.parameters for question in self.survey.questions])
 
     #######################
@@ -677,15 +750,19 @@ class Jobs(Base):
     #######################
     # Serialization methods
     #######################
+
+    def _to_dict(self):
+        return {
+            "survey": self.survey._to_dict(),
+            "agents": [agent._to_dict() for agent in self.agents],
+            "models": [model._to_dict() for model in self.models],
+            "scenarios": [scenario._to_dict() for scenario in self.scenarios],
+        }
+
     @add_edsl_version
     def to_dict(self) -> dict:
         """Convert the Jobs instance to a dictionary."""
-        return {
-            "survey": self.survey.to_dict(),
-            "agents": [agent.to_dict() for agent in self.agents],
-            "models": [model.to_dict() for model in self.models],
-            "scenarios": [scenario.to_dict() for scenario in self.scenarios],
-        }
+        return self._to_dict()
 
     @classmethod
     @remove_edsl_version
@@ -704,7 +781,13 @@ class Jobs(Base):
         )
 
     def __eq__(self, other: Jobs) -> bool:
-        """Return True if the Jobs instance is equal to another Jobs instance."""
+        """Return True if the Jobs instance is equal to another Jobs instance.
+
+        >>> from edsl.jobs import Jobs
+        >>> Jobs.example() == Jobs.example()
+        True
+
+        """
         return self.to_dict() == other.to_dict()
 
     #######################
@@ -712,11 +795,16 @@ class Jobs(Base):
     #######################
     @classmethod
     def example(
-        cls, throw_exception_probability: int = 0, randomize: bool = False
+        cls,
+        throw_exception_probability: float = 0.0,
+        randomize: bool = False,
+        test_model=False,
     ) -> Jobs:
         """Return an example Jobs instance.
 
         :param throw_exception_probability: the probability that an exception will be thrown when answering a question. This is useful for testing error handling.
+        :param randomize: whether to randomize the job by adding a random string to the period
+        :param test_model: whether to use a test model
 
         >>> Jobs.example()
         Jobs(...)
@@ -730,6 +818,11 @@ class Jobs(Base):
 
         addition = "" if not randomize else str(uuid4())
 
+        if test_model:
+            from edsl.language_models import LanguageModel
+
+            m = LanguageModel.example(test_model=True)
+
         # (status, question, period)
         agent_answers = {
             ("Joyful", "how_feeling", "morning"): "OK",
@@ -777,7 +870,10 @@ class Jobs(Base):
                 Scenario({"period": "afternoon"}),
             ]
         )
-        job = base_survey.by(scenario_list).by(joy_agent, sad_agent)
+        if test_model:
+            job = base_survey.by(m).by(scenario_list).by(joy_agent, sad_agent)
+        else:
+            job = base_survey.by(scenario_list).by(joy_agent, sad_agent)
 
         return job
 
@@ -802,7 +898,7 @@ def main():
 
     job = Jobs.example()
     len(job) == 8
-    results = job.run(debug=True, cache=Cache())
+    results = job.run(cache=Cache())
     len(results) == 8
     results
 

edsl/jobs/buckets/BucketCollection.py

@@ -13,6 +13,8 @@ class BucketCollection(UserDict):
     def __init__(self, infinity_buckets=False):
         super().__init__()
         self.infinity_buckets = infinity_buckets
+        self.models_to_services = {}
+        self.services_to_buckets = {}
 
     def __repr__(self):
         return f"BucketCollection({self.data})"
@@ -21,6 +23,7 @@ class BucketCollection(UserDict):
         """Adds a model to the bucket collection.
 
         This will create the token and request buckets for the model."""
+
         # compute the TPS and RPS from the model
         if not self.infinity_buckets:
             TPS = model.TPM / 60.0
@@ -29,22 +32,28 @@ class BucketCollection(UserDict):
             TPS = float("inf")
             RPS = float("inf")
 
-        # create the buckets
-        requests_bucket = TokenBucket(
-            bucket_name=model.model,
-            bucket_type="requests",
-            capacity=RPS,
-            refill_rate=RPS,
-        )
-        tokens_bucket = TokenBucket(
-            bucket_name=model.model, bucket_type="tokens", capacity=TPS, refill_rate=TPS
-        )
-        model_buckets = ModelBuckets(requests_bucket, tokens_bucket)
-        if model in self:
-            # it if already exists, combine the buckets
-            self[model] += model_buckets
+        if model.model not in self.models_to_services:
+            service = model._inference_service_
+            if service not in self.services_to_buckets:
+                requests_bucket = TokenBucket(
+                    bucket_name=service,
+                    bucket_type="requests",
+                    capacity=RPS,
+                    refill_rate=RPS,
+                )
+                tokens_bucket = TokenBucket(
+                    bucket_name=service,
+                    bucket_type="tokens",
+                    capacity=TPS,
+                    refill_rate=TPS,
+                )
+                self.services_to_buckets[service] = ModelBuckets(
+                    requests_bucket, tokens_bucket
+                )
+            self.models_to_services[model.model] = service
+            self[model] = self.services_to_buckets[service]
         else:
-            self[model] = model_buckets
+            self[model] = self.services_to_buckets[self.models_to_services[model.model]]
 
     def visualize(self) -> dict:
         """Visualize the token and request buckets for each model."""