edsl 0.1.36.dev6__py3-none-any.whl → 0.1.37.dev2__py3-none-any.whl

edsl/jobs/Jobs.py

@@ -1,1112 +1,1112 @@
-# """The Jobs class is a collection of agents, scenarios and models and one survey."""
-from __future__ import annotations
-import warnings
-import requests
-from itertools import product
-from typing import Optional, Union, Sequence, Generator
-
-from edsl.Base import Base
-from edsl.exceptions import MissingAPIKeyError
-from edsl.jobs.buckets.BucketCollection import BucketCollection
-from edsl.jobs.interviews.Interview import Interview
-from edsl.jobs.runners.JobsRunnerAsyncio import JobsRunnerAsyncio
-from edsl.utilities.decorators import add_edsl_version, remove_edsl_version
-
-from edsl.data.RemoteCacheSync import RemoteCacheSync
-from edsl.exceptions.coop import CoopServerResponseError
-
-
-class Jobs(Base):
-    """
-    A collection of agents, scenarios and models and one survey.
-    The actual running of a job is done by a `JobsRunner`, which is a subclass of `JobsRunner`.
-    The `JobsRunner` is chosen by the user, and is stored in the `jobs_runner_name` attribute.
-    """
-
-    def __init__(
-        self,
-        survey: "Survey",
-        agents: Optional[list["Agent"]] = None,
-        models: Optional[list["LanguageModel"]] = None,
-        scenarios: Optional[list["Scenario"]] = None,
-    ):
-        """Initialize a Jobs instance.
-
-        :param survey: the survey to be used in the job
-        :param agents: a list of agents
-        :param models: a list of models
-        :param scenarios: a list of scenarios
-        """
-        self.survey = survey
-        self.agents: "AgentList" = agents
-        self.scenarios: "ScenarioList" = scenarios
-        self.models = models
-
-        self.__bucket_collection = None
-
-    # these setters and getters are used to ensure that the agents, models, and scenarios are stored as AgentList, ModelList, and ScenarioList objects
-
-    @property
-    def models(self):
-        return self._models
-
-    @models.setter
-    def models(self, value):
-        from edsl import ModelList
-
-        if value:
-            if not isinstance(value, ModelList):
-                self._models = ModelList(value)
-            else:
-                self._models = value
-        else:
-            self._models = ModelList([])
-
-    @property
-    def agents(self):
-        return self._agents
-
-    @agents.setter
-    def agents(self, value):
-        from edsl import AgentList
-
-        if value:
-            if not isinstance(value, AgentList):
-                self._agents = AgentList(value)
-            else:
-                self._agents = value
-        else:
-            self._agents = AgentList([])
-
-    @property
-    def scenarios(self):
-        return self._scenarios
-
-    @scenarios.setter
-    def scenarios(self, value):
-        from edsl import ScenarioList
-
-        if value:
-            if not isinstance(value, ScenarioList):
-                self._scenarios = ScenarioList(value)
-            else:
-                self._scenarios = value
-        else:
-            self._scenarios = ScenarioList([])
-
-    def by(
-        self,
-        *args: Union[
-            "Agent",
-            "Scenario",
-            "LanguageModel",
-            Sequence[Union["Agent", "Scenario", "LanguageModel"]],
-        ],
-    ) -> Jobs:
-        """
-        Add Agents, Scenarios and LanguageModels to a job. If no objects of this type exist in the Jobs instance, it stores the new objects as a list in the corresponding attribute. Otherwise, it combines the new objects with existing objects using the object's `__add__` method.
-
-        This 'by' is intended to create a fluent interface.
-
-        >>> from edsl import Survey
-        >>> from edsl import QuestionFreeText
-        >>> q = QuestionFreeText(question_name="name", question_text="What is your name?")
-        >>> j = Jobs(survey = Survey(questions=[q]))
-        >>> j
-        Jobs(survey=Survey(...), agents=AgentList([]), models=ModelList([]), scenarios=ScenarioList([]))
-        >>> from edsl import Agent; a = Agent(traits = {"status": "Sad"})
-        >>> j.by(a).agents
-        AgentList([Agent(traits = {'status': 'Sad'})])
-
-        :param args: objects or a sequence (list, tuple, ...) of objects of the same type
-
-        Notes:
-        - all objects must implement the 'get_value', 'set_value', and `__add__` methods
-        - agents: traits of new agents are combined with traits of existing agents. New and existing agents should not have overlapping traits, and do not increase the # agents in the instance
-        - scenarios: traits of new scenarios are combined with traits of old existing. New scenarios will overwrite overlapping traits, and do not increase the number of scenarios in the instance
-        - models: new models overwrite old models.
-        """
-        passed_objects = self._turn_args_to_list(
-            args
-        )  # objects can also be passed comma-separated
-
-        current_objects, objects_key = self._get_current_objects_of_this_type(
-            passed_objects[0]
-        )
-
-        if not current_objects:
-            new_objects = passed_objects
-        else:
-            new_objects = self._merge_objects(passed_objects, current_objects)
-
-        setattr(self, objects_key, new_objects)  # update the job
-        return self
-
-    def prompts(self) -> "Dataset":
-        """Return a Dataset of prompts that will be used.
-
-
-        >>> from edsl.jobs import Jobs
-        >>> Jobs.example().prompts()
-        Dataset(...)
-        """
-        from edsl import Coop
-
-        c = Coop()
-        price_lookup = c.fetch_prices()
-
-        interviews = self.interviews()
-        # data = []
-        interview_indices = []
-        question_names = []
-        user_prompts = []
-        system_prompts = []
-        scenario_indices = []
-        agent_indices = []
-        models = []
-        costs = []
-        from edsl.results.Dataset import Dataset
-
-        for interview_index, interview in enumerate(interviews):
-            invigilators = [
-                interview._get_invigilator(question)
-                for question in self.survey.questions
-            ]
-            for _, invigilator in enumerate(invigilators):
-                prompts = invigilator.get_prompts()
-                user_prompt = prompts["user_prompt"]
-                system_prompt = prompts["system_prompt"]
-                user_prompts.append(user_prompt)
-                system_prompts.append(system_prompt)
-                agent_index = self.agents.index(invigilator.agent)
-                agent_indices.append(agent_index)
-                interview_indices.append(interview_index)
-                scenario_index = self.scenarios.index(invigilator.scenario)
-                scenario_indices.append(scenario_index)
-                models.append(invigilator.model.model)
-                question_names.append(invigilator.question.question_name)
-
-                prompt_cost = self.estimate_prompt_cost(
-                    system_prompt=system_prompt,
-                    user_prompt=user_prompt,
-                    price_lookup=price_lookup,
-                    inference_service=invigilator.model._inference_service_,
-                    model=invigilator.model.model,
-                )
-                costs.append(prompt_cost["cost"])
-
-        d = Dataset(
-            [
-                {"user_prompt": user_prompts},
-                {"system_prompt": system_prompts},
-                {"interview_index": interview_indices},
-                {"question_name": question_names},
-                {"scenario_index": scenario_indices},
-                {"agent_index": agent_indices},
-                {"model": models},
-                {"estimated_cost": costs},
-            ]
-        )
-        return d
-
-    def show_prompts(self, all=False) -> None:
-        """Print the prompts."""
-        if all:
-            self.prompts().to_scenario_list().print(format="rich")
-        else:
-            self.prompts().select(
-                "user_prompt", "system_prompt"
-            ).to_scenario_list().print(format="rich")
-
-    @staticmethod
-    def estimate_prompt_cost(
-        system_prompt: str,
-        user_prompt: str,
-        price_lookup: dict,
-        inference_service: str,
-        model: str,
-    ) -> dict:
-        """Estimates the cost of a prompt. Takes piping into account."""
-
-        def get_piping_multiplier(prompt: str):
-            """Returns 2 if a prompt includes Jinja braces, and 1 otherwise."""
-
-            if "{{" in prompt and "}}" in prompt:
-                return 2
-            return 1
-
-        # Look up prices per token
-        key = (inference_service, model)
-
-        try:
-            relevant_prices = price_lookup[key]
-            output_price_per_token = 1 / float(
-                relevant_prices["output"]["one_usd_buys"]
-            )
-            input_price_per_token = 1 / float(relevant_prices["input"]["one_usd_buys"])
-        except KeyError:
-            # A KeyError is likely to occur if we cannot retrieve prices (the price_lookup dict is empty)
-            # Use a sensible default
-
-            import warnings
-
-            warnings.warn(
-                "Price data could not be retrieved. Using default estimates for input and output token prices. Input: $0.15 / 1M tokens; Output: $0.60 / 1M tokens"
-            )
-
-            output_price_per_token = 0.00000015  # $0.15 / 1M tokens
-            input_price_per_token = 0.00000060  # $0.60 / 1M tokens
-
-        # Compute the number of characters (double if the question involves piping)
-        user_prompt_chars = len(str(user_prompt)) * get_piping_multiplier(
-            str(user_prompt)
-        )
-        system_prompt_chars = len(str(system_prompt)) * get_piping_multiplier(
-            str(system_prompt)
-        )
-
-        # Convert into tokens (1 token approx. equals 4 characters)
-        input_tokens = (user_prompt_chars + system_prompt_chars) // 4
-        output_tokens = input_tokens
-
-        cost = (
-            input_tokens * input_price_per_token
-            + output_tokens * output_price_per_token
-        )
-
-        return {
-            "input_tokens": input_tokens,
-            "output_tokens": output_tokens,
-            "cost": cost,
-        }
-
-    def estimate_job_cost_from_external_prices(self, price_lookup: dict) -> dict:
-        """
-        Estimates the cost of a job according to the following assumptions:
-
-        - 1 token = 4 characters.
-        - Input tokens = output tokens.
-
-        price_lookup is an external pricing dictionary.
-        """
-
-        import pandas as pd
-
-        interviews = self.interviews()
-        data = []
-        for interview in interviews:
-            invigilators = [
-                interview._get_invigilator(question)
-                for question in self.survey.questions
-            ]
-            for invigilator in invigilators:
-                prompts = invigilator.get_prompts()
-
-                # By this point, agent and scenario data has already been added to the prompts
-                user_prompt = prompts["user_prompt"]
-                system_prompt = prompts["system_prompt"]
-                inference_service = invigilator.model._inference_service_
-                model = invigilator.model.model
-
-                prompt_cost = self.estimate_prompt_cost(
-                    system_prompt=system_prompt,
-                    user_prompt=user_prompt,
-                    price_lookup=price_lookup,
-                    inference_service=inference_service,
-                    model=model,
-                )
-
-                data.append(
-                    {
-                        "user_prompt": user_prompt,
-                        "system_prompt": system_prompt,
-                        "estimated_input_tokens": prompt_cost["input_tokens"],
-                        "estimated_output_tokens": prompt_cost["output_tokens"],
-                        "estimated_cost": prompt_cost["cost"],
-                        "inference_service": inference_service,
-                        "model": model,
-                    }
-                )
-
-        df = pd.DataFrame.from_records(data)
-
-        df = (
-            df.groupby(["inference_service", "model"])
-            .agg(
-                {
-                    "estimated_cost": "sum",
-                    "estimated_input_tokens": "sum",
-                    "estimated_output_tokens": "sum",
-                }
-            )
-            .reset_index()
-        )
-
-        estimated_costs_by_model = df.to_dict("records")
-
-        estimated_total_cost = sum(
-            model["estimated_cost"] for model in estimated_costs_by_model
-        )
-        estimated_total_input_tokens = sum(
-            model["estimated_input_tokens"] for model in estimated_costs_by_model
-        )
-        estimated_total_output_tokens = sum(
-            model["estimated_output_tokens"] for model in estimated_costs_by_model
-        )
-
-        output = {
-            "estimated_total_cost": estimated_total_cost,
-            "estimated_total_input_tokens": estimated_total_input_tokens,
-            "estimated_total_output_tokens": estimated_total_output_tokens,
-            "model_costs": estimated_costs_by_model,
-        }
-
-        return output
-
-    def estimate_job_cost(self) -> dict:
-        """
-        Estimates the cost of a job according to the following assumptions:
-
-        - 1 token = 4 characters.
-        - Input tokens = output tokens.
-
-        Fetches prices from Coop.
-        """
-        from edsl import Coop
-
-        c = Coop()
-        price_lookup = c.fetch_prices()
-
-        return self.estimate_job_cost_from_external_prices(price_lookup=price_lookup)
-
-    @staticmethod
-    def compute_job_cost(job_results: "Results") -> float:
-        """
-        Computes the cost of a completed job in USD.
-        """
-        total_cost = 0
-        for result in job_results:
-            for key in result.raw_model_response:
-                if key.endswith("_cost"):
-                    result_cost = result.raw_model_response[key]
-
-                    question_name = key.removesuffix("_cost")
-                    cache_used = result.cache_used_dict[question_name]
-
-                    if isinstance(result_cost, (int, float)) and not cache_used:
-                        total_cost += result_cost
-
-        return total_cost
-
-    @staticmethod
-    def _get_container_class(object):
-        from edsl.agents.AgentList import AgentList
-        from edsl.agents.Agent import Agent
-        from edsl.scenarios.Scenario import Scenario
-        from edsl.scenarios.ScenarioList import ScenarioList
-        from edsl.language_models.ModelList import ModelList
-
-        if isinstance(object, Agent):
-            return AgentList
-        elif isinstance(object, Scenario):
-            return ScenarioList
-        elif isinstance(object, ModelList):
-            return ModelList
-        else:
-            return list
-
-    @staticmethod
-    def _turn_args_to_list(args):
-        """Return a list of the first argument if it is a sequence, otherwise returns a list of all the arguments.
-
-        Example:
-
-        >>> Jobs._turn_args_to_list([1,2,3])
-        [1, 2, 3]
-
-        """
-
-        def did_user_pass_a_sequence(args):
-            """Return True if the user passed a sequence, False otherwise.
-
-            Example:
-
-            >>> did_user_pass_a_sequence([1,2,3])
-            True
-
-            >>> did_user_pass_a_sequence(1)
-            False
-            """
-            return len(args) == 1 and isinstance(args[0], Sequence)
-
-        if did_user_pass_a_sequence(args):
-            container_class = Jobs._get_container_class(args[0][0])
-            return container_class(args[0])
-        else:
-            container_class = Jobs._get_container_class(args[0])
-            return container_class(args)
-
-    def _get_current_objects_of_this_type(
-        self, object: Union["Agent", "Scenario", "LanguageModel"]
-    ) -> tuple[list, str]:
-        from edsl.agents.Agent import Agent
-        from edsl.scenarios.Scenario import Scenario
-        from edsl.language_models.LanguageModel import LanguageModel
-
-        """Return the current objects of the same type as the first argument.
-
-        >>> from edsl.jobs import Jobs
-        >>> j = Jobs.example()
-        >>> j._get_current_objects_of_this_type(j.agents[0])
-        (AgentList([Agent(traits = {'status': 'Joyful'}), Agent(traits = {'status': 'Sad'})]), 'agents')
-        """
-        class_to_key = {
-            Agent: "agents",
-            Scenario: "scenarios",
-            LanguageModel: "models",
-        }
-        for class_type in class_to_key:
-            if isinstance(object, class_type) or issubclass(
-                object.__class__, class_type
-            ):
-                key = class_to_key[class_type]
-                break
-        else:
-            raise ValueError(
-                f"First argument must be an Agent, Scenario, or LanguageModel, not {object}"
-            )
-        current_objects = getattr(self, key, None)
-        return current_objects, key
-
-    @staticmethod
-    def _get_empty_container_object(object):
-        from edsl import AgentList
-        from edsl import Agent
-        from edsl import Scenario
-        from edsl import ScenarioList
-
-        if isinstance(object, Agent):
-            return AgentList([])
-        elif isinstance(object, Scenario):
-            return ScenarioList([])
-        else:
-            return []
-
-    @staticmethod
-    def _merge_objects(passed_objects, current_objects) -> list:
-        """
-        Combine all the existing objects with the new objects.
-
-        For example, if the user passes in 3 agents,
-        and there are 2 existing agents, this will create 6 new agents
-
-        >>> Jobs(survey = [])._merge_objects([1,2,3], [4,5,6])
-        [5, 6, 7, 6, 7, 8, 7, 8, 9]
-        """
-        new_objects = Jobs._get_empty_container_object(passed_objects[0])
-        for current_object in current_objects:
-            for new_object in passed_objects:
-                new_objects.append(current_object + new_object)
-        return new_objects
-
-    def interviews(self) -> list[Interview]:
-        """
-        Return a list of :class:`edsl.jobs.interviews.Interview` objects.
-
-        It returns one Interview for each combination of Agent, Scenario, and LanguageModel.
-        If any of Agents, Scenarios, or LanguageModels are missing, it fills in with defaults.
-
-        >>> from edsl.jobs import Jobs
-        >>> j = Jobs.example()
-        >>> len(j.interviews())
-        4
-        >>> j.interviews()[0]
-        Interview(agent = Agent(traits = {'status': 'Joyful'}), survey = Survey(...), scenario = Scenario({'period': 'morning'}), model = Model(...))
-        """
-        if hasattr(self, "_interviews"):
-            return self._interviews
-        else:
-            return list(self._create_interviews())
-
-    @classmethod
-    def from_interviews(cls, interview_list):
-        """Return a Jobs instance from a list of interviews.
-
-        This is useful when you have, say, a list of failed interviews and you want to create
-        a new job with only those interviews.
-        """
-        survey = interview_list[0].survey
-        # get all the models
-        models = list(set([interview.model for interview in interview_list]))
-        jobs = cls(survey)
-        jobs.models = models
-        jobs._interviews = interview_list
-        return jobs
-
-    def _create_interviews(self) -> Generator[Interview, None, None]:
-        """
-        Generate interviews.
-
-        Note that this sets the agents, model and scenarios if they have not been set. This is a side effect of the method.
-        This is useful because a user can create a job without setting the agents, models, or scenarios, and the job will still run,
-        with us filling in defaults.
-
-
-        """
-        # if no agents, models, or scenarios are set, set them to defaults
-        from edsl.agents.Agent import Agent
-        from edsl.language_models.registry import Model
-        from edsl.scenarios.Scenario import Scenario
-
-        self.agents = self.agents or [Agent()]
-        self.models = self.models or [Model()]
-        self.scenarios = self.scenarios or [Scenario()]
-        for agent, scenario, model in product(self.agents, self.scenarios, self.models):
-            yield Interview(
-                survey=self.survey,
-                agent=agent,
-                scenario=scenario,
-                model=model,
-                skip_retry=self.skip_retry,
-                raise_validation_errors=self.raise_validation_errors,
-            )
-
-    def create_bucket_collection(self) -> BucketCollection:
-        """
-        Create a collection of buckets for each model.
-
-        These buckets are used to track API calls and token usage.
-
-        >>> from edsl.jobs import Jobs
-        >>> from edsl import Model
-        >>> j = Jobs.example().by(Model(temperature = 1), Model(temperature = 0.5))
-        >>> bc = j.create_bucket_collection()
-        >>> bc
-        BucketCollection(...)
-        """
-        bucket_collection = BucketCollection()
-        for model in self.models:
-            bucket_collection.add_model(model)
-        return bucket_collection
-
-    @property
-    def bucket_collection(self) -> BucketCollection:
-        """Return the bucket collection. If it does not exist, create it."""
-        if self.__bucket_collection is None:
-            self.__bucket_collection = self.create_bucket_collection()
-        return self.__bucket_collection
-
-    def html(self):
-        """Return the HTML representations for each scenario"""
-        links = []
-        for index, scenario in enumerate(self.scenarios):
-            links.append(
-                self.survey.html(
-                    scenario=scenario, return_link=True, cta=f"Scenario {index}"
-                )
-            )
-        return links
-
-    def __hash__(self):
-        """Allow the model to be used as a key in a dictionary.
-
-        >>> from edsl.jobs import Jobs
-        >>> hash(Jobs.example())
-        846655441787442972
-
-        """
-        from edsl.utilities.utilities import dict_hash
-
-        return dict_hash(self._to_dict())
-
-    def _output(self, message) -> None:
-        """Check if a Job is verbose. If so, print the message."""
-        if hasattr(self, "verbose") and self.verbose:
-            print(message)
-
-    def _check_parameters(self, strict=False, warn=False) -> None:
-        """Check if the parameters in the survey and scenarios are consistent.
-
-        >>> from edsl import QuestionFreeText
-        >>> from edsl import Survey
-        >>> from edsl import Scenario
-        >>> q = QuestionFreeText(question_text = "{{poo}}", question_name = "ugly_question")
-        >>> j = Jobs(survey = Survey(questions=[q]))
-        >>> with warnings.catch_warnings(record=True) as w:
-        ...     j._check_parameters(warn = True)
-        ...     assert len(w) == 1
-        ...     assert issubclass(w[-1].category, UserWarning)
-        ...     assert "The following parameters are in the survey but not in the scenarios" in str(w[-1].message)
-
-        >>> q = QuestionFreeText(question_text = "{{poo}}", question_name = "ugly_question")
-        >>> s = Scenario({'plop': "A", 'poo': "B"})
-        >>> j = Jobs(survey = Survey(questions=[q])).by(s)
-        >>> j._check_parameters(strict = True)
-        Traceback (most recent call last):
-        ...
-        ValueError: The following parameters are in the scenarios but not in the survey: {'plop'}
-
-        >>> q = QuestionFreeText(question_text = "Hello", question_name = "ugly_question")
-        >>> s = Scenario({'ugly_question': "B"})
-        >>> j = Jobs(survey = Survey(questions=[q])).by(s)
-        >>> j._check_parameters()
-        Traceback (most recent call last):
-        ...
-        ValueError: The following names are in both the survey question_names and the scenario keys: {'ugly_question'}. This will create issues.
-        """
-        survey_parameters: set = self.survey.parameters
-        scenario_parameters: set = self.scenarios.parameters
-
-        msg0, msg1, msg2 = None, None, None
-
-        # look for key issues
-        if intersection := set(self.scenarios.parameters) & set(
-            self.survey.question_names
-        ):
-            msg0 = f"The following names are in both the survey question_names and the scenario keys: {intersection}. This will create issues."
-
-            raise ValueError(msg0)
-
-        if in_survey_but_not_in_scenarios := survey_parameters - scenario_parameters:
-            msg1 = f"The following parameters are in the survey but not in the scenarios: {in_survey_but_not_in_scenarios}"
-        if in_scenarios_but_not_in_survey := scenario_parameters - survey_parameters:
-            msg2 = f"The following parameters are in the scenarios but not in the survey: {in_scenarios_but_not_in_survey}"
-
-        if msg1 or msg2:
-            message = "\n".join(filter(None, [msg1, msg2]))
-            if strict:
-                raise ValueError(message)
-            else:
-                if warn:
-                    warnings.warn(message)
-
-        if self.scenarios.has_jinja_braces:
-            warnings.warn(
-                "The scenarios have Jinja braces ({{ and }}). Converting to '<<' and '>>'. If you want a different conversion, use the convert_jinja_braces method first to modify the scenario."
-            )
-            self.scenarios = self.scenarios.convert_jinja_braces()
-
-    @property
-    def skip_retry(self):
-        if not hasattr(self, "_skip_retry"):
-            return False
-        return self._skip_retry
-
-    @property
-    def raise_validation_errors(self):
-        if not hasattr(self, "_raise_validation_errors"):
-            return False
-        return self._raise_validation_errors
-
-    def create_remote_inference_job(
-        self, iterations: int = 1, remote_inference_description: Optional[str] = None
-    ):
-        """ """
-        from edsl.coop.coop import Coop
-
-        coop = Coop()
-        self._output("Remote inference activated. Sending job to server...")
-        remote_job_creation_data = coop.remote_inference_create(
-            self,
-            description=remote_inference_description,
-            status="queued",
-            iterations=iterations,
-        )
-        job_uuid = remote_job_creation_data.get("uuid")
-        print(f"Job sent to server. (Job uuid={job_uuid}).")
-        return remote_job_creation_data
-
-    @staticmethod
-    def check_status(job_uuid):
-        from edsl.coop.coop import Coop
-
-        coop = Coop()
-        return coop.remote_inference_get(job_uuid)
-
-    def poll_remote_inference_job(
-        self, remote_job_creation_data: dict
-    ) -> Union[Results, None]:
-        from edsl.coop.coop import Coop
-        import time
-        from datetime import datetime
-        from edsl.config import CONFIG
-
-        expected_parrot_url = CONFIG.get("EXPECTED_PARROT_URL")
-
-        job_uuid = remote_job_creation_data.get("uuid")
-
-        coop = Coop()
-        job_in_queue = True
-        while job_in_queue:
-            remote_job_data = coop.remote_inference_get(job_uuid)
-            status = remote_job_data.get("status")
-            if status == "cancelled":
-                print("\r" + " " * 80 + "\r", end="")
-                print("Job cancelled by the user.")
-                print(
-                    f"See {expected_parrot_url}/home/remote-inference for more details."
-                )
-                return None
-            elif status == "failed":
-                print("\r" + " " * 80 + "\r", end="")
-                print("Job failed.")
-                print(
-                    f"See {expected_parrot_url}/home/remote-inference for more details."
-                )
-                return None
-            elif status == "completed":
-                results_uuid = remote_job_data.get("results_uuid")
-                results = coop.get(results_uuid, expected_object_type="results")
-                print("\r" + " " * 80 + "\r", end="")
-                url = f"{expected_parrot_url}/content/{results_uuid}"
-                print(f"Job completed and Results stored on Coop: {url}.")
-                return results
-            else:
-                duration = 5
-                time_checked = datetime.now().strftime("%Y-%m-%d %I:%M:%S %p")
-                frames = ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"]
-                start_time = time.time()
-                i = 0
-                while time.time() - start_time < duration:
-                    print(
-                        f"\r{frames[i % len(frames)]} Job status: {status} - last update: {time_checked}",
-                        end="",
-                        flush=True,
-                    )
-                    time.sleep(0.1)
-                    i += 1
-
-    def use_remote_inference(self, disable_remote_inference: bool):
-        if disable_remote_inference:
-            return False
-        if not disable_remote_inference:
-            try:
-                from edsl import Coop
-
-                user_edsl_settings = Coop().edsl_settings
-                return user_edsl_settings.get("remote_inference", False)
-            except requests.ConnectionError:
-                pass
-            except CoopServerResponseError as e:
-                pass
-
-        return False
-
-    def use_remote_cache(self):
-        try:
-            from edsl import Coop
-
-            user_edsl_settings = Coop().edsl_settings
-            return user_edsl_settings.get("remote_caching", False)
-        except requests.ConnectionError:
-            pass
-        except CoopServerResponseError as e:
-            pass
-
-        return False
-
-    def check_api_keys(self):
-        from edsl import Model
-
-        for model in self.models + [Model()]:
-            if not model.has_valid_api_key():
-                raise MissingAPIKeyError(
-                    model_name=str(model.model),
-                    inference_service=model._inference_service_,
-                )
-
-    def run(
-        self,
-        n: int = 1,
-        progress_bar: bool = False,
-        stop_on_exception: bool = False,
-        cache: Union[Cache, bool] = None,
-        check_api_keys: bool = False,
-        sidecar_model: Optional[LanguageModel] = None,
-        verbose: bool = False,
-        print_exceptions=True,
-        remote_cache_description: Optional[str] = None,
-        remote_inference_description: Optional[str] = None,
-        skip_retry: bool = False,
-        raise_validation_errors: bool = False,
-        disable_remote_inference: bool = False,
-    ) -> Results:
-        """
-        Runs the Job: conducts Interviews and returns their results.
-
-        :param n: how many times to run each interview
-        :param progress_bar: shows a progress bar
-        :param stop_on_exception: stops the job if an exception is raised
-        :param cache: a cache object to store results
-        :param check_api_keys: check if the API keys are valid
-        :param batch_mode: run the job in batch mode i.e., no expecation of interaction with the user
-        :param verbose: prints messages
-        :param remote_cache_description: specifies a description for this group of entries in the remote cache
-        :param remote_inference_description: specifies a description for the remote inference job
-        """
-        from edsl.coop.coop import Coop
-
-        self._check_parameters()
-        self._skip_retry = skip_retry
-        self._raise_validation_errors = raise_validation_errors
-
-        self.verbose = verbose
-
-        if remote_inference := self.use_remote_inference(disable_remote_inference):
-            remote_job_creation_data = self.create_remote_inference_job(
-                iterations=n, remote_inference_description=remote_inference_description
-            )
-            results = self.poll_remote_inference_job(remote_job_creation_data)
-            if results is None:
-                self._output("Job failed.")
-            return results
-
-        if check_api_keys:
-            self.check_api_keys()
-
-        # handle cache
-        if cache is None or cache is True:
-            from edsl.data.CacheHandler import CacheHandler
-
-            cache = CacheHandler().get_cache()
-        if cache is False:
-            from edsl.data.Cache import Cache
-
-            cache = Cache()
-
-        remote_cache = self.use_remote_cache()
-        with RemoteCacheSync(
-            coop=Coop(),
-            cache=cache,
-            output_func=self._output,
-            remote_cache=remote_cache,
-            remote_cache_description=remote_cache_description,
-        ) as r:
-            results = self._run_local(
-                n=n,
-                progress_bar=progress_bar,
-                cache=cache,
-                stop_on_exception=stop_on_exception,
-                sidecar_model=sidecar_model,
-                print_exceptions=print_exceptions,
-                raise_validation_errors=raise_validation_errors,
-            )
-
-        results.cache = cache.new_entries_cache()
-        return results
-
-    def _run_local(self, *args, **kwargs):
-        """Run the job locally."""
-
-        results = JobsRunnerAsyncio(self).run(*args, **kwargs)
-        return results
-
-    async def run_async(self, cache=None, n=1, **kwargs):
-        """Run asynchronously."""
-        results = await JobsRunnerAsyncio(self).run_async(cache=cache, n=n, **kwargs)
-        return results
-
-    def all_question_parameters(self):
-        """Return all the fields in the questions in the survey.
-        >>> from edsl.jobs import Jobs
-        >>> Jobs.example().all_question_parameters()
-        {'period'}
-        """
-        return set.union(*[question.parameters for question in self.survey.questions])
-
-    #######################
-    # Dunder methods
-    #######################
-    def print(self):
-        from rich import print_json
-        import json
-
-        print_json(json.dumps(self.to_dict()))
-
-    def __repr__(self) -> str:
-        """Return an eval-able string representation of the Jobs instance."""
-        return f"Jobs(survey={repr(self.survey)}, agents={repr(self.agents)}, models={repr(self.models)}, scenarios={repr(self.scenarios)})"
-
-    def _repr_html_(self) -> str:
-        from rich import print_json
-        import json
-
-        print_json(json.dumps(self.to_dict()))
-
-    def __len__(self) -> int:
-        """Return the maximum number of questions that will be asked while running this job.
-        Note that this is the maximum number of questions, not the actual number of questions that will be asked, as some questions may be skipped.
-
-        >>> from edsl.jobs import Jobs
-        >>> len(Jobs.example())
-        8
-        """
-        number_of_questions = (
-            len(self.agents or [1])
-            * len(self.scenarios or [1])
-            * len(self.models or [1])
-            * len(self.survey)
-        )
-        return number_of_questions
-
-    #######################
-    # Serialization methods
-    #######################
-
-    def _to_dict(self):
-        return {
-            "survey": self.survey._to_dict(),
-            "agents": [agent._to_dict() for agent in self.agents],
-            "models": [model._to_dict() for model in self.models],
-            "scenarios": [scenario._to_dict() for scenario in self.scenarios],
-        }
-
-    @add_edsl_version
-    def to_dict(self) -> dict:
-        """Convert the Jobs instance to a dictionary."""
-        return self._to_dict()
-
-    @classmethod
-    @remove_edsl_version
-    def from_dict(cls, data: dict) -> Jobs:
-        """Creates a Jobs instance from a dictionary."""
-        from edsl import Survey
-        from edsl.agents.Agent import Agent
-        from edsl.language_models.LanguageModel import LanguageModel
-        from edsl.scenarios.Scenario import Scenario
-
-        return cls(
-            survey=Survey.from_dict(data["survey"]),
-            agents=[Agent.from_dict(agent) for agent in data["agents"]],
-            models=[LanguageModel.from_dict(model) for model in data["models"]],
-            scenarios=[Scenario.from_dict(scenario) for scenario in data["scenarios"]],
-        )
-
-    def __eq__(self, other: Jobs) -> bool:
-        """Return True if the Jobs instance is equal to another Jobs instance.
-
-        >>> from edsl.jobs import Jobs
-        >>> Jobs.example() == Jobs.example()
-        True
-
-        """
-        return self.to_dict() == other.to_dict()
-
-    #######################
-    # Example methods
-    #######################
-    @classmethod
-    def example(
-        cls,
-        throw_exception_probability: float = 0.0,
-        randomize: bool = False,
-        test_model=False,
-    ) -> Jobs:
-        """Return an example Jobs instance.
-
-        :param throw_exception_probability: the probability that an exception will be thrown when answering a question. This is useful for testing error handling.
-        :param randomize: whether to randomize the job by adding a random string to the period
-        :param test_model: whether to use a test model
-
-        >>> Jobs.example()
-        Jobs(...)
-
-        """
-        import random
-        from uuid import uuid4
-        from edsl.questions import QuestionMultipleChoice
-        from edsl.agents.Agent import Agent
-        from edsl.scenarios.Scenario import Scenario
-
-        addition = "" if not randomize else str(uuid4())
-
-        if test_model:
-            from edsl.language_models import LanguageModel
-
-            m = LanguageModel.example(test_model=True)
-
-        # (status, question, period)
-        agent_answers = {
-            ("Joyful", "how_feeling", "morning"): "OK",
-            ("Joyful", "how_feeling", "afternoon"): "Great",
-            ("Joyful", "how_feeling_yesterday", "morning"): "Great",
-            ("Joyful", "how_feeling_yesterday", "afternoon"): "Good",
-            ("Sad", "how_feeling", "morning"): "Terrible",
-            ("Sad", "how_feeling", "afternoon"): "OK",
-            ("Sad", "how_feeling_yesterday", "morning"): "OK",
-            ("Sad", "how_feeling_yesterday", "afternoon"): "Terrible",
-        }
-
-        def answer_question_directly(self, question, scenario):
-            """Return the answer to a question. This is a method that can be added to an agent."""
-
-            if random.random() < throw_exception_probability:
-                raise Exception("Error!")
-            return agent_answers[
-                (self.traits["status"], question.question_name, scenario["period"])
-            ]
-
-        sad_agent = Agent(traits={"status": "Sad"})
-        joy_agent = Agent(traits={"status": "Joyful"})
-
-        sad_agent.add_direct_question_answering_method(answer_question_directly)
-        joy_agent.add_direct_question_answering_method(answer_question_directly)
-
-        q1 = QuestionMultipleChoice(
-            question_text="How are you this {{ period }}?",
-            question_options=["Good", "Great", "OK", "Terrible"],
-            question_name="how_feeling",
-        )
-        q2 = QuestionMultipleChoice(
-            question_text="How were you feeling yesterday {{ period }}?",
-            question_options=["Good", "Great", "OK", "Terrible"],
-            question_name="how_feeling_yesterday",
-        )
-        from edsl import Survey, ScenarioList
-
-        base_survey = Survey(questions=[q1, q2])
-
-        scenario_list = ScenarioList(
-            [
-                Scenario({"period": f"morning{addition}"}),
-                Scenario({"period": "afternoon"}),
-            ]
-        )
-        if test_model:
-            job = base_survey.by(m).by(scenario_list).by(joy_agent, sad_agent)
-        else:
-            job = base_survey.by(scenario_list).by(joy_agent, sad_agent)
-
-        return job
-
-    def rich_print(self):
-        """Print a rich representation of the Jobs instance."""
-        from rich.table import Table
-
-        table = Table(title="Jobs")
-        table.add_column("Jobs")
-        table.add_row(self.survey.rich_print())
-        return table
-
-    def code(self):
-        """Return the code to create this instance."""
-        raise NotImplementedError
-
-
-def main():
-    """Run the module's doctests."""
-    from edsl.jobs import Jobs
-    from edsl.data.Cache import Cache
-
-    job = Jobs.example()
-    len(job) == 8
-    results = job.run(cache=Cache())
-    len(results) == 8
-    results
-
-
-if __name__ == "__main__":
-    """Run the module's doctests."""
-    import doctest
-
-    doctest.testmod(optionflags=doctest.ELLIPSIS)
+# """The Jobs class is a collection of agents, scenarios and models and one survey."""
+from __future__ import annotations
+import warnings
+import requests
+from itertools import product
+from typing import Optional, Union, Sequence, Generator
+
+from edsl.Base import Base
+from edsl.exceptions import MissingAPIKeyError
+from edsl.jobs.buckets.BucketCollection import BucketCollection
+from edsl.jobs.interviews.Interview import Interview
+from edsl.jobs.runners.JobsRunnerAsyncio import JobsRunnerAsyncio
+from edsl.utilities.decorators import add_edsl_version, remove_edsl_version
+
+from edsl.data.RemoteCacheSync import RemoteCacheSync
+from edsl.exceptions.coop import CoopServerResponseError
+
+
+class Jobs(Base):
+    """
+    A collection of agents, scenarios and models and one survey.
+    The actual running of a job is done by a `JobsRunner`, which is a subclass of `JobsRunner`.
+    The `JobsRunner` is chosen by the user, and is stored in the `jobs_runner_name` attribute.
+    """
+
+    def __init__(
+        self,
+        survey: "Survey",
+        agents: Optional[list["Agent"]] = None,
+        models: Optional[list["LanguageModel"]] = None,
+        scenarios: Optional[list["Scenario"]] = None,
+    ):
+        """Initialize a Jobs instance.
+
+        :param survey: the survey to be used in the job
+        :param agents: a list of agents
+        :param models: a list of models
+        :param scenarios: a list of scenarios
+        """
+        self.survey = survey
+        self.agents: "AgentList" = agents
+        self.scenarios: "ScenarioList" = scenarios
+        self.models = models
+
+        self.__bucket_collection = None
+
+    # these setters and getters are used to ensure that the agents, models, and scenarios are stored as AgentList, ModelList, and ScenarioList objects
+
+    @property
+    def models(self):
+        return self._models
+
+    @models.setter
+    def models(self, value):
+        from edsl import ModelList
+
+        if value:
+            if not isinstance(value, ModelList):
+                self._models = ModelList(value)
+            else:
+                self._models = value
+        else:
+            self._models = ModelList([])
+
+    @property
+    def agents(self):
+        return self._agents
+
+    @agents.setter
+    def agents(self, value):
+        from edsl import AgentList
+
+        if value:
+            if not isinstance(value, AgentList):
+                self._agents = AgentList(value)
+            else:
+                self._agents = value
+        else:
+            self._agents = AgentList([])
+
+    @property
+    def scenarios(self):
+        return self._scenarios
+
+    @scenarios.setter
+    def scenarios(self, value):
+        from edsl import ScenarioList
+
+        if value:
+            if not isinstance(value, ScenarioList):
+                self._scenarios = ScenarioList(value)
+            else:
+                self._scenarios = value
+        else:
+            self._scenarios = ScenarioList([])
+
+    def by(
+        self,
+        *args: Union[
+            "Agent",
+            "Scenario",
+            "LanguageModel",
+            Sequence[Union["Agent", "Scenario", "LanguageModel"]],
+        ],
+    ) -> Jobs:
+        """
+        Add Agents, Scenarios and LanguageModels to a job. If no objects of this type exist in the Jobs instance, it stores the new objects as a list in the corresponding attribute. Otherwise, it combines the new objects with existing objects using the object's `__add__` method.
+
+        This 'by' is intended to create a fluent interface.
+
+        >>> from edsl import Survey
+        >>> from edsl import QuestionFreeText
+        >>> q = QuestionFreeText(question_name="name", question_text="What is your name?")
+        >>> j = Jobs(survey = Survey(questions=[q]))
+        >>> j
+        Jobs(survey=Survey(...), agents=AgentList([]), models=ModelList([]), scenarios=ScenarioList([]))
+        >>> from edsl import Agent; a = Agent(traits = {"status": "Sad"})
+        >>> j.by(a).agents
+        AgentList([Agent(traits = {'status': 'Sad'})])
+
+        :param args: objects or a sequence (list, tuple, ...) of objects of the same type
+
+        Notes:
+        - all objects must implement the 'get_value', 'set_value', and `__add__` methods
+        - agents: traits of new agents are combined with traits of existing agents. New and existing agents should not have overlapping traits, and do not increase the # agents in the instance
+        - scenarios: traits of new scenarios are combined with traits of old existing. New scenarios will overwrite overlapping traits, and do not increase the number of scenarios in the instance
+        - models: new models overwrite old models.
+        """
+        passed_objects = self._turn_args_to_list(
+            args
+        )  # objects can also be passed comma-separated
+
+        current_objects, objects_key = self._get_current_objects_of_this_type(
+            passed_objects[0]
+        )
+
+        if not current_objects:
+            new_objects = passed_objects
+        else:
+            new_objects = self._merge_objects(passed_objects, current_objects)
+
+        setattr(self, objects_key, new_objects)  # update the job
+        return self
+
+    def prompts(self) -> "Dataset":
+        """Return a Dataset of prompts that will be used.
+
+
+        >>> from edsl.jobs import Jobs
+        >>> Jobs.example().prompts()
+        Dataset(...)
+        """
+        from edsl import Coop
+
+        c = Coop()
+        price_lookup = c.fetch_prices()
+
+        interviews = self.interviews()
+        # data = []
+        interview_indices = []
+        question_names = []
+        user_prompts = []
+        system_prompts = []
+        scenario_indices = []
+        agent_indices = []
+        models = []
+        costs = []
+        from edsl.results.Dataset import Dataset
+
+        for interview_index, interview in enumerate(interviews):
+            invigilators = [
+                interview._get_invigilator(question)
+                for question in self.survey.questions
+            ]
+            for _, invigilator in enumerate(invigilators):
+                prompts = invigilator.get_prompts()
+                user_prompt = prompts["user_prompt"]
+                system_prompt = prompts["system_prompt"]
+                user_prompts.append(user_prompt)
+                system_prompts.append(system_prompt)
+                agent_index = self.agents.index(invigilator.agent)
+                agent_indices.append(agent_index)
+                interview_indices.append(interview_index)
+                scenario_index = self.scenarios.index(invigilator.scenario)
+                scenario_indices.append(scenario_index)
+                models.append(invigilator.model.model)
+                question_names.append(invigilator.question.question_name)
+
+                prompt_cost = self.estimate_prompt_cost(
+                    system_prompt=system_prompt,
+                    user_prompt=user_prompt,
+                    price_lookup=price_lookup,
+                    inference_service=invigilator.model._inference_service_,
+                    model=invigilator.model.model,
+                )
+                costs.append(prompt_cost["cost"])
+
+        d = Dataset(
+            [
+                {"user_prompt": user_prompts},
+                {"system_prompt": system_prompts},
+                {"interview_index": interview_indices},
+                {"question_name": question_names},
+                {"scenario_index": scenario_indices},
+                {"agent_index": agent_indices},
+                {"model": models},
+                {"estimated_cost": costs},
+            ]
+        )
+        return d
+
+    def show_prompts(self, all=False) -> None:
+        """Print the prompts."""
+        if all:
+            self.prompts().to_scenario_list().print(format="rich")
+        else:
+            self.prompts().select(
+                "user_prompt", "system_prompt"
+            ).to_scenario_list().print(format="rich")
+
+    @staticmethod
+    def estimate_prompt_cost(
+        system_prompt: str,
+        user_prompt: str,
+        price_lookup: dict,
+        inference_service: str,
+        model: str,
+    ) -> dict:
+        """Estimates the cost of a prompt. Takes piping into account."""
+
+        def get_piping_multiplier(prompt: str):
+            """Returns 2 if a prompt includes Jinja braces, and 1 otherwise."""
+
+            if "{{" in prompt and "}}" in prompt:
+                return 2
+            return 1
+
+        # Look up prices per token
+        key = (inference_service, model)
+
+        try:
+            relevant_prices = price_lookup[key]
+            output_price_per_token = 1 / float(
+                relevant_prices["output"]["one_usd_buys"]
+            )
+            input_price_per_token = 1 / float(relevant_prices["input"]["one_usd_buys"])
+        except KeyError:
+            # A KeyError is likely to occur if we cannot retrieve prices (the price_lookup dict is empty)
+            # Use a sensible default
+
+            import warnings
+
+            warnings.warn(
+                "Price data could not be retrieved. Using default estimates for input and output token prices. Input: $0.15 / 1M tokens; Output: $0.60 / 1M tokens"
+            )
+
+            output_price_per_token = 0.00000015  # $0.15 / 1M tokens
+            input_price_per_token = 0.00000060  # $0.60 / 1M tokens
+
+        # Compute the number of characters (double if the question involves piping)
+        user_prompt_chars = len(str(user_prompt)) * get_piping_multiplier(
+            str(user_prompt)
+        )
+        system_prompt_chars = len(str(system_prompt)) * get_piping_multiplier(
+            str(system_prompt)
+        )
+
+        # Convert into tokens (1 token approx. equals 4 characters)
+        input_tokens = (user_prompt_chars + system_prompt_chars) // 4
+        output_tokens = input_tokens
+
+        cost = (
+            input_tokens * input_price_per_token
+            + output_tokens * output_price_per_token
+        )
+
+        return {
+            "input_tokens": input_tokens,
+            "output_tokens": output_tokens,
+            "cost": cost,
+        }
+
+    def estimate_job_cost_from_external_prices(self, price_lookup: dict) -> dict:
+        """
+        Estimates the cost of a job according to the following assumptions:
+
+        - 1 token = 4 characters.
+        - Input tokens = output tokens.
+
+        price_lookup is an external pricing dictionary.
+        """
+
+        import pandas as pd
+
+        interviews = self.interviews()
+        data = []
+        for interview in interviews:
+            invigilators = [
+                interview._get_invigilator(question)
+                for question in self.survey.questions
+            ]
+            for invigilator in invigilators:
+                prompts = invigilator.get_prompts()
+
+                # By this point, agent and scenario data has already been added to the prompts
+                user_prompt = prompts["user_prompt"]
+                system_prompt = prompts["system_prompt"]
+                inference_service = invigilator.model._inference_service_
+                model = invigilator.model.model
+
+                prompt_cost = self.estimate_prompt_cost(
+                    system_prompt=system_prompt,
+                    user_prompt=user_prompt,
+                    price_lookup=price_lookup,
+                    inference_service=inference_service,
+                    model=model,
+                )
+
+                data.append(
+                    {
+                        "user_prompt": user_prompt,
+                        "system_prompt": system_prompt,
+                        "estimated_input_tokens": prompt_cost["input_tokens"],
+                        "estimated_output_tokens": prompt_cost["output_tokens"],
+                        "estimated_cost": prompt_cost["cost"],
+                        "inference_service": inference_service,
+                        "model": model,
+                    }
+                )
+
+        df = pd.DataFrame.from_records(data)
+
+        df = (
+            df.groupby(["inference_service", "model"])
+            .agg(
+                {
+                    "estimated_cost": "sum",
+                    "estimated_input_tokens": "sum",
+                    "estimated_output_tokens": "sum",
+                }
+            )
+            .reset_index()
+        )
+
+        estimated_costs_by_model = df.to_dict("records")
+
+        estimated_total_cost = sum(
+            model["estimated_cost"] for model in estimated_costs_by_model
+        )
+        estimated_total_input_tokens = sum(
+            model["estimated_input_tokens"] for model in estimated_costs_by_model
+        )
+        estimated_total_output_tokens = sum(
+            model["estimated_output_tokens"] for model in estimated_costs_by_model
+        )
+
+        output = {
+            "estimated_total_cost": estimated_total_cost,
+            "estimated_total_input_tokens": estimated_total_input_tokens,
+            "estimated_total_output_tokens": estimated_total_output_tokens,
+            "model_costs": estimated_costs_by_model,
+        }
+
+        return output
+
+    def estimate_job_cost(self) -> dict:
+        """
+        Estimates the cost of a job according to the following assumptions:
+
+        - 1 token = 4 characters.
+        - Input tokens = output tokens.
+
+        Fetches prices from Coop.
+        """
+        from edsl import Coop
+
+        c = Coop()
+        price_lookup = c.fetch_prices()
+
+        return self.estimate_job_cost_from_external_prices(price_lookup=price_lookup)
+
+    @staticmethod
+    def compute_job_cost(job_results: "Results") -> float:
+        """
+        Computes the cost of a completed job in USD.
+        """
+        total_cost = 0
+        for result in job_results:
+            for key in result.raw_model_response:
+                if key.endswith("_cost"):
+                    result_cost = result.raw_model_response[key]
+
+                    question_name = key.removesuffix("_cost")
+                    cache_used = result.cache_used_dict[question_name]
+
+                    if isinstance(result_cost, (int, float)) and not cache_used:
+                        total_cost += result_cost
+
+        return total_cost
+
+    @staticmethod
+    def _get_container_class(object):
+        from edsl.agents.AgentList import AgentList
+        from edsl.agents.Agent import Agent
+        from edsl.scenarios.Scenario import Scenario
+        from edsl.scenarios.ScenarioList import ScenarioList
+        from edsl.language_models.ModelList import ModelList
+
+        if isinstance(object, Agent):
+            return AgentList
+        elif isinstance(object, Scenario):
+            return ScenarioList
+        elif isinstance(object, ModelList):
+            return ModelList
+        else:
+            return list
+
+    @staticmethod
+    def _turn_args_to_list(args):
+        """Return a list of the first argument if it is a sequence, otherwise returns a list of all the arguments.
+
+        Example:
+
+        >>> Jobs._turn_args_to_list([1,2,3])
+        [1, 2, 3]
+
+        """
+
+        def did_user_pass_a_sequence(args):
+            """Return True if the user passed a sequence, False otherwise.
+
+            Example:
+
+            >>> did_user_pass_a_sequence([1,2,3])
+            True
+
+            >>> did_user_pass_a_sequence(1)
+            False
+            """
+            return len(args) == 1 and isinstance(args[0], Sequence)
+
+        if did_user_pass_a_sequence(args):
+            container_class = Jobs._get_container_class(args[0][0])
+            return container_class(args[0])
+        else:
+            container_class = Jobs._get_container_class(args[0])
+            return container_class(args)
+
+    def _get_current_objects_of_this_type(
+        self, object: Union["Agent", "Scenario", "LanguageModel"]
+    ) -> tuple[list, str]:
+        from edsl.agents.Agent import Agent
+        from edsl.scenarios.Scenario import Scenario
+        from edsl.language_models.LanguageModel import LanguageModel
+
+        """Return the current objects of the same type as the first argument.
+
+        >>> from edsl.jobs import Jobs
+        >>> j = Jobs.example()
+        >>> j._get_current_objects_of_this_type(j.agents[0])
+        (AgentList([Agent(traits = {'status': 'Joyful'}), Agent(traits = {'status': 'Sad'})]), 'agents')
+        """
+        class_to_key = {
+            Agent: "agents",
+            Scenario: "scenarios",
+            LanguageModel: "models",
+        }
+        for class_type in class_to_key:
+            if isinstance(object, class_type) or issubclass(
+                object.__class__, class_type
+            ):
+                key = class_to_key[class_type]
+                break
+        else:
+            raise ValueError(
+                f"First argument must be an Agent, Scenario, or LanguageModel, not {object}"
+            )
+        current_objects = getattr(self, key, None)
+        return current_objects, key
+
+    @staticmethod
+    def _get_empty_container_object(object):
+        from edsl import AgentList
+        from edsl import Agent
+        from edsl import Scenario
+        from edsl import ScenarioList
+
+        if isinstance(object, Agent):
+            return AgentList([])
+        elif isinstance(object, Scenario):
+            return ScenarioList([])
+        else:
+            return []
+
+    @staticmethod
+    def _merge_objects(passed_objects, current_objects) -> list:
+        """
+        Combine all the existing objects with the new objects.
+
+        For example, if the user passes in 3 agents,
+        and there are 2 existing agents, this will create 6 new agents
+
+        >>> Jobs(survey = [])._merge_objects([1,2,3], [4,5,6])
+        [5, 6, 7, 6, 7, 8, 7, 8, 9]
+        """
+        new_objects = Jobs._get_empty_container_object(passed_objects[0])
+        for current_object in current_objects:
+            for new_object in passed_objects:
+                new_objects.append(current_object + new_object)
+        return new_objects
+
+    def interviews(self) -> list[Interview]:
+        """
+        Return a list of :class:`edsl.jobs.interviews.Interview` objects.
+
+        It returns one Interview for each combination of Agent, Scenario, and LanguageModel.
+        If any of Agents, Scenarios, or LanguageModels are missing, it fills in with defaults.
+
+        >>> from edsl.jobs import Jobs
+        >>> j = Jobs.example()
+        >>> len(j.interviews())
+        4
+        >>> j.interviews()[0]
+        Interview(agent = Agent(traits = {'status': 'Joyful'}), survey = Survey(...), scenario = Scenario({'period': 'morning'}), model = Model(...))
+        """
+        if hasattr(self, "_interviews"):
+            return self._interviews
+        else:
+            return list(self._create_interviews())
+
+    @classmethod
+    def from_interviews(cls, interview_list):
+        """Return a Jobs instance from a list of interviews.
+
+        This is useful when you have, say, a list of failed interviews and you want to create
+        a new job with only those interviews.
+        """
+        survey = interview_list[0].survey
+        # get all the models
+        models = list(set([interview.model for interview in interview_list]))
+        jobs = cls(survey)
+        jobs.models = models
+        jobs._interviews = interview_list
+        return jobs
+
+    def _create_interviews(self) -> Generator[Interview, None, None]:
+        """
+        Generate interviews.
+
+        Note that this sets the agents, model and scenarios if they have not been set. This is a side effect of the method.
+        This is useful because a user can create a job without setting the agents, models, or scenarios, and the job will still run,
+        with us filling in defaults.
+
+
+        """
+        # if no agents, models, or scenarios are set, set them to defaults
+        from edsl.agents.Agent import Agent
+        from edsl.language_models.registry import Model
+        from edsl.scenarios.Scenario import Scenario
+
+        self.agents = self.agents or [Agent()]
+        self.models = self.models or [Model()]
+        self.scenarios = self.scenarios or [Scenario()]
+        for agent, scenario, model in product(self.agents, self.scenarios, self.models):
+            yield Interview(
+                survey=self.survey,
+                agent=agent,
+                scenario=scenario,
+                model=model,
+                skip_retry=self.skip_retry,
+                raise_validation_errors=self.raise_validation_errors,
+            )
+
+    def create_bucket_collection(self) -> BucketCollection:
+        """
+        Create a collection of buckets for each model.
+
+        These buckets are used to track API calls and token usage.
+
+        >>> from edsl.jobs import Jobs
+        >>> from edsl import Model
+        >>> j = Jobs.example().by(Model(temperature = 1), Model(temperature = 0.5))
+        >>> bc = j.create_bucket_collection()
+        >>> bc
+        BucketCollection(...)
+        """
+        bucket_collection = BucketCollection()
+        for model in self.models:
+            bucket_collection.add_model(model)
+        return bucket_collection
+
+    @property
+    def bucket_collection(self) -> BucketCollection:
+        """Return the bucket collection. If it does not exist, create it."""
+        if self.__bucket_collection is None:
+            self.__bucket_collection = self.create_bucket_collection()
+        return self.__bucket_collection
+
+    def html(self):
+        """Return the HTML representations for each scenario"""
+        links = []
+        for index, scenario in enumerate(self.scenarios):
+            links.append(
+                self.survey.html(
+                    scenario=scenario, return_link=True, cta=f"Scenario {index}"
+                )
+            )
+        return links
+
+    def __hash__(self):
+        """Allow the model to be used as a key in a dictionary.
+
+        >>> from edsl.jobs import Jobs
+        >>> hash(Jobs.example())
+        846655441787442972
+
+        """
+        from edsl.utilities.utilities import dict_hash
+
+        return dict_hash(self._to_dict())
+
+    def _output(self, message) -> None:
+        """Check if a Job is verbose. If so, print the message."""
+        if hasattr(self, "verbose") and self.verbose:
+            print(message)
+
+    def _check_parameters(self, strict=False, warn=False) -> None:
+        """Check if the parameters in the survey and scenarios are consistent.
+
+        >>> from edsl import QuestionFreeText
+        >>> from edsl import Survey
+        >>> from edsl import Scenario
+        >>> q = QuestionFreeText(question_text = "{{poo}}", question_name = "ugly_question")
+        >>> j = Jobs(survey = Survey(questions=[q]))
+        >>> with warnings.catch_warnings(record=True) as w:
+        ...     j._check_parameters(warn = True)
+        ...     assert len(w) == 1
+        ...     assert issubclass(w[-1].category, UserWarning)
+        ...     assert "The following parameters are in the survey but not in the scenarios" in str(w[-1].message)
+
+        >>> q = QuestionFreeText(question_text = "{{poo}}", question_name = "ugly_question")
+        >>> s = Scenario({'plop': "A", 'poo': "B"})
+        >>> j = Jobs(survey = Survey(questions=[q])).by(s)
+        >>> j._check_parameters(strict = True)
+        Traceback (most recent call last):
+        ...
+        ValueError: The following parameters are in the scenarios but not in the survey: {'plop'}
+
+        >>> q = QuestionFreeText(question_text = "Hello", question_name = "ugly_question")
+        >>> s = Scenario({'ugly_question': "B"})
+        >>> j = Jobs(survey = Survey(questions=[q])).by(s)
+        >>> j._check_parameters()
+        Traceback (most recent call last):
+        ...
+        ValueError: The following names are in both the survey question_names and the scenario keys: {'ugly_question'}. This will create issues.
+        """
+        survey_parameters: set = self.survey.parameters
+        scenario_parameters: set = self.scenarios.parameters
+
+        msg0, msg1, msg2 = None, None, None
+
+        # look for key issues
+        if intersection := set(self.scenarios.parameters) & set(
+            self.survey.question_names
+        ):
+            msg0 = f"The following names are in both the survey question_names and the scenario keys: {intersection}. This will create issues."
+
+            raise ValueError(msg0)
+
+        if in_survey_but_not_in_scenarios := survey_parameters - scenario_parameters:
+            msg1 = f"The following parameters are in the survey but not in the scenarios: {in_survey_but_not_in_scenarios}"
+        if in_scenarios_but_not_in_survey := scenario_parameters - survey_parameters:
+            msg2 = f"The following parameters are in the scenarios but not in the survey: {in_scenarios_but_not_in_survey}"
+
+        if msg1 or msg2:
+            message = "\n".join(filter(None, [msg1, msg2]))
+            if strict:
+                raise ValueError(message)
+            else:
+                if warn:
+                    warnings.warn(message)
+
+        if self.scenarios.has_jinja_braces:
+            warnings.warn(
+                "The scenarios have Jinja braces ({{ and }}). Converting to '<<' and '>>'. If you want a different conversion, use the convert_jinja_braces method first to modify the scenario."
+            )
+            self.scenarios = self.scenarios.convert_jinja_braces()
+
+    @property
+    def skip_retry(self):
+        if not hasattr(self, "_skip_retry"):
+            return False
+        return self._skip_retry
+
+    @property
+    def raise_validation_errors(self):
+        if not hasattr(self, "_raise_validation_errors"):
+            return False
+        return self._raise_validation_errors
+
+    def create_remote_inference_job(
+        self, iterations: int = 1, remote_inference_description: Optional[str] = None
+    ):
+        """ """
+        from edsl.coop.coop import Coop
+
+        coop = Coop()
+        self._output("Remote inference activated. Sending job to server...")
+        remote_job_creation_data = coop.remote_inference_create(
+            self,
+            description=remote_inference_description,
+            status="queued",
+            iterations=iterations,
+        )
+        job_uuid = remote_job_creation_data.get("uuid")
+        print(f"Job sent to server. (Job uuid={job_uuid}).")
+        return remote_job_creation_data
+
+    @staticmethod
+    def check_status(job_uuid):
+        from edsl.coop.coop import Coop
+
+        coop = Coop()
+        return coop.remote_inference_get(job_uuid)
+
+    def poll_remote_inference_job(
+        self, remote_job_creation_data: dict
+    ) -> Union[Results, None]:
+        from edsl.coop.coop import Coop
+        import time
+        from datetime import datetime
+        from edsl.config import CONFIG
+
+        expected_parrot_url = CONFIG.get("EXPECTED_PARROT_URL")
+
+        job_uuid = remote_job_creation_data.get("uuid")
+
+        coop = Coop()
+        job_in_queue = True
+        while job_in_queue:
+            remote_job_data = coop.remote_inference_get(job_uuid)
+            status = remote_job_data.get("status")
+            if status == "cancelled":
+                print("\r" + " " * 80 + "\r", end="")
+                print("Job cancelled by the user.")
+                print(
+                    f"See {expected_parrot_url}/home/remote-inference for more details."
+                )
+                return None
+            elif status == "failed":
+                print("\r" + " " * 80 + "\r", end="")
+                print("Job failed.")
+                print(
+                    f"See {expected_parrot_url}/home/remote-inference for more details."
+                )
+                return None
+            elif status == "completed":
+                results_uuid = remote_job_data.get("results_uuid")
+                results = coop.get(results_uuid, expected_object_type="results")
+                print("\r" + " " * 80 + "\r", end="")
+                url = f"{expected_parrot_url}/content/{results_uuid}"
+                print(f"Job completed and Results stored on Coop: {url}.")
+                return results
+            else:
+                duration = 5
+                time_checked = datetime.now().strftime("%Y-%m-%d %I:%M:%S %p")
+                frames = ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"]
+                start_time = time.time()
+                i = 0
+                while time.time() - start_time < duration:
+                    print(
+                        f"\r{frames[i % len(frames)]} Job status: {status} - last update: {time_checked}",
+                        end="",
+                        flush=True,
+                    )
+                    time.sleep(0.1)
+                    i += 1
+
+    def use_remote_inference(self, disable_remote_inference: bool):
+        if disable_remote_inference:
+            return False
+        if not disable_remote_inference:
+            try:
+                from edsl import Coop
+
+                user_edsl_settings = Coop().edsl_settings
+                return user_edsl_settings.get("remote_inference", False)
+            except requests.ConnectionError:
+                pass
+            except CoopServerResponseError as e:
+                pass
+
+        return False
+
+    def use_remote_cache(self):
+        try:
+            from edsl import Coop
+
+            user_edsl_settings = Coop().edsl_settings
+            return user_edsl_settings.get("remote_caching", False)
+        except requests.ConnectionError:
+            pass
+        except CoopServerResponseError as e:
+            pass
+
+        return False
+
+    def check_api_keys(self):
+        from edsl import Model
+
+        for model in self.models + [Model()]:
+            if not model.has_valid_api_key():
+                raise MissingAPIKeyError(
+                    model_name=str(model.model),
+                    inference_service=model._inference_service_,
+                )
+
+    def run(
+        self,
+        n: int = 1,
+        progress_bar: bool = False,
+        stop_on_exception: bool = False,
+        cache: Union[Cache, bool] = None,
+        check_api_keys: bool = False,
+        sidecar_model: Optional[LanguageModel] = None,
+        verbose: bool = False,
+        print_exceptions=True,
+        remote_cache_description: Optional[str] = None,
+        remote_inference_description: Optional[str] = None,
+        skip_retry: bool = False,
+        raise_validation_errors: bool = False,
+        disable_remote_inference: bool = False,
+    ) -> Results:
+        """
+        Runs the Job: conducts Interviews and returns their results.
+
+        :param n: how many times to run each interview
+        :param progress_bar: shows a progress bar
+        :param stop_on_exception: stops the job if an exception is raised
+        :param cache: a cache object to store results
+        :param check_api_keys: check if the API keys are valid
+        :param batch_mode: run the job in batch mode i.e., no expecation of interaction with the user
+        :param verbose: prints messages
+        :param remote_cache_description: specifies a description for this group of entries in the remote cache
+        :param remote_inference_description: specifies a description for the remote inference job
+        """
+        from edsl.coop.coop import Coop
+
+        self._check_parameters()
+        self._skip_retry = skip_retry
+        self._raise_validation_errors = raise_validation_errors
+
+        self.verbose = verbose
+
+        if remote_inference := self.use_remote_inference(disable_remote_inference):
+            remote_job_creation_data = self.create_remote_inference_job(
+                iterations=n, remote_inference_description=remote_inference_description
+            )
+            results = self.poll_remote_inference_job(remote_job_creation_data)
+            if results is None:
+                self._output("Job failed.")
+            return results
+
+        if check_api_keys:
+            self.check_api_keys()
+
+        # handle cache
+        if cache is None or cache is True:
+            from edsl.data.CacheHandler import CacheHandler
+
+            cache = CacheHandler().get_cache()
+        if cache is False:
+            from edsl.data.Cache import Cache
+
+            cache = Cache()
+
+        remote_cache = self.use_remote_cache()
+        with RemoteCacheSync(
+            coop=Coop(),
+            cache=cache,
+            output_func=self._output,
+            remote_cache=remote_cache,
+            remote_cache_description=remote_cache_description,
+        ) as r:
+            results = self._run_local(
+                n=n,
+                progress_bar=progress_bar,
+                cache=cache,
+                stop_on_exception=stop_on_exception,
+                sidecar_model=sidecar_model,
+                print_exceptions=print_exceptions,
+                raise_validation_errors=raise_validation_errors,
+            )
+
+        results.cache = cache.new_entries_cache()
+        return results
+
+    def _run_local(self, *args, **kwargs):
+        """Run the job locally."""
+
+        results = JobsRunnerAsyncio(self).run(*args, **kwargs)
+        return results
+
+    async def run_async(self, cache=None, n=1, **kwargs):
+        """Run asynchronously."""
+        results = await JobsRunnerAsyncio(self).run_async(cache=cache, n=n, **kwargs)
+        return results
+
+    def all_question_parameters(self):
+        """Return all the fields in the questions in the survey.
+        >>> from edsl.jobs import Jobs
+        >>> Jobs.example().all_question_parameters()
+        {'period'}
+        """
+        return set.union(*[question.parameters for question in self.survey.questions])
+
+    #######################
+    # Dunder methods
+    #######################
+    def print(self):
+        from rich import print_json
+        import json
+
+        print_json(json.dumps(self.to_dict()))
+
+    def __repr__(self) -> str:
+        """Return an eval-able string representation of the Jobs instance."""
+        return f"Jobs(survey={repr(self.survey)}, agents={repr(self.agents)}, models={repr(self.models)}, scenarios={repr(self.scenarios)})"
+
+    def _repr_html_(self) -> str:
+        from rich import print_json
+        import json
+
+        print_json(json.dumps(self.to_dict()))
+
+    def __len__(self) -> int:
+        """Return the maximum number of questions that will be asked while running this job.
+        Note that this is the maximum number of questions, not the actual number of questions that will be asked, as some questions may be skipped.
+
+        >>> from edsl.jobs import Jobs
+        >>> len(Jobs.example())
+        8
+        """
+        number_of_questions = (
+            len(self.agents or [1])
+            * len(self.scenarios or [1])
+            * len(self.models or [1])
+            * len(self.survey)
+        )
+        return number_of_questions
+
+    #######################
+    # Serialization methods
+    #######################
+
+    def _to_dict(self):
+        return {
+            "survey": self.survey._to_dict(),
+            "agents": [agent._to_dict() for agent in self.agents],
+            "models": [model._to_dict() for model in self.models],
+            "scenarios": [scenario._to_dict() for scenario in self.scenarios],
+        }
+
+    @add_edsl_version
+    def to_dict(self) -> dict:
+        """Convert the Jobs instance to a dictionary."""
+        return self._to_dict()
+
+    @classmethod
+    @remove_edsl_version
+    def from_dict(cls, data: dict) -> Jobs:
+        """Creates a Jobs instance from a dictionary."""
+        from edsl import Survey
+        from edsl.agents.Agent import Agent
+        from edsl.language_models.LanguageModel import LanguageModel
+        from edsl.scenarios.Scenario import Scenario
+
+        return cls(
+            survey=Survey.from_dict(data["survey"]),
+            agents=[Agent.from_dict(agent) for agent in data["agents"]],
+            models=[LanguageModel.from_dict(model) for model in data["models"]],
+            scenarios=[Scenario.from_dict(scenario) for scenario in data["scenarios"]],
+        )
+
+    def __eq__(self, other: Jobs) -> bool:
+        """Return True if the Jobs instance is equal to another Jobs instance.
+
+        >>> from edsl.jobs import Jobs
+        >>> Jobs.example() == Jobs.example()
+        True
+
+        """
+        return self.to_dict() == other.to_dict()
+
+    #######################
+    # Example methods
+    #######################
+    @classmethod
+    def example(
+        cls,
+        throw_exception_probability: float = 0.0,
+        randomize: bool = False,
+        test_model=False,
+    ) -> Jobs:
+        """Return an example Jobs instance.
+
+        :param throw_exception_probability: the probability that an exception will be thrown when answering a question. This is useful for testing error handling.
+        :param randomize: whether to randomize the job by adding a random string to the period
+        :param test_model: whether to use a test model
+
+        >>> Jobs.example()
+        Jobs(...)
+
+        """
+        import random
+        from uuid import uuid4
+        from edsl.questions import QuestionMultipleChoice
+        from edsl.agents.Agent import Agent
+        from edsl.scenarios.Scenario import Scenario
+
+        addition = "" if not randomize else str(uuid4())
+
+        if test_model:
+            from edsl.language_models import LanguageModel
+
+            m = LanguageModel.example(test_model=True)
+
+        # (status, question, period)
+        agent_answers = {
+            ("Joyful", "how_feeling", "morning"): "OK",
+            ("Joyful", "how_feeling", "afternoon"): "Great",
+            ("Joyful", "how_feeling_yesterday", "morning"): "Great",
+            ("Joyful", "how_feeling_yesterday", "afternoon"): "Good",
+            ("Sad", "how_feeling", "morning"): "Terrible",
+            ("Sad", "how_feeling", "afternoon"): "OK",
+            ("Sad", "how_feeling_yesterday", "morning"): "OK",
+            ("Sad", "how_feeling_yesterday", "afternoon"): "Terrible",
+        }
+
+        def answer_question_directly(self, question, scenario):
+            """Return the answer to a question. This is a method that can be added to an agent."""
+
+            if random.random() < throw_exception_probability:
+                raise Exception("Error!")
+            return agent_answers[
+                (self.traits["status"], question.question_name, scenario["period"])
+            ]
+
+        sad_agent = Agent(traits={"status": "Sad"})
+        joy_agent = Agent(traits={"status": "Joyful"})
+
+        sad_agent.add_direct_question_answering_method(answer_question_directly)
+        joy_agent.add_direct_question_answering_method(answer_question_directly)
+
+        q1 = QuestionMultipleChoice(
+            question_text="How are you this {{ period }}?",
+            question_options=["Good", "Great", "OK", "Terrible"],
+            question_name="how_feeling",
+        )
+        q2 = QuestionMultipleChoice(
+            question_text="How were you feeling yesterday {{ period }}?",
+            question_options=["Good", "Great", "OK", "Terrible"],
+            question_name="how_feeling_yesterday",
+        )
+        from edsl import Survey, ScenarioList
+
+        base_survey = Survey(questions=[q1, q2])
+
+        scenario_list = ScenarioList(
+            [
+                Scenario({"period": f"morning{addition}"}),
+                Scenario({"period": "afternoon"}),
+            ]
+        )
+        if test_model:
+            job = base_survey.by(m).by(scenario_list).by(joy_agent, sad_agent)
+        else:
+            job = base_survey.by(scenario_list).by(joy_agent, sad_agent)
+
+        return job
+
+    def rich_print(self):
+        """Print a rich representation of the Jobs instance."""
+        from rich.table import Table
+
+        table = Table(title="Jobs")
+        table.add_column("Jobs")
+        table.add_row(self.survey.rich_print())
+        return table
+
+    def code(self):
+        """Return the code to create this instance."""
+        raise NotImplementedError
+
+
+def main():
+    """Run the module's doctests."""
+    from edsl.jobs import Jobs
+    from edsl.data.Cache import Cache
+
+    job = Jobs.example()
+    len(job) == 8
+    results = job.run(cache=Cache())
+    len(results) == 8
+    results
+
+
+if __name__ == "__main__":
+    """Run the module's doctests."""
+    import doctest
+
+    doctest.testmod(optionflags=doctest.ELLIPSIS)