edsl 0.1.14__py3-none-any.whl → 0.1.40__py3-none-any.whl

edsl/jobs/JobsChecks.py

@@ -0,0 +1,172 @@
+import os
+from edsl.exceptions.general import MissingAPIKeyError
+
+
+class JobsChecks:
+    def __init__(self, jobs):
+        """Checks a Jobs object for missing API keys and other requirements."""
+        self.jobs = jobs
+
+    def check_api_keys(self) -> None:
+        from edsl.language_models.model import Model
+
+        if len(self.jobs.models) == 0:
+            models = [Model()]
+        else:
+            models = self.jobs.models
+
+        for model in models:  # + [Model()]:
+            if not model.has_valid_api_key():
+                raise MissingAPIKeyError(
+                    model_name=str(model.model),
+                    inference_service=model._inference_service_,
+                )
+
+    def get_missing_api_keys(self) -> set:
+        """
+        Returns a list of the api keys that a user needs to run this job, but does not currently have in their .env file.
+        """
+        missing_api_keys = set()
+
+        from edsl.language_models.model import Model
+        from edsl.enums import service_to_api_keyname
+
+        for model in self.jobs.models + [Model()]:
+            if not model.has_valid_api_key():
+                key_name = service_to_api_keyname.get(
+                    model._inference_service_, "NOT FOUND"
+                )
+                missing_api_keys.add(key_name)
+
+        return missing_api_keys
+
+    def user_has_ep_api_key(self) -> bool:
+        """
+        Returns True if the user has an EXPECTED_PARROT_API_KEY in their env.
+
+        Otherwise, returns False.
+        """
+
+        coop_api_key = os.getenv("EXPECTED_PARROT_API_KEY")
+
+        if coop_api_key is not None:
+            return True
+        else:
+            return False
+
+    def user_has_all_model_keys(self):
+        """
+        Returns True if the user has all model keys required to run their job.
+
+        Otherwise, returns False.
+        """
+
+        try:
+            self.check_api_keys()
+            return True
+        except MissingAPIKeyError:
+            return False
+        except Exception:
+            raise
+
+    def needs_external_llms(self) -> bool:
+        """
+        Returns True if the job needs external LLMs to run.
+
+        Otherwise, returns False.
+        """
+        # These cases are necessary to skip the API key check during doctests
+
+        # Accounts for Results.example()
+        all_agents_answer_questions_directly = len(self.jobs.agents) > 0 and all(
+            [hasattr(a, "answer_question_directly") for a in self.jobs.agents]
+        )
+
+        # Accounts for InterviewExceptionEntry.example()
+        only_model_is_test = set([m.model for m in self.jobs.models]) == set(["test"])
+
+        # Accounts for Survey.__call__
+        all_questions_are_functional = set(
+            [q.question_type for q in self.jobs.survey.questions]
+        ) == set(["functional"])
+
+        if (
+            all_agents_answer_questions_directly
+            or only_model_is_test
+            or all_questions_are_functional
+        ):
+            return False
+        else:
+            return True
+
+    def needs_key_process(self):
+        """
+        A User needs the key process when:
+        1. They don't have all the model keys
+        2. They don't have the EP API
+        3. They need external LLMs to run the job
+        """
+        return (
+            not self.user_has_all_model_keys()
+            and not self.user_has_ep_api_key()
+            and self.needs_external_llms()
+        )
+
+    def status(self) -> dict:
+        """
+        Returns a dictionary with the status of the job checks.
+        """
+        return {
+            "user_has_ep_api_key": self.user_has_ep_api_key(),
+            "user_has_all_model_keys": self.user_has_all_model_keys(),
+            "needs_external_llms": self.needs_external_llms(),
+            "needs_key_process": self.needs_key_process(),
+        }
+
+    def key_process(self):
+        import secrets
+        from dotenv import load_dotenv
+        from edsl.config import CONFIG
+        from edsl.coop.coop import Coop
+        from edsl.utilities.utilities import write_api_key_to_env
+
+        missing_api_keys = self.get_missing_api_keys()
+
+        edsl_auth_token = secrets.token_urlsafe(16)
+
+        print("You're missing some of the API keys needed to run this job:")
+        for api_key in missing_api_keys:
+            print(f"     🔑 {api_key}")
+        print(
+            "\nYou can either add the missing keys to your .env file, or use remote inference."
+        )
+        print("Remote inference allows you to run jobs on our server.")
+
+        coop = Coop()
+        coop._display_login_url(
+            edsl_auth_token=edsl_auth_token,
+            link_description="\n🚀 To use remote inference, sign up at the following link:",
+        )
+
+        print(
+            "\nOnce you log in, we will automatically retrieve your Expected Parrot API key and continue your job remotely."
+        )
+
+        api_key = coop._poll_for_api_key(edsl_auth_token)
+
+        if api_key is None:
+            print("\nTimed out waiting for login. Please try again.")
+            return
+
+        path_to_env = write_api_key_to_env(api_key)
+        print("\n✨ API key retrieved and written to .env file at the following path:")
+        print(f"    {path_to_env}")
+
+        # Retrieve API key so we can continue running the job
+        load_dotenv()
+
+
+if __name__ == "__main__":
+    import doctest
+
+    doctest.testmod()

edsl/jobs/JobsComponentConstructor.py

@@ -0,0 +1,189 @@
+from typing import Union, Sequence, TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from edsl.agents.Agent import Agent
+    from edsl.language_models.LanguageModel import LanguageModel
+    from edsl.scenarios.Scenario import Scenario
+    from edsl.jobs.Jobs import Jobs
+
+
+class JobsComponentConstructor:
+    "Handles the creation of Agents, Scenarios, and LanguageModels in a job."
+
+    def __init__(self, jobs: "Jobs"):
+        self.jobs = jobs
+
+    def by(
+        self,
+        *args: Union[
+            "Agent",
+            "Scenario",
+            "LanguageModel",
+            Sequence[Union["Agent", "Scenario", "LanguageModel"]],
+        ],
+    ) -> "Jobs":
+        """
+        Add Agents, Scenarios and LanguageModels to a job.
+
+        :param args: objects or a sequence (list, tuple, ...) of objects of the same type
+
+        If no objects of this type exist in the Jobs instance, it stores the new objects as a list in the corresponding attribute.
+        Otherwise, it combines the new objects with existing objects using the object's `__add__` method.
+
+        This 'by' is intended to create a fluent interface.
+
+        >>> from edsl.surveys import Survey
+        >>> from edsl.questions import QuestionFreeText
+        >>> q = QuestionFreeText(question_name="name", question_text="What is your name?")
+        >>> from edsl.jobs import Jobs
+        >>> j = Jobs(survey = Survey(questions=[q]))
+        >>> j
+        Jobs(survey=Survey(...), agents=AgentList([]), models=ModelList([]), scenarios=ScenarioList([]))
+        >>> from edsl import Agent; a = Agent(traits = {"status": "Sad"})
+        >>> j.by(a).agents
+        AgentList([Agent(traits = {'status': 'Sad'})])
+
+
+        Notes:
+        - all objects must implement the 'get_value', 'set_value', and `__add__` methods
+        - agents: traits of new agents are combined with traits of existing agents. New and existing agents should not have overlapping traits, and do not increase the # agents in the instance
+        - scenarios: traits of new scenarios are combined with traits of old existing. New scenarios will overwrite overlapping traits, and do not increase the number of scenarios in the instance
+        - models: new models overwrite old models.
+        """
+        from edsl.results.Dataset import Dataset
+
+        if isinstance(
+            args[0], Dataset
+        ):  # let the user use a Dataset as if it were a ScenarioList
+            args = args[0].to_scenario_list()
+
+        passed_objects = self._turn_args_to_list(
+            args
+        )  # objects can also be passed comma-separated
+
+        current_objects, objects_key = self._get_current_objects_of_this_type(
+            passed_objects[0]
+        )
+
+        if not current_objects:
+            new_objects = passed_objects
+        else:
+            new_objects = self._merge_objects(passed_objects, current_objects)
+
+        setattr(self.jobs, objects_key, new_objects)  # update the job object
+        return self.jobs
+
+    @staticmethod
+    def _turn_args_to_list(args):
+        """Return a list of the first argument if it is a sequence, otherwise returns a list of all the arguments.
+
+        Example:
+
+        >>> JobsComponentConstructor._turn_args_to_list([1,2,3])
+        [1, 2, 3]
+
+        """
+
+        def did_user_pass_a_sequence(args):
+            """Return True if the user passed a sequence, False otherwise.
+
+            Example:
+
+            >>> did_user_pass_a_sequence([1,2,3])
+            True
+
+            >>> did_user_pass_a_sequence(1)
+            False
+            """
+            return len(args) == 1 and isinstance(args[0], Sequence)
+
+        if did_user_pass_a_sequence(args):
+            container_class = JobsComponentConstructor._get_container_class(args[0][0])
+            return container_class(args[0])
+        else:
+            container_class = JobsComponentConstructor._get_container_class(args[0])
+            return container_class(args)
+
+    def _get_current_objects_of_this_type(
+        self, object: Union["Agent", "Scenario", "LanguageModel"]
+    ) -> tuple[list, str]:
+        from edsl.agents.Agent import Agent
+        from edsl.scenarios.Scenario import Scenario
+        from edsl.language_models.LanguageModel import LanguageModel
+
+        """Return the current objects of the same type as the first argument.
+
+        >>> from edsl.jobs import Jobs
+        >>> j = JobsComponentConstructor(Jobs.example())
+        >>> j._get_current_objects_of_this_type(j.agents[0])
+        (AgentList([Agent(traits = {'status': 'Joyful'}), Agent(traits = {'status': 'Sad'})]), 'agents')
+        """
+        class_to_key = {
+            Agent: "agents",
+            Scenario: "scenarios",
+            LanguageModel: "models",
+        }
+        for class_type in class_to_key:
+            if isinstance(object, class_type) or issubclass(
+                object.__class__, class_type
+            ):
+                key = class_to_key[class_type]
+                break
+        else:
+            raise ValueError(
+                f"First argument must be an Agent, Scenario, or LanguageModel, not {object}"
+            )
+        current_objects = getattr(self.jobs, key, None)
+        return current_objects, key
+
+    @staticmethod
+    def _get_empty_container_object(object):
+        from edsl.agents.AgentList import AgentList
+        from edsl.scenarios.ScenarioList import ScenarioList
+
+        return {"Agent": AgentList([]), "Scenario": ScenarioList([])}.get(
+            object.__class__.__name__, []
+        )
+
+    @staticmethod
+    def _merge_objects(passed_objects, current_objects) -> list:
+        """
+        Combine all the existing objects with the new objects.
+
+        For example, if the user passes in 3 agents,
+        and there are 2 existing agents, this will create 6 new agents
+        >>> from edsl.jobs import Jobs
+        >>> JobsComponentConstructor(Jobs(survey = []))._merge_objects([1,2,3], [4,5,6])
+        [5, 6, 7, 6, 7, 8, 7, 8, 9]
+        """
+        new_objects = JobsComponentConstructor._get_empty_container_object(
+            passed_objects[0]
+        )
+        for current_object in current_objects:
+            for new_object in passed_objects:
+                new_objects.append(current_object + new_object)
+        return new_objects
+
+    @staticmethod
+    def _get_container_class(object):
+        from edsl.agents.AgentList import AgentList
+        from edsl.agents.Agent import Agent
+        from edsl.scenarios.Scenario import Scenario
+        from edsl.scenarios.ScenarioList import ScenarioList
+        from edsl.language_models.ModelList import ModelList
+
+        if isinstance(object, Agent):
+            return AgentList
+        elif isinstance(object, Scenario):
+            return ScenarioList
+        elif isinstance(object, ModelList):
+            return ModelList
+        else:
+            return list
+
+
+if __name__ == "__main__":
+    """Run the module's doctests."""
+    import doctest
+
+    doctest.testmod(optionflags=doctest.ELLIPSIS)

edsl/jobs/JobsPrompts.py

@@ -0,0 +1,270 @@
+from typing import List, TYPE_CHECKING
+
+from edsl.results.Dataset import Dataset
+
+if TYPE_CHECKING:
+    from edsl.jobs import Jobs
+
+    # from edsl.jobs.interviews.Interview import Interview
+    # from edsl.results.Dataset import Dataset
+    # from edsl.agents.AgentList import AgentList
+    # from edsl.scenarios.ScenarioList import ScenarioList
+    # from edsl.surveys.Survey import Survey
+
+from edsl.jobs.FetchInvigilator import FetchInvigilator
+
+
+class JobsPrompts:
+    def __init__(self, jobs: "Jobs"):
+        self.interviews = jobs.interviews()
+        self.agents = jobs.agents
+        self.scenarios = jobs.scenarios
+        self.survey = jobs.survey
+        self._price_lookup = None
+
+    @property
+    def price_lookup(self):
+        if self._price_lookup is None:
+            from edsl.coop.coop import Coop
+
+            c = Coop()
+            self._price_lookup = c.fetch_prices()
+        return self._price_lookup
+
+    def prompts(self) -> "Dataset":
+        """Return a Dataset of prompts that will be used.
+
+        >>> from edsl.jobs import Jobs
+        >>> Jobs.example().prompts()
+        Dataset(...)
+        """
+        interviews = self.interviews
+        interview_indices = []
+        question_names = []
+        user_prompts = []
+        system_prompts = []
+        scenario_indices = []
+        agent_indices = []
+        models = []
+        costs = []
+
+        for interview_index, interview in enumerate(interviews):
+            invigilators = [
+                FetchInvigilator(interview)(question)
+                for question in interview.survey.questions
+            ]
+            for _, invigilator in enumerate(invigilators):
+                prompts = invigilator.get_prompts()
+                user_prompt = prompts["user_prompt"]
+                system_prompt = prompts["system_prompt"]
+                user_prompts.append(user_prompt)
+                system_prompts.append(system_prompt)
+                agent_index = self.agents.index(invigilator.agent)
+                agent_indices.append(agent_index)
+                interview_indices.append(interview_index)
+                scenario_index = self.scenarios.index(invigilator.scenario)
+                scenario_indices.append(scenario_index)
+                models.append(invigilator.model.model)
+                question_names.append(invigilator.question.question_name)
+
+                prompt_cost = self.estimate_prompt_cost(
+                    system_prompt=system_prompt,
+                    user_prompt=user_prompt,
+                    price_lookup=self.price_lookup,
+                    inference_service=invigilator.model._inference_service_,
+                    model=invigilator.model.model,
+                )
+                costs.append(prompt_cost["cost_usd"])
+
+        d = Dataset(
+            [
+                {"user_prompt": user_prompts},
+                {"system_prompt": system_prompts},
+                {"interview_index": interview_indices},
+                {"question_name": question_names},
+                {"scenario_index": scenario_indices},
+                {"agent_index": agent_indices},
+                {"model": models},
+                {"estimated_cost": costs},
+            ]
+        )
+        return d
+
+    @staticmethod
+    def estimate_prompt_cost(
+        system_prompt: str,
+        user_prompt: str,
+        price_lookup: dict,
+        inference_service: str,
+        model: str,
+    ) -> dict:
+        """Estimates the cost of a prompt. Takes piping into account."""
+        import math
+
+        def get_piping_multiplier(prompt: str):
+            """Returns 2 if a prompt includes Jinja braces, and 1 otherwise."""
+
+            if "{{" in prompt and "}}" in prompt:
+                return 2
+            return 1
+
+        # Look up prices per token
+        key = (inference_service, model)
+
+        try:
+            relevant_prices = price_lookup[key]
+
+            service_input_token_price = float(
+                relevant_prices["input"]["service_stated_token_price"]
+            )
+            service_input_token_qty = float(
+                relevant_prices["input"]["service_stated_token_qty"]
+            )
+            input_price_per_token = service_input_token_price / service_input_token_qty
+
+            service_output_token_price = float(
+                relevant_prices["output"]["service_stated_token_price"]
+            )
+            service_output_token_qty = float(
+                relevant_prices["output"]["service_stated_token_qty"]
+            )
+            output_price_per_token = (
+                service_output_token_price / service_output_token_qty
+            )
+
+        except KeyError:
+            # A KeyError is likely to occur if we cannot retrieve prices (the price_lookup dict is empty)
+            # Use a sensible default
+
+            import warnings
+
+            warnings.warn(
+                "Price data could not be retrieved. Using default estimates for input and output token prices. Input: $0.15 / 1M tokens; Output: $0.60 / 1M tokens"
+            )
+            input_price_per_token = 0.00000015  # $0.15 / 1M tokens
+            output_price_per_token = 0.00000060  # $0.60 / 1M tokens
+
+        # Compute the number of characters (double if the question involves piping)
+        user_prompt_chars = len(str(user_prompt)) * get_piping_multiplier(
+            str(user_prompt)
+        )
+        system_prompt_chars = len(str(system_prompt)) * get_piping_multiplier(
+            str(system_prompt)
+        )
+
+        # Convert into tokens (1 token approx. equals 4 characters)
+        input_tokens = (user_prompt_chars + system_prompt_chars) // 4
+
+        output_tokens = math.ceil(0.75 * input_tokens)
+
+        cost = (
+            input_tokens * input_price_per_token
+            + output_tokens * output_price_per_token
+        )
+
+        return {
+            "input_tokens": input_tokens,
+            "output_tokens": output_tokens,
+            "cost_usd": cost,
+        }
+
+    def estimate_job_cost_from_external_prices(
+        self, price_lookup: dict, iterations: int = 1
+    ) -> dict:
+        """
+        Estimates the cost of a job according to the following assumptions:
+
+        - 1 token = 4 characters.
+        - For each prompt, output tokens = input tokens * 0.75, rounded up to the nearest integer.
+
+        price_lookup is an external pricing dictionary.
+        """
+
+        import pandas as pd
+
+        interviews = self.interviews
+        data = []
+        for interview in interviews:
+            invigilators = [
+                FetchInvigilator(interview)(question)
+                for question in self.survey.questions
+            ]
+            for invigilator in invigilators:
+                prompts = invigilator.get_prompts()
+
+                # By this point, agent and scenario data has already been added to the prompts
+                user_prompt = prompts["user_prompt"]
+                system_prompt = prompts["system_prompt"]
+                inference_service = invigilator.model._inference_service_
+                model = invigilator.model.model
+
+                prompt_cost = self.estimate_prompt_cost(
+                    system_prompt=system_prompt,
+                    user_prompt=user_prompt,
+                    price_lookup=price_lookup,
+                    inference_service=inference_service,
+                    model=model,
+                )
+
+                data.append(
+                    {
+                        "user_prompt": user_prompt,
+                        "system_prompt": system_prompt,
+                        "estimated_input_tokens": prompt_cost["input_tokens"],
+                        "estimated_output_tokens": prompt_cost["output_tokens"],
+                        "estimated_cost_usd": prompt_cost["cost_usd"],
+                        "inference_service": inference_service,
+                        "model": model,
+                    }
+                )
+
+        df = pd.DataFrame.from_records(data)
+
+        df = (
+            df.groupby(["inference_service", "model"])
+            .agg(
+                {
+                    "estimated_cost_usd": "sum",
+                    "estimated_input_tokens": "sum",
+                    "estimated_output_tokens": "sum",
+                }
+            )
+            .reset_index()
+        )
+        df["estimated_cost_usd"] = df["estimated_cost_usd"] * iterations
+        df["estimated_input_tokens"] = df["estimated_input_tokens"] * iterations
+        df["estimated_output_tokens"] = df["estimated_output_tokens"] * iterations
+
+        estimated_costs_by_model = df.to_dict("records")
+
+        estimated_total_cost = sum(
+            model["estimated_cost_usd"] for model in estimated_costs_by_model
+        )
+        estimated_total_input_tokens = sum(
+            model["estimated_input_tokens"] for model in estimated_costs_by_model
+        )
+        estimated_total_output_tokens = sum(
+            model["estimated_output_tokens"] for model in estimated_costs_by_model
+        )
+
+        output = {
+            "estimated_total_cost_usd": estimated_total_cost,
+            "estimated_total_input_tokens": estimated_total_input_tokens,
+            "estimated_total_output_tokens": estimated_total_output_tokens,
+            "model_costs": estimated_costs_by_model,
+        }
+
+        return output
+
+    def estimate_job_cost(self, iterations: int = 1) -> dict:
+        """
+        Estimates the cost of a job according to the following assumptions:
+
+        - 1 token = 4 characters.
+        - For each prompt, output tokens = input tokens * 0.75, rounded up to the nearest integer.
+
+        Fetches prices from Coop.
+        """
+        return self.estimate_job_cost_from_external_prices(
+            price_lookup=self.price_lookup, iterations=iterations
+        )