edsl 0.1.40.dev2__py3-none-any.whl → 0.1.42__py3-none-any.whl

edsl/__init__.py

@@ -21,6 +21,7 @@ from edsl.questions import QuestionFunctional
 from edsl.questions import QuestionLikertFive
 from edsl.questions import QuestionList
 from edsl.questions import QuestionMatrix
+from edsl.questions import QuestionDict
 from edsl.questions import QuestionLinearScale
 from edsl.questions import QuestionNumerical
 from edsl.questions import QuestionYesNo

edsl/__version__.py

@@ -1 +1 @@
-__version__ = "0.1.40.dev2"
+__version__ = "0.1.42"

edsl/agents/Agent.py

@@ -906,7 +906,7 @@ class Agent(Base):
         {'traits': {'age': 10, 'hair': 'brown', 'height': 5.5}, 'instruction': 'Have fun.', 'edsl_version': '...', 'edsl_class_name': 'Agent'}
         """
         d = {}
-        d["traits"] = copy.deepcopy(self.traits)
+        d["traits"] = copy.deepcopy(dict(self._traits))
         if self.name:
             d["name"] = self.name
         if self.set_instructions:

edsl/agents/Invigilator.py

@@ -1,6 +1,6 @@
 """Module for creating Invigilators, which are objects to administer a question to an Agent."""
 
-from typing import Dict, Any, Optional, TYPE_CHECKING
+from typing import Dict, Any, Optional, TYPE_CHECKING, Literal
 
 from edsl.utilities.decorators import sync_wrapper
 from edsl.exceptions.questions import QuestionAnswerValidationError
@@ -12,6 +12,7 @@ if TYPE_CHECKING:
     from edsl.scenarios.Scenario import Scenario
     from edsl.surveys.Survey import Survey
 
+PromptType = Literal["user_prompt", "system_prompt", "encoded_image", "files_list"]
 
 NA = "Not Applicable"
 
@@ -19,7 +20,7 @@ NA = "Not Applicable"
 class InvigilatorAI(InvigilatorBase):
     """An invigilator that uses an AI model to answer questions."""
 
-    def get_prompts(self) -> Dict[str, "Prompt"]:
+    def get_prompts(self) -> Dict[PromptType, "Prompt"]:
         """Return the prompts used."""
         return self.prompt_constructor.get_prompts()
 
@@ -48,13 +49,14 @@ class InvigilatorAI(InvigilatorBase):
         """Store the response in the invigilator, in case it is needed later because of validation failure."""
         self.raw_model_response = agent_response_dict.model_outputs.response
         self.generated_tokens = agent_response_dict.edsl_dict.generated_tokens
+        self.cache_key = agent_response_dict.model_outputs.cache_key
 
-    async def async_answer_question(self) -> AgentResponseDict:
+    async def async_answer_question(self) -> EDSLResultObjectInput:
         """Answer a question using the AI model.
 
         >>> i = InvigilatorAI.example()
         """
-        agent_response_dict = await self.async_get_agent_response()
+        agent_response_dict: AgentResponseDict = await self.async_get_agent_response()
         self.store_response(agent_response_dict)
         return self._extract_edsl_result_entry_and_validate(agent_response_dict)
 

edsl/agents/InvigilatorBase.py

@@ -135,6 +135,7 @@ class InvigilatorBase(ABC):
         d["additional_prompt_data"] = data["additional_prompt_data"]
 
         d = cls(**d)
+        return d
 
     def __repr__(self) -> str:
         """Return a string representation of the Invigilator.
@@ -143,7 +144,7 @@ class InvigilatorBase(ABC):
         'InvigilatorExample(...)'
 
         """
-        return f"{self.__class__.__name__}(agent={repr(self.agent)}, question={repr(self.question)}, scneario={repr(self.scenario)}, model={repr(self.model)}, memory_plan={repr(self.memory_plan)}, current_answers={repr(self.current_answers)}, iteration{repr(self.iteration)}, additional_prompt_data={repr(self.additional_prompt_data)}, cache={repr(self.cache)})"
+        return f"{self.__class__.__name__}(agent={repr(self.agent)}, question={repr(self.question)}, scenario={repr(self.scenario)}, model={repr(self.model)}, memory_plan={repr(self.memory_plan)}, current_answers={repr(self.current_answers)}, iteration={repr(self.iteration)}, additional_prompt_data={repr(self.additional_prompt_data)}, cache={repr(self.cache)})"
 
     def get_failed_task_result(self, failure_reason: str) -> EDSLResultObjectInput:
         """Return an AgentResponseDict used in case the question-asking fails.

edsl/agents/QuestionTemplateReplacementsBuilder.py

@@ -1,4 +1,4 @@
-from jinja2 import Environment, meta
+from jinja2 import Environment, meta, TemplateSyntaxError
 from typing import Any, Set, TYPE_CHECKING
 
 if TYPE_CHECKING:
@@ -29,7 +29,12 @@ class QuestionTemplateReplacementsBuilder:
         Set[str]: A set of variable names found in the template
         """
         env = Environment()
-        ast = env.parse(template_str)
+        try:
+            ast = env.parse(template_str)
+        except TemplateSyntaxError:
+            print(f"Error parsing template: {template_str}")
+            raise
+
         return meta.find_undeclared_variables(ast)
 
     @staticmethod

edsl/coop/coop.py

@@ -111,13 +111,13 @@ class Coop(CoopFunctionsMixin):
         url = f"{self.api_url}/{uri}"
         method = method.upper()
         if payload is None:
-            timeout = 20
+            timeout = 40
         elif (
             method.upper() == "POST"
             and "json_string" in payload
             and payload.get("json_string") is not None
         ):
-            timeout = max(20, (len(payload.get("json_string", "")) // (1024 * 1024)))
+            timeout = max(40, (len(payload.get("json_string", "")) // (1024 * 1024)))
         try:
             if method in ["GET", "DELETE"]:
                 response = requests.request(
@@ -533,6 +533,7 @@ class Coop(CoopFunctionsMixin):
             uri="api/v0/remote-cache/many",
             method="POST",
             payload=payload,
+            timeout=40,
         )
         self._resolve_server_response(response)
         response_json = response.json()
@@ -563,6 +564,7 @@ class Coop(CoopFunctionsMixin):
             uri="api/v0/remote-cache/get-many",
             method="POST",
             payload={"keys": exclude_keys},
+            timeout=40,
         )
         self._resolve_server_response(response)
         return [
@@ -581,6 +583,7 @@ class Coop(CoopFunctionsMixin):
             uri="api/v0/remote-cache/get-diff",
             method="POST",
             payload={"keys": client_cacheentry_keys},
+            timeout=40,
         )
         self._resolve_server_response(response)
         response_json = response.json()
@@ -891,6 +894,38 @@ class Coop(CoopFunctionsMixin):
         data = response.json()
         return ServiceToModelsMapping(data)
 
+    def fetch_working_models(self) -> list[dict]:
+        """
+        Fetch a list of working models from Coop.
+
+        Example output:
+
+        [
+            {
+                "service": "openai",
+                "model": "gpt-4o",
+                "works_with_text": True,
+                "works_with_images": True,
+                "usd_per_1M_input_tokens": 2.5,
+                "usd_per_1M_output_tokens": 10.0,
+            }
+        ]
+        """
+        response = self._send_server_request(uri="api/v0/working-models", method="GET")
+        self._resolve_server_response(response)
+        data = response.json()
+        return [
+            {
+                "service": record.get("service"),
+                "model": record.get("model"),
+                "works_with_text": record.get("works_with_text"),
+                "works_with_images": record.get("works_with_images"),
+                "usd_per_1M_input_tokens": record.get("input_price_per_1M_tokens"),
+                "usd_per_1M_output_tokens": record.get("output_price_per_1M_tokens"),
+            }
+            for record in data
+        ]
+
     def fetch_rate_limit_config_vars(self) -> dict:
         """
         Fetch a dict of rate limit config vars from Coop.

edsl/data/Cache.py

@@ -535,6 +535,13 @@ class Cache(Base):
         """
         return html
 
+    def subset(self, keys: list[str]) -> Cache:
+        """
+        Return a subset of the Cache with the specified keys.
+        """
+        new_data = {k: v for k, v in self.data.items() if k in keys}
+        return Cache(data=new_data)
+
     def view(self) -> None:
         """View the Cache in a new browser tab."""
         import tempfile

edsl/data/RemoteCacheSync.py

@@ -112,18 +112,18 @@ class RemoteCacheSync(AbstractContextManager):
         missing_count = len(diff.client_missing_entries)
 
         if missing_count == 0:
-            self._output("No new entries to add to local cache.")
+        #     self._output("No new entries to add to local cache.")
             return
 
-        self._output(
-            f"Updating local cache with {missing_count:,} new "
-            f"{'entry' if missing_count == 1 else 'entries'} from remote..."
-        )
+        # self._output(
+        #     f"Updating local cache with {missing_count:,} new "
+        #     f"{'entry' if missing_count == 1 else 'entries'} from remote..."
+        # )
 
         self.cache.add_from_dict(
             {entry.key: entry for entry in diff.client_missing_entries}
         )
-        self._output("Local cache updated!")
+        # self._output("Local cache updated!")
 
     def _get_entries_to_upload(self, diff: CacheDifference) -> CacheEntriesList:
         """Determines which entries need to be uploaded to remote cache."""
@@ -154,23 +154,23 @@ class RemoteCacheSync(AbstractContextManager):
         upload_count = len(entries_to_upload)
 
         if upload_count > 0:
-            self._output(
-                f"Updating remote cache with {upload_count:,} new "
-                f"{'entry' if upload_count == 1 else 'entries'}..."
-            )
+            # self._output(
+            #     f"Updating remote cache with {upload_count:,} new "
+            #     f"{'entry' if upload_count == 1 else 'entries'}..."
+            # )
 
             self.coop.remote_cache_create_many(
                 entries_to_upload,
                 visibility="private",
                 description=self.remote_cache_description,
             )
-            self._output("Remote cache updated!")
-        else:
-            self._output("No new entries to add to remote cache.")
+            # self._output("Remote cache updated!")
+        # else:
+            # self._output("No new entries to add to remote cache.")
 
-        self._output(
-            f"There are {len(self.cache.keys()):,} entries in the local cache."
-        )
+        # self._output(
+            # f"There are {len(self.cache.keys()):,} entries in the local cache."
+        # )
 
 
 if __name__ == "__main__":

edsl/enums.py

@@ -66,6 +66,7 @@ class InferenceServiceType(EnumWithChecks):
     MISTRAL = "mistral"
     TOGETHER = "together"
     PERPLEXITY = "perplexity"
+    DEEPSEEK = "deepseek"
 
 
 # unavoidable violation of the DRY principle but it is necessary
@@ -84,6 +85,7 @@ InferenceServiceLiteral = Literal[
     "mistral",
     "together",
     "perplexity",
+    "deepseek",
 ]
 
 available_models_urls = {
@@ -107,6 +109,7 @@ service_to_api_keyname = {
     InferenceServiceType.MISTRAL.value: "MISTRAL_API_KEY",
     InferenceServiceType.TOGETHER.value: "TOGETHER_API_KEY",
     InferenceServiceType.PERPLEXITY.value: "PERPLEXITY_API_KEY",
+    InferenceServiceType.DEEPSEEK.value: "DEEPSEEK_API_KEY",
 }
 
 

edsl/exceptions/jobs.py

@@ -10,15 +10,7 @@ class JobsRunError(JobsErrors):
 
 
 class MissingRemoteInferenceError(JobsErrors):
-    def __init__(self):
-        message = dedent(
-            """\\
-                        You are trying to run the job remotely, but you have not set the EXPECTED_PARROT_INFERENCE_URL environment variable.
-                        This remote running service is not quite ready yet! 
-                        But please see https://docs.expectedparrot.com/en/latest/coop.html for what we are working on.
-                        """
-        )
-        super().__init__(message)
+    pass
 
 
 class InterviewError(Exception):

edsl/exceptions/language_models.py

@@ -34,11 +34,15 @@ class LanguageModelNotFound(LanguageModelExceptions):
         msg = dedent(
             f"""\
             Model {model_name} not found.
-            To create an instance, you can do: 
-            >>> m = Model('gpt-4-1106-preview', temperature=0.5, ...)
+            To create an instance of this model, pass the model name to a `Model` object.
+            You can optionally pass additional parameters to the model, e.g.: 
+            >>> m = Model('gpt-4-1106-preview', temperature=0.5)
             
-            To get the default model, you can leave out the model name. 
-            To see the available models, you can do:
+            To use the default model, simply run your job without specifying a model.
+            To check the default model, run the following code:
+            >>> Model()
+
+            To see information about all available models, run the following code:
             >>> Model.available()
 
             See https://docs.expectedparrot.com/en/latest/language_models.html#available-models for more details.

edsl/exceptions/questions.py

@@ -16,7 +16,8 @@ class QuestionErrors(Exception):
 class QuestionAnswerValidationError(QuestionErrors):
     documentation = "https://docs.expectedparrot.com/en/latest/exceptions.html"
 
-    explanation = """This when the answer coming from the Language Model does not conform to the expectation for that question type.
+    explanation = """
+    This can occur when the answer coming from the Language Model does not conform to the expectations for the question type.
     For example, if the question is a multiple choice question, the answer should be drawn from the list of options provided.
     """
 
@@ -52,28 +53,24 @@ class QuestionAnswerValidationError(QuestionErrors):
 
     def to_html_dict(self):
         return {
-            "error_type": ("Name of the exception", "p", "/p", self.__class__.__name__),
-            "explaination": ("Explanation", "p", "/p", self.explanation),
-            "edsl answer": (
-                "What model returned",
+            "Exception type": ("p", "/p", self.__class__.__name__),
+            "Explanation": ("p", "/p", self.explanation),
+            "EDSL response": (
                 "pre",
                 "/pre",
                 json.dumps(self.data, indent=2),
             ),
-            "validating_model": (
-                "Pydantic model for answers",
+            "Validating model": (
                 "pre",
                 "/pre",
                 json.dumps(self.model.model_json_schema(), indent=2),
             ),
-            "error_message": (
-                "Error message Pydantic returned",
+            "Error message": (
                 "p",
                 "/p",
                 self.message,
             ),
-            "documentation_url": (
-                "URL to EDSL docs",
+            "Documentation": (
                 f"a href='{self.documentation}'",
                 "/a",
                 self.documentation,

edsl/inference_services/DeepSeekService.py

@@ -0,0 +1,18 @@
+import aiohttp
+import json
+import requests
+from typing import Any, List
+
+# from edsl.inference_services.InferenceServiceABC import InferenceServiceABC
+from edsl.language_models import LanguageModel
+
+from edsl.inference_services.OpenAIService import OpenAIService
+
+
+class DeepSeekService(OpenAIService):
+    """DeepInfra service class."""
+
+    _inference_service_ = "deepseek"
+    _env_key_name_ = "DEEPSEEK_API_KEY"
+    _base_url_ = "https://api.deepseek.com"
+    _models_list_cache: List[str] = []

edsl/inference_services/registry.py

@@ -13,6 +13,7 @@ from edsl.inference_services.OllamaService import OllamaService
 from edsl.inference_services.TestService import TestService
 from edsl.inference_services.TogetherAIService import TogetherAIService
 from edsl.inference_services.PerplexityService import PerplexityService
+from edsl.inference_services.DeepSeekService import DeepSeekService
 
 try:
     from edsl.inference_services.MistralAIService import MistralAIService
@@ -33,6 +34,7 @@ services = [
     TestService,
     TogetherAIService,
     PerplexityService,
+    DeepSeekService,
 ]
 
 if mistral_available:

edsl/jobs/AnswerQuestionFunctionConstructor.py

@@ -139,7 +139,7 @@ class AnswerQuestionFunctionConstructor:
         *,
         question: "QuestionBase",
         task=None,
-    ) -> "AgentResponseDict":
+    ) -> "EDSLResultObjectInput":
 
         from tenacity import (
             retry,

edsl/jobs/Jobs.py

@@ -499,7 +499,6 @@ class Jobs(Base):
             jc.check_api_keys()
 
     async def _execute_with_remote_cache(self, run_job_async: bool) -> Results:
-
         use_remote_cache = self.use_remote_cache()
 
         from edsl.coop.coop import Coop
@@ -508,43 +507,48 @@ class Jobs(Base):
 
         assert isinstance(self.run_config.environment.cache, Cache)
 
-        with RemoteCacheSync(
-            coop=Coop(),
-            cache=self.run_config.environment.cache,
-            output_func=self._output,
-            remote_cache=use_remote_cache,
-            remote_cache_description=self.run_config.parameters.remote_cache_description,
-        ):
-            runner = JobsRunnerAsyncio(self, environment=self.run_config.environment)
-            if run_job_async:
-                results = await runner.run_async(self.run_config.parameters)
-            else:
-                results = runner.run(self.run_config.parameters)
+        # with RemoteCacheSync(
+        #     coop=Coop(),
+        #     cache=self.run_config.environment.cache,
+        #     output_func=self._output,
+        #     remote_cache=use_remote_cache,
+        #     remote_cache_description=self.run_config.parameters.remote_cache_description,
+        # ):
+        runner = JobsRunnerAsyncio(self, environment=self.run_config.environment)
+        if run_job_async:
+            results = await runner.run_async(self.run_config.parameters)
+        else:
+            results = runner.run(self.run_config.parameters)
         return results
 
-    def _setup_and_check(self) -> Tuple[RunConfig, Optional[Results]]:
+    # def _setup_and_check(self) -> Tuple[RunConfig, Optional[Results]]:
+    #     self._prepare_to_run()
+    #     self._check_if_remote_keys_ok()
 
-        self._prepare_to_run()
-        self._check_if_remote_keys_ok()
-
-        # first try to run the job remotely
-        if results := self._remote_results():
-            return results
+    #     # first try to run the job remotely
+    #     results = self._remote_results()
+    #     #breakpoint()
+    #     if results is not None:
+    #         return results
 
-        self._check_if_local_keys_ok()
-        return None
+    #     self._check_if_local_keys_ok()
+    #     return None
 
     @property
     def num_interviews(self):
         if self.run_config.parameters.n is None:
             return len(self)
         else:
-            len(self) * self.run_config.parameters.n
+            return len(self) * self.run_config.parameters.n
 
-    def _run(self, config: RunConfig):
+    def _run(self, config: RunConfig) -> Union[None, "Results"]:
         "Shared code for run and run_async"
         if config.environment.cache is not None:
             self.run_config.environment.cache = config.environment.cache
+        if config.environment.jobs_runner_status is not None:
+            self.run_config.environment.jobs_runner_status = (
+                config.environment.jobs_runner_status
+            )
 
         if config.environment.bucket_collection is not None:
             self.run_config.environment.bucket_collection = (
@@ -579,7 +583,7 @@ class Jobs(Base):
         # first try to run the job remotely
         if results := self._remote_results():
             return results
-
+                
         self._check_if_local_keys_ok()
 
         if config.environment.bucket_collection is None:
@@ -587,6 +591,8 @@ class Jobs(Base):
                 self.create_bucket_collection()
             )
 
+        return None
+
     @with_config
     def run(self, *, config: RunConfig) -> "Results":
         """
@@ -606,7 +612,10 @@ class Jobs(Base):
         :param bucket_collection: A BucketCollection object to track API calls
         :param key_lookup: A KeyLookup object to manage API keys
         """
-        self._run(config)
+        potentially_completed_results = self._run(config)
+        
+        if potentially_completed_results is not None:
+            return potentially_completed_results
 
         return asyncio.run(self._execute_with_remote_cache(run_job_async=False))
 
@@ -646,20 +655,19 @@ class Jobs(Base):
         }
 
     def __len__(self) -> int:
-        """Return the maximum number of questions that will be asked while running this job.
-        Note that this is the maximum number of questions, not the actual number of questions that will be asked, as some questions may be skipped.
+        """Return the number of interviews that will be conducted for one iteration of this job.
+        An interview is the result of one survey, taken by one agent, with one model, with one scenario.
 
         >>> from edsl.jobs import Jobs
         >>> len(Jobs.example())
-        8
+        4
         """
-        number_of_questions = (
+        number_of_interviews = (
             len(self.agents or [1])
             * len(self.scenarios or [1])
             * len(self.models or [1])
-            * len(self.survey)
         )
-        return number_of_questions
+        return number_of_interviews
 
     def to_dict(self, add_edsl_version=True):
         d = {
@@ -810,9 +818,9 @@ def main():
     from edsl.data.Cache import Cache
 
     job = Jobs.example()
-    len(job) == 8
+    len(job) == 4
     results = job.run(cache=Cache())
-    len(results) == 8
+    len(results) == 4
     results
 
 

edsl/jobs/JobsPrompts.py

@@ -12,6 +12,7 @@ if TYPE_CHECKING:
     # from edsl.surveys.Survey import Survey
 
 from edsl.jobs.FetchInvigilator import FetchInvigilator
+from edsl.data.CacheEntry import CacheEntry
 
 
 class JobsPrompts:
@@ -47,7 +48,7 @@ class JobsPrompts:
         agent_indices = []
         models = []
         costs = []
-
+        cache_keys = []
         for interview_index, interview in enumerate(interviews):
             invigilators = [
                 FetchInvigilator(interview)(question)
@@ -76,6 +77,14 @@ class JobsPrompts:
                 )
                 costs.append(prompt_cost["cost_usd"])
 
+                cache_key = CacheEntry.gen_key(
+                    model=invigilator.model.model,
+                    parameters=invigilator.model.parameters,
+                    system_prompt=system_prompt,
+                    user_prompt=user_prompt,
+                    iteration=0,  # TODO how to handle when there are multiple iterations?
+                )
+                cache_keys.append(cache_key)
         d = Dataset(
             [
                 {"user_prompt": user_prompts},
@@ -86,6 +95,7 @@ class JobsPrompts:
                 {"agent_index": agent_indices},
                 {"model": models},
                 {"estimated_cost": costs},
+                {"cache_key": cache_keys},
             ]
         )
         return d

edsl/jobs/JobsRemoteInferenceHandler.py

@@ -219,6 +219,7 @@ class JobsRemoteInferenceHandler:
         job_info.logger.add_info("results_uuid", results_uuid)
         results = object_fetcher(results_uuid, expected_object_type="results")
         results_url = remote_job_data.get("results_url")
+        job_info.logger.add_info("results_url", results_url)
         job_info.logger.update(
             f"Job completed and Results stored on Coop: {results_url}",
             status=JobsStatus.COMPLETED,

edsl/jobs/JobsRemoteInferenceLogger.py

@@ -32,7 +32,7 @@ class JobsInfo:
     pretty_names = {
         "job_uuid": "Job UUID",
         "progress_bar_url": "Progress Bar URL",
-        "error_report_url": "Error Report URL",
+        "error_report_url": "Exceptions Report URL",
         "results_uuid": "Results UUID",
         "results_url": "Results URL",
     }

edsl/jobs/interviews/Interview.py

@@ -153,7 +153,7 @@ class Interview:
 
         >>> i = Interview.example()
         >>> hash(i)
-        193593189022259693
+        767745459362662063
         """
         d = {
             "agent": self.agent.to_dict(add_edsl_version=add_edsl_version),
@@ -213,10 +213,6 @@ class Interview:
     async def async_conduct_interview(
         self,
         run_config: Optional["RunConfig"] = None,
-        #     model_buckets: Optional[ModelBuckets] = None,
-        #     stop_on_exception: bool = False,
-        #     raise_validation_errors: bool = True,
-        #     key_lookup: Optional[KeyLookup] = None,
     ) -> tuple["Answers", List[dict[str, Any]]]:
         """
         Conduct an Interview asynchronously.
@@ -313,7 +309,7 @@ class Interview:
 
         def handle_task(task, invigilator):
             try:
-                result = task.result()
+                result: Answers = task.result()
             except asyncio.CancelledError as e:  # task was cancelled
                 result = invigilator.get_failed_task_result(
                     failure_reason="Task was cancelled."