edsl 0.1.40.dev2__py3-none-any.whl → 0.1.41__py3-none-any.whl

edsl/__init__.py

@@ -21,6 +21,7 @@ from edsl.questions import QuestionFunctional
 from edsl.questions import QuestionLikertFive
 from edsl.questions import QuestionList
 from edsl.questions import QuestionMatrix
+from edsl.questions import QuestionDict
 from edsl.questions import QuestionLinearScale
 from edsl.questions import QuestionNumerical
 from edsl.questions import QuestionYesNo

edsl/__version__.py

@@ -1 +1 @@
-__version__ = "0.1.40.dev2"
+__version__ = "0.1.41"

edsl/agents/Agent.py

@@ -906,7 +906,7 @@ class Agent(Base):
         {'traits': {'age': 10, 'hair': 'brown', 'height': 5.5}, 'instruction': 'Have fun.', 'edsl_version': '...', 'edsl_class_name': 'Agent'}
         """
         d = {}
-        d["traits"] = copy.deepcopy(self.traits)
+        d["traits"] = copy.deepcopy(dict(self._traits))
         if self.name:
             d["name"] = self.name
         if self.set_instructions:

edsl/agents/Invigilator.py

@@ -48,13 +48,14 @@ class InvigilatorAI(InvigilatorBase):
         """Store the response in the invigilator, in case it is needed later because of validation failure."""
         self.raw_model_response = agent_response_dict.model_outputs.response
         self.generated_tokens = agent_response_dict.edsl_dict.generated_tokens
+        self.cache_key = agent_response_dict.model_outputs.cache_key
 
-    async def async_answer_question(self) -> AgentResponseDict:
+    async def async_answer_question(self) -> EDSLResultObjectInput:
         """Answer a question using the AI model.
 
         >>> i = InvigilatorAI.example()
         """
-        agent_response_dict = await self.async_get_agent_response()
+        agent_response_dict: AgentResponseDict = await self.async_get_agent_response()
         self.store_response(agent_response_dict)
         return self._extract_edsl_result_entry_and_validate(agent_response_dict)
 

edsl/coop/coop.py

@@ -111,13 +111,13 @@ class Coop(CoopFunctionsMixin):
         url = f"{self.api_url}/{uri}"
         method = method.upper()
         if payload is None:
-            timeout = 20
+            timeout = 40
         elif (
             method.upper() == "POST"
             and "json_string" in payload
             and payload.get("json_string") is not None
         ):
-            timeout = max(20, (len(payload.get("json_string", "")) // (1024 * 1024)))
+            timeout = max(40, (len(payload.get("json_string", "")) // (1024 * 1024)))
         try:
             if method in ["GET", "DELETE"]:
                 response = requests.request(
@@ -533,6 +533,7 @@ class Coop(CoopFunctionsMixin):
             uri="api/v0/remote-cache/many",
             method="POST",
             payload=payload,
+            timeout=40,
         )
         self._resolve_server_response(response)
         response_json = response.json()
@@ -563,6 +564,7 @@ class Coop(CoopFunctionsMixin):
             uri="api/v0/remote-cache/get-many",
             method="POST",
             payload={"keys": exclude_keys},
+            timeout=40,
         )
         self._resolve_server_response(response)
         return [
@@ -581,6 +583,7 @@ class Coop(CoopFunctionsMixin):
             uri="api/v0/remote-cache/get-diff",
             method="POST",
             payload={"keys": client_cacheentry_keys},
+            timeout=40,
         )
         self._resolve_server_response(response)
         response_json = response.json()

edsl/data/Cache.py

@@ -535,6 +535,13 @@ class Cache(Base):
         """
         return html
 
+    def subset(self, keys: list[str]) -> Cache:
+        """
+        Return a subset of the Cache with the specified keys.
+        """
+        new_data = {k: v for k, v in self.data.items() if k in keys}
+        return Cache(data=new_data)
+
     def view(self) -> None:
         """View the Cache in a new browser tab."""
         import tempfile

edsl/data/RemoteCacheSync.py

@@ -112,18 +112,18 @@ class RemoteCacheSync(AbstractContextManager):
         missing_count = len(diff.client_missing_entries)
 
         if missing_count == 0:
-            self._output("No new entries to add to local cache.")
+        #     self._output("No new entries to add to local cache.")
             return
 
-        self._output(
-            f"Updating local cache with {missing_count:,} new "
-            f"{'entry' if missing_count == 1 else 'entries'} from remote..."
-        )
+        # self._output(
+        #     f"Updating local cache with {missing_count:,} new "
+        #     f"{'entry' if missing_count == 1 else 'entries'} from remote..."
+        # )
 
         self.cache.add_from_dict(
             {entry.key: entry for entry in diff.client_missing_entries}
         )
-        self._output("Local cache updated!")
+        # self._output("Local cache updated!")
 
     def _get_entries_to_upload(self, diff: CacheDifference) -> CacheEntriesList:
         """Determines which entries need to be uploaded to remote cache."""
@@ -154,23 +154,23 @@ class RemoteCacheSync(AbstractContextManager):
         upload_count = len(entries_to_upload)
 
         if upload_count > 0:
-            self._output(
-                f"Updating remote cache with {upload_count:,} new "
-                f"{'entry' if upload_count == 1 else 'entries'}..."
-            )
+            # self._output(
+            #     f"Updating remote cache with {upload_count:,} new "
+            #     f"{'entry' if upload_count == 1 else 'entries'}..."
+            # )
 
             self.coop.remote_cache_create_many(
                 entries_to_upload,
                 visibility="private",
                 description=self.remote_cache_description,
             )
-            self._output("Remote cache updated!")
-        else:
-            self._output("No new entries to add to remote cache.")
+            # self._output("Remote cache updated!")
+        # else:
+            # self._output("No new entries to add to remote cache.")
 
-        self._output(
-            f"There are {len(self.cache.keys()):,} entries in the local cache."
-        )
+        # self._output(
+            # f"There are {len(self.cache.keys()):,} entries in the local cache."
+        # )
 
 
 if __name__ == "__main__":

edsl/jobs/AnswerQuestionFunctionConstructor.py

@@ -139,7 +139,7 @@ class AnswerQuestionFunctionConstructor:
         *,
         question: "QuestionBase",
         task=None,
-    ) -> "AgentResponseDict":
+    ) -> "EDSLResultObjectInput":
 
         from tenacity import (
             retry,

edsl/jobs/Jobs.py

@@ -499,7 +499,6 @@ class Jobs(Base):
             jc.check_api_keys()
 
     async def _execute_with_remote_cache(self, run_job_async: bool) -> Results:
-
         use_remote_cache = self.use_remote_cache()
 
         from edsl.coop.coop import Coop
@@ -508,22 +507,21 @@ class Jobs(Base):
 
         assert isinstance(self.run_config.environment.cache, Cache)
 
-        with RemoteCacheSync(
-            coop=Coop(),
-            cache=self.run_config.environment.cache,
-            output_func=self._output,
-            remote_cache=use_remote_cache,
-            remote_cache_description=self.run_config.parameters.remote_cache_description,
-        ):
-            runner = JobsRunnerAsyncio(self, environment=self.run_config.environment)
-            if run_job_async:
-                results = await runner.run_async(self.run_config.parameters)
-            else:
-                results = runner.run(self.run_config.parameters)
+        # with RemoteCacheSync(
+        #     coop=Coop(),
+        #     cache=self.run_config.environment.cache,
+        #     output_func=self._output,
+        #     remote_cache=use_remote_cache,
+        #     remote_cache_description=self.run_config.parameters.remote_cache_description,
+        # ):
+        runner = JobsRunnerAsyncio(self, environment=self.run_config.environment)
+        if run_job_async:
+            results = await runner.run_async(self.run_config.parameters)
+        else:
+            results = runner.run(self.run_config.parameters)
         return results
 
     def _setup_and_check(self) -> Tuple[RunConfig, Optional[Results]]:
-
         self._prepare_to_run()
         self._check_if_remote_keys_ok()
 
@@ -539,12 +537,16 @@ class Jobs(Base):
         if self.run_config.parameters.n is None:
             return len(self)
         else:
-            len(self) * self.run_config.parameters.n
+            return len(self) * self.run_config.parameters.n
 
     def _run(self, config: RunConfig):
         "Shared code for run and run_async"
         if config.environment.cache is not None:
             self.run_config.environment.cache = config.environment.cache
+        if config.environment.jobs_runner_status is not None:
+            self.run_config.environment.jobs_runner_status = (
+                config.environment.jobs_runner_status
+            )
 
         if config.environment.bucket_collection is not None:
             self.run_config.environment.bucket_collection = (
@@ -646,20 +648,19 @@ class Jobs(Base):
         }
 
     def __len__(self) -> int:
-        """Return the maximum number of questions that will be asked while running this job.
-        Note that this is the maximum number of questions, not the actual number of questions that will be asked, as some questions may be skipped.
+        """Return the number of interviews that will be conducted for one iteration of this job.
+        An interview is the result of one survey, taken by one agent, with one model, with one scenario.
 
         >>> from edsl.jobs import Jobs
         >>> len(Jobs.example())
-        8
+        4
         """
-        number_of_questions = (
+        number_of_interviews = (
             len(self.agents or [1])
             * len(self.scenarios or [1])
             * len(self.models or [1])
-            * len(self.survey)
         )
-        return number_of_questions
+        return number_of_interviews
 
     def to_dict(self, add_edsl_version=True):
         d = {
@@ -810,9 +811,9 @@ def main():
     from edsl.data.Cache import Cache
 
     job = Jobs.example()
-    len(job) == 8
+    len(job) == 4
     results = job.run(cache=Cache())
-    len(results) == 8
+    len(results) == 4
     results
 
 

edsl/jobs/interviews/Interview.py

@@ -213,10 +213,6 @@ class Interview:
     async def async_conduct_interview(
         self,
         run_config: Optional["RunConfig"] = None,
-        #     model_buckets: Optional[ModelBuckets] = None,
-        #     stop_on_exception: bool = False,
-        #     raise_validation_errors: bool = True,
-        #     key_lookup: Optional[KeyLookup] = None,
     ) -> tuple["Answers", List[dict[str, Any]]]:
         """
         Conduct an Interview asynchronously.
@@ -313,7 +309,7 @@ class Interview:
 
         def handle_task(task, invigilator):
             try:
-                result = task.result()
+                result: Answers = task.result()
             except asyncio.CancelledError as e:  # task was cancelled
                 result = invigilator.get_failed_task_result(
                     failure_reason="Task was cancelled."

edsl/jobs/interviews/InterviewExceptionEntry.py

@@ -1,5 +1,6 @@
 import traceback
 import datetime
+from edsl.agents.InvigilatorBase import InvigilatorBase
 
 
 class InterviewExceptionEntry:
@@ -9,7 +10,7 @@ class InterviewExceptionEntry:
         self,
         *,
         exception: Exception,
-        invigilator: "Invigilator",
+        invigilator: "InvigilatorBase",
         traceback_format="text",
         answers=None,
     ):
@@ -20,6 +21,8 @@ class InterviewExceptionEntry:
         self.traceback_format = traceback_format
         self.answers = answers
 
+        # breakpoint()
+
     @property
     def question_type(self):
         # return self.failed_question.question.question_type
@@ -163,12 +166,16 @@ class InterviewExceptionEntry:
         >>> entry = InterviewExceptionEntry.example()
         >>> _ = entry.to_dict()
         """
-        return {
+        invigilator = (
+            self.invigilator.to_dict() if self.invigilator is not None else None
+        )
+        d = {
             "exception": self.serialize_exception(self.exception),
             "time": self.time,
             "traceback": self.traceback,
-            "invigilator": self.invigilator.to_dict(),
+            "invigilator": invigilator,
         }
+        return d
 
     @classmethod
     def from_dict(cls, data: dict) -> "InterviewExceptionEntry":
@@ -176,7 +183,10 @@ class InterviewExceptionEntry:
         from edsl.agents.Invigilator import InvigilatorAI
 
         exception = cls.deserialize_exception(data["exception"])
-        invigilator = InvigilatorAI.from_dict(data["invigilator"])
+        if data["invigilator"] is None:
+            invigilator = None
+        else:
+            invigilator = InvigilatorAI.from_dict(data["invigilator"])
         return cls(exception=exception, invigilator=invigilator)
 
 

edsl/jobs/runners/JobsRunnerAsyncio.py

@@ -44,7 +44,16 @@ class JobsRunnerAsyncio:
             data.append(result)
             task_history.add_interview(interview)
 
-        return Results(survey=self.jobs.survey, task_history=task_history, data=data)
+        results = Results(survey=self.jobs.survey, task_history=task_history, data=data)
+
+        relevant_cache = results.relevant_cache(self.environment.cache)
+
+        return Results(
+            survey=self.jobs.survey,
+            task_history=task_history,
+            data=data,
+            cache=relevant_cache,
+        )
 
     def simple_run(self):
         data = asyncio.run(self.run_async())
@@ -93,16 +102,16 @@ class JobsRunnerAsyncio:
 
             self.completed = True
 
-        def run_progress_bar(stop_event) -> None:
+        def run_progress_bar(stop_event, jobs_runner_status) -> None:
             """Runs the progress bar in a separate thread."""
-            self.jobs_runner_status.update_progress(stop_event)
+            jobs_runner_status.update_progress(stop_event)
 
         def set_up_progress_bar(progress_bar: bool, jobs_runner_status):
             progress_thread = None
             if progress_bar and jobs_runner_status.has_ep_api_key():
                 jobs_runner_status.setup()
                 progress_thread = threading.Thread(
-                    target=run_progress_bar, args=(stop_event,)
+                    target=run_progress_bar, args=(stop_event, jobs_runner_status)
                 )
                 progress_thread.start()
             elif progress_bar:
@@ -115,8 +124,9 @@ class JobsRunnerAsyncio:
             survey=self.jobs.survey,
             data=[],
             task_history=TaskHistory(),
-            cache=self.environment.cache.new_entries_cache(),
+            #           cache=self.environment.cache.new_entries_cache(),
         )
+
         stop_event = threading.Event()
         progress_thread = set_up_progress_bar(
             parameters.progress_bar, run_config.environment.jobs_runner_status
@@ -140,7 +150,9 @@ class JobsRunnerAsyncio:
             if exception_to_raise:
                 raise exception_to_raise
 
-            results.cache = self.environment.cache.new_entries_cache()
+            relevant_cache = results.relevant_cache(self.environment.cache)
+            results.cache = relevant_cache
+            # breakpoint()
             results.bucket_collection = self.environment.bucket_collection
 
             from edsl.jobs.results_exceptions_handler import ResultsExceptionsHandler

edsl/jobs/runners/JobsRunnerStatus.py

@@ -148,7 +148,8 @@ class JobsRunnerStatusBase(ABC):
         }
 
         model_queues = {}
-        for model, bucket in self.jobs_runner.bucket_collection.items():
+        # for model, bucket in self.jobs_runner.bucket_collection.items():
+        for model, bucket in self.jobs_runner.environment.bucket_collection.items():
             model_name = model.model
             model_queues[model_name] = {
                 "language_model_name": model_name,

edsl/language_models/key_management/KeyLookupBuilder.py

@@ -36,7 +36,7 @@ class KeyLookupBuilder:
 
     >>> builder = KeyLookupBuilder(fetch_order=("config", "env"))
     >>> builder.DEFAULT_RPM
-    10
+    100
     >>> builder.DEFAULT_TPM
     2000000
     >>> builder.fetch_order
@@ -54,8 +54,12 @@ class KeyLookupBuilder:
     ('openai', 'rpm')
     """
 
-    DEFAULT_RPM = 10
-    DEFAULT_TPM = 2000000
+    # DEFAULT_RPM = 10
+    # DEFAULT_TPM = 2000000
+    from edsl.config import CONFIG
+
+    DEFAULT_RPM = int(CONFIG.get("EDSL_SERVICE_RPM_BASELINE"))
+    DEFAULT_TPM = int(CONFIG.get("EDSL_SERVICE_TPM_BASELINE"))
 
     def __init__(self, fetch_order: Optional[tuple[str]] = None):
         if fetch_order is None:

edsl/questions/QuestionBudget.py

@@ -8,7 +8,7 @@ from edsl.questions.descriptors import IntegerDescriptor, QuestionOptionsDescrip
 from edsl.questions.response_validator_abc import ResponseValidatorABC
 
 
-class BudgewResponseValidator(ResponseValidatorABC):
+class BudgetResponseValidator(ResponseValidatorABC):
     valid_examples = []
 
     invalid_examples = []
@@ -64,7 +64,7 @@ class QuestionBudget(QuestionBase):
     budget_sum: int = IntegerDescriptor(none_allowed=False)
     question_options: list[str] = QuestionOptionsDescriptor(q_budget=True)
     _response_model = None
-    response_validator_class = BudgewResponseValidator
+    response_validator_class = BudgetResponseValidator
 
     def __init__(
         self,