edsl 0.1.35__py3-none-any.whl → 0.1.36__py3-none-any.whl

edsl/jobs/interviews/Interview.py

@@ -28,7 +28,7 @@ from edsl.jobs.interviews.InterviewExceptionCollection import (
     InterviewExceptionCollection,
 )
 
-from edsl.jobs.interviews.InterviewStatusMixin import InterviewStatusMixin
+# from edsl.jobs.interviews.InterviewStatusMixin import InterviewStatusMixin
 
 from edsl.surveys.base import EndOfSurvey
 from edsl.jobs.buckets.ModelBuckets import ModelBuckets
@@ -44,6 +44,10 @@ from edsl.agents.InvigilatorBase import InvigilatorBase
 
 from edsl.exceptions.language_models import LanguageModelNoResponseError
 
+from edsl.jobs.interviews.InterviewStatusLog import InterviewStatusLog
+from edsl.jobs.tokens.InterviewTokenUsage import InterviewTokenUsage
+from edsl.jobs.interviews.InterviewStatusDictionary import InterviewStatusDictionary
+
 
 from edsl import CONFIG
 
@@ -52,7 +56,7 @@ EDSL_BACKOFF_MAX_SEC = float(CONFIG.get("EDSL_BACKOFF_MAX_SEC"))
 EDSL_MAX_ATTEMPTS = int(CONFIG.get("EDSL_MAX_ATTEMPTS"))
 
 
-class Interview(InterviewStatusMixin):
+class Interview:
     """
     An 'interview' is one agent answering one survey, with one language model, for a given scenario.
 
@@ -100,9 +104,7 @@ class Interview(InterviewStatusMixin):
 
         """
         self.agent = agent
-        # what I would like to do
-        self.survey = copy.deepcopy(survey)  # survey  copy.deepcopy(survey)
-        # self.survey = survey
+        self.survey = copy.deepcopy(survey)
         self.scenario = scenario
         self.model = model
         self.debug = debug
@@ -113,8 +115,6 @@ class Interview(InterviewStatusMixin):
         ] = Answers()  # will get filled in as interview progresses
         self.sidecar_model = sidecar_model
 
-        # self.stop_on_exception = False
-
         # Trackers
         self.task_creators = TaskCreators()  # tracks the task creators
         self.exceptions = InterviewExceptionCollection()
@@ -131,14 +131,41 @@ class Interview(InterviewStatusMixin):
 
         self.failed_questions = []
 
+    @property
+    def has_exceptions(self) -> bool:
+        """Return True if there are exceptions."""
+        return len(self.exceptions) > 0
+
+    @property
+    def task_status_logs(self) -> InterviewStatusLog:
+        """Return the task status logs for the interview.
+
+        The keys are the question names; the values are the lists of status log changes for each task.
+        """
+        for task_creator in self.task_creators.values():
+            self._task_status_log_dict[
+                task_creator.question.question_name
+            ] = task_creator.status_log
+        return self._task_status_log_dict
+
+    @property
+    def token_usage(self) -> InterviewTokenUsage:
+        """Determine how many tokens were used for the interview."""
+        return self.task_creators.token_usage
+
+    @property
+    def interview_status(self) -> InterviewStatusDictionary:
+        """Return a dictionary mapping task status codes to counts."""
+        return self.task_creators.interview_status
+
     # region: Serialization
-    def _to_dict(self, include_exceptions=False) -> dict[str, Any]:
+    def _to_dict(self, include_exceptions=True) -> dict[str, Any]:
         """Return a dictionary representation of the Interview instance.
         This is just for hashing purposes.
 
         >>> i = Interview.example()
         >>> hash(i)
-        1646262796627658719
+        1217840301076717434
         """
         d = {
             "agent": self.agent._to_dict(),
@@ -150,11 +177,39 @@ class Interview(InterviewStatusMixin):
         }
         if include_exceptions:
             d["exceptions"] = self.exceptions.to_dict()
+        return d
+
+    @classmethod
+    def from_dict(cls, d: dict[str, Any]) -> "Interview":
+        """Return an Interview instance from a dictionary."""
+        agent = Agent.from_dict(d["agent"])
+        survey = Survey.from_dict(d["survey"])
+        scenario = Scenario.from_dict(d["scenario"])
+        model = LanguageModel.from_dict(d["model"])
+        iteration = d["iteration"]
+        interview = cls(
+            agent=agent,
+            survey=survey,
+            scenario=scenario,
+            model=model,
+            iteration=iteration,
+        )
+        if "exceptions" in d:
+            exceptions = InterviewExceptionCollection.from_dict(d["exceptions"])
+            interview.exceptions = exceptions
+        return interview
 
     def __hash__(self) -> int:
         from edsl.utilities.utilities import dict_hash
 
-        return dict_hash(self._to_dict())
+        return dict_hash(self._to_dict(include_exceptions=False))
+
+    def __eq__(self, other: "Interview") -> bool:
+        """
+        >>> from edsl.jobs.interviews.Interview import Interview; i = Interview.example(); d = i._to_dict(); i2 = Interview.from_dict(d); i == i2
+        True
+        """
+        return hash(self) == hash(other)
 
     # endregion
 

edsl/jobs/interviews/InterviewExceptionCollection.py

@@ -34,6 +34,15 @@ class InterviewExceptionCollection(UserDict):
         newdata = {k: [e.to_dict() for e in v] for k, v in self.data.items()}
         return newdata
 
+    @classmethod
+    def from_dict(cls, data: dict) -> "InterviewExceptionCollection":
+        """Create an InterviewExceptionCollection from a dictionary."""
+        collection = cls()
+        for question_name, entries in data.items():
+            for entry in entries:
+                collection.add(question_name, InterviewExceptionEntry.from_dict(entry))
+        return collection
+
     def _repr_html_(self) -> str:
         from edsl.utilities.utilities import data_to_html
 

edsl/jobs/interviews/InterviewExceptionEntry.py

@@ -1,8 +1,5 @@
 import traceback
 import datetime
-import time
-from collections import UserDict
-from edsl.jobs.FailedQuestion import FailedQuestion
 
 
 class InterviewExceptionEntry:
@@ -12,7 +9,6 @@ class InterviewExceptionEntry:
         self,
         *,
         exception: Exception,
-        # failed_question: FailedQuestion,
         invigilator: "Invigilator",
         traceback_format="text",
         answers=None,
@@ -137,22 +133,48 @@ class InterviewExceptionEntry:
         console.print(tb)
         return html_output.getvalue()
 
+    @staticmethod
+    def serialize_exception(exception: Exception) -> dict:
+        return {
+            "type": type(exception).__name__,
+            "message": str(exception),
+            "traceback": "".join(
+                traceback.format_exception(
+                    type(exception), exception, exception.__traceback__
+                )
+            ),
+        }
+
+    @staticmethod
+    def deserialize_exception(data: dict) -> Exception:
+        try:
+            exception_class = globals()[data["type"]]
+        except KeyError:
+            exception_class = Exception
+        return exception_class(data["message"])
+
     def to_dict(self) -> dict:
         """Return the exception as a dictionary.
 
         >>> entry = InterviewExceptionEntry.example()
-        >>> entry.to_dict()['exception']
-        ValueError()
-
+        >>> _ = entry.to_dict()
         """
         return {
-            "exception": self.exception,
+            "exception": self.serialize_exception(self.exception),
             "time": self.time,
             "traceback": self.traceback,
-            # "failed_question": self.failed_question.to_dict(),
             "invigilator": self.invigilator.to_dict(),
         }
 
+    @classmethod
+    def from_dict(cls, data: dict) -> "InterviewExceptionEntry":
+        """Create an InterviewExceptionEntry from a dictionary."""
+        from edsl.agents.Invigilator import InvigilatorAI
+
+        exception = cls.deserialize_exception(data["exception"])
+        invigilator = InvigilatorAI.from_dict(data["invigilator"])
+        return cls(exception=exception, invigilator=invigilator)
+
     def push(self):
         from edsl import Coop
 

edsl/jobs/runners/JobsRunnerAsyncio.py

@@ -1,18 +1,12 @@
 from __future__ import annotations
 import time
-import math
 import asyncio
-import functools
 import threading
 from typing import Coroutine, List, AsyncGenerator, Optional, Union, Generator
 from contextlib import contextmanager
 from collections import UserList
 
-from rich.live import Live
-from rich.console import Console
-
 from edsl.results.Results import Results
-from edsl import shared_globals
 from edsl.jobs.interviews.Interview import Interview
 from edsl.jobs.runners.JobsRunnerStatus import JobsRunnerStatus
 
@@ -48,8 +42,6 @@ class JobsRunnerAsyncio:
         self.bucket_collection: "BucketCollection" = jobs.bucket_collection
         self.total_interviews: List["Interview"] = []
 
-        # self.jobs_runner_status = JobsRunnerStatus(self, n=1)
-
     async def run_async_generator(
         self,
         cache: Cache,
@@ -181,6 +173,7 @@ class JobsRunnerAsyncio:
             ] = question_name_to_prompts[answer_key_name]["system_prompt"]
 
         raw_model_results_dictionary = {}
+        cache_used_dictionary = {}
         for result in valid_results:
             question_name = result.question_name
             raw_model_results_dictionary[
@@ -195,6 +188,7 @@ class JobsRunnerAsyncio:
                 else 1.0 / result.cost
             )
             raw_model_results_dictionary[question_name + "_one_usd_buys"] = one_use_buys
+            cache_used_dictionary[question_name] = result.cache_used
 
         result = Result(
             agent=interview.agent,
@@ -207,6 +201,7 @@ class JobsRunnerAsyncio:
             survey=interview.survey,
             generated_tokens=generated_tokens_dict,
             comments_dict=comments_dict,
+            cache_used_dict=cache_used_dictionary,
         )
         result.interview_hash = hash(interview)
 
@@ -225,17 +220,16 @@ class JobsRunnerAsyncio:
         }
         interview_hashes = list(interview_lookup.keys())
 
+        task_history = TaskHistory(self.total_interviews, include_traceback=False)
+
         results = Results(
             survey=self.jobs.survey,
             data=sorted(
                 raw_results, key=lambda x: interview_hashes.index(x.interview_hash)
             ),
+            task_history=task_history,
+            cache=cache,
         )
-        results.cache = cache
-        results.task_history = TaskHistory(
-            self.total_interviews, include_traceback=False
-        )
-        results.has_unfixed_exceptions = results.task_history.has_unfixed_exceptions
         results.bucket_collection = self.bucket_collection
 
         if results.has_unfixed_exceptions and print_exceptions:
@@ -263,6 +257,7 @@ class JobsRunnerAsyncio:
             except Exception as e:
                 print(e)
                 remote_logging = False
+
             if remote_logging:
                 filestore = HTMLFileStore(filepath)
                 coop_details = filestore.push(description="Error report")

edsl/jobs/tasks/QuestionTaskCreator.py

@@ -1,20 +1,16 @@
 import asyncio
 from typing import Callable, Union, List
 from collections import UserList, UserDict
-import time
 
 from edsl.jobs.buckets import ModelBuckets
 from edsl.exceptions import InterviewErrorPriorTaskCanceled
 
 from edsl.jobs.interviews.InterviewStatusDictionary import InterviewStatusDictionary
 from edsl.jobs.tasks.task_status_enum import TaskStatus, TaskStatusDescriptor
-
-# from edsl.jobs.tasks.task_management import TokensUsed
 from edsl.jobs.tasks.TaskStatusLog import TaskStatusLog
 from edsl.jobs.tokens.InterviewTokenUsage import InterviewTokenUsage
 from edsl.jobs.tokens.TokenUsage import TokenUsage
 from edsl.jobs.Answers import Answers
-
 from edsl.questions.QuestionBase import QuestionBase
 
 
@@ -130,7 +126,7 @@ class QuestionTaskCreator(UserList):
         await self.tokens_bucket.get_tokens(requested_tokens)
 
         if (estimated_wait_time := self.requests_bucket.wait_time(1)) > 0:
-            self.waiting = True
+            self.waiting = True  #  do we need this?
             self.task_status = TaskStatus.WAITING_FOR_REQUEST_CAPACITY
 
         await self.requests_bucket.get_tokens(1, cheat_bucket_capacity=True)

edsl/jobs/tasks/TaskHistory.py

@@ -8,7 +8,7 @@ from edsl.jobs.tasks.task_status_enum import TaskStatus
 
 
 class TaskHistory:
-    def __init__(self, interviews: List["Interview"], include_traceback=False):
+    def __init__(self, interviews: List["Interview"], include_traceback: bool = False):
         """
         The structure of a TaskHistory exception
 
@@ -25,6 +25,7 @@ class TaskHistory:
 
     @classmethod
     def example(cls):
+        """ """
         from edsl.jobs.interviews.Interview import Interview
 
         from edsl.jobs.Jobs import Jobs
@@ -38,6 +39,7 @@ class TaskHistory:
             skip_retry=True,
             cache=False,
             raise_validation_errors=True,
+            disable_remote_inference=True,
         )
 
         return cls(results.task_history.total_interviews)
@@ -72,14 +74,29 @@ class TaskHistory:
 
     def to_dict(self):
         """Return the TaskHistory as a dictionary."""
+        # return {
+        #     "exceptions": [
+        #         e.to_dict(include_traceback=self.include_traceback)
+        #         for e in self.exceptions
+        #     ],
+        #     "indices": self.indices,
+        # }
         return {
-            "exceptions": [
-                e.to_dict(include_traceback=self.include_traceback)
-                for e in self.exceptions
-            ],
-            "indices": self.indices,
+            "interviews": [i._to_dict() for i in self.total_interviews],
+            "include_traceback": self.include_traceback,
         }
 
+    @classmethod
+    def from_dict(cls, data: dict):
+        """Create a TaskHistory from a dictionary."""
+        if data is None:
+            return cls([], include_traceback=False)
+
+        from edsl.jobs.interviews.Interview import Interview
+
+        interviews = [Interview.from_dict(i) for i in data["interviews"]]
+        return cls(interviews, include_traceback=data["include_traceback"])
+
     @property
     def has_exceptions(self) -> bool:
         """Return True if there are any exceptions.
@@ -259,7 +276,6 @@ class TaskHistory:
                 question_type = interview.survey.get_question(
                     question_name
                 ).question_type
-                # breakpoint()
                 if (question_name, question_type) not in exceptions_by_question_name:
                     exceptions_by_question_name[(question_name, question_type)] = 0
                 exceptions_by_question_name[(question_name, question_type)] += len(

edsl/language_models/LanguageModel.py

@@ -348,6 +348,9 @@ class LanguageModel(
         """
         # parameters = dict({})
 
+        # this is the case when data is loaded from a dict after serialization
+        if "parameters" in passed_parameter_dict:
+            passed_parameter_dict = passed_parameter_dict["parameters"]
         return {
             parameter_name: passed_parameter_dict.get(parameter_name, default_value)
             for parameter_name, default_value in default_parameter_dict.items()

edsl/prompts/Prompt.py

@@ -17,25 +17,23 @@ class PreserveUndefined(Undefined):
 
 
 from edsl.exceptions.prompts import TemplateRenderError
-from edsl.prompts.prompt_config import (
-    C2A,
-    names_to_component_types,
-    ComponentTypes,
-    NEGATIVE_INFINITY,
-)
-from edsl.prompts.registry import RegisterPromptsMeta
+
+# from edsl.prompts.prompt_config import (
+#     C2A,
+#     names_to_component_types,
+#     ComponentTypes,
+#     NEGATIVE_INFINITY,
+# )
+# from edsl.prompts.registry import RegisterPromptsMeta
 from edsl.Base import PersistenceMixin, RichPrintingMixin
 
 MAX_NESTING = 100
 
 
-class PromptBase(
-    PersistenceMixin, RichPrintingMixin, ABC, metaclass=RegisterPromptsMeta
-):
+class Prompt(PersistenceMixin, RichPrintingMixin):
     """Class for creating a prompt to be used in a survey."""
 
     default_instructions: Optional[str] = "Do good things, friendly LLM!"
-    component_type = ComponentTypes.GENERIC
 
     def _repr_html_(self):
         """Return an HTML representation of the Prompt."""
@@ -111,12 +109,6 @@ class PromptBase(
         with open(folder_path.joinpath(file_name), "r") as f:
             text = f.read()
         return cls(text=text)
-        # Resolve the path to get the absolute path
-        # absolute_path = folder_path.resolve()
-        # env = Environment(loader=FileSystemLoader(absolute_path))
-        # template = env.get_template(file_name)
-        # rendered_text = template.render({})
-        # return cls(text=rendered_text)
 
     @property
     def text(self):
@@ -281,6 +273,7 @@ class PromptBase(
         try:
             previous_text = None
             for _ in range(MAX_NESTING):
+                # breakpoint()
                 rendered_text = env.from_string(text).render(
                     primary_replacement, **additional_replacements
                 )
@@ -323,9 +316,8 @@ class PromptBase(
         Prompt(text=\"""Hello, {{person}}\""")
 
         """
-        class_name = data["class_name"]
-        cls = RegisterPromptsMeta._registry.get(class_name, Prompt)
-        return cls(text=data["text"])
+        # class_name = data["class_name"]
+        return Prompt(text=data["text"])
 
     def rich_print(self):
         """Display an object as a table."""
@@ -346,27 +338,21 @@ class PromptBase(
         return cls(cls.default_instructions)
 
 
-class Prompt(PromptBase):
-    """A prompt to be used in a survey."""
-
-    component_type = ComponentTypes.GENERIC
-
-
 if __name__ == "__main__":
     print("Running doctests...")
     import doctest
 
     doctest.testmod()
 
-from edsl.prompts.library.question_multiple_choice import *
-from edsl.prompts.library.agent_instructions import *
-from edsl.prompts.library.agent_persona import *
-
-from edsl.prompts.library.question_budget import *
-from edsl.prompts.library.question_checkbox import *
-from edsl.prompts.library.question_freetext import *
-from edsl.prompts.library.question_linear_scale import *
-from edsl.prompts.library.question_numerical import *
-from edsl.prompts.library.question_rank import *
-from edsl.prompts.library.question_extract import *
-from edsl.prompts.library.question_list import *
+# from edsl.prompts.library.question_multiple_choice import *
+# from edsl.prompts.library.agent_instructions import *
+# from edsl.prompts.library.agent_persona import *
+
+# from edsl.prompts.library.question_budget import *
+# from edsl.prompts.library.question_checkbox import *
+# from edsl.prompts.library.question_freetext import *
+# from edsl.prompts.library.question_linear_scale import *
+# from edsl.prompts.library.question_numerical import *
+# from edsl.prompts.library.question_rank import *
+# from edsl.prompts.library.question_extract import *
+# from edsl.prompts.library.question_list import *

edsl/prompts/__init__.py

@@ -1,2 +1,2 @@
-from edsl.prompts.registry import get_classes
+# from edsl.prompts.registry import get_classes
 from edsl.prompts.Prompt import Prompt

edsl/questions/QuestionBasePromptsMixin.py

@@ -38,29 +38,29 @@ class QuestionBasePromptsMixin:
     #     ) as file:
     #         return file.read()
 
-    @classmethod
-    def applicable_prompts(
-        cls, model: Optional[str] = None
-    ) -> list[type["PromptBase"]]:
-        """Get the prompts that are applicable to the question type.
+    # @classmethod
+    # def applicable_prompts(
+    #     cls, model: Optional[str] = None
+    # ) -> list[type["PromptBase"]]:
+    #     """Get the prompts that are applicable to the question type.
 
-        :param model: The language model to use.
+    #     :param model: The language model to use.
 
-        >>> from edsl.questions import QuestionFreeText
-        >>> QuestionFreeText.applicable_prompts()
-        [<class 'edsl.prompts.library.question_freetext.FreeText'>]
+    #     >>> from edsl.questions import QuestionFreeText
+    #     >>> QuestionFreeText.applicable_prompts()
+    #     [<class 'edsl.prompts.library.question_freetext.FreeText'>]
 
-        :param model: The language model to use. If None, assumes does not matter.
+    #     :param model: The language model to use. If None, assumes does not matter.
 
-        """
-        from edsl.prompts.registry import get_classes as prompt_lookup
+    #     """
+    #     from edsl.prompts.registry import get_classes as prompt_lookup
 
-        applicable_prompts = prompt_lookup(
-            component_type="question_instructions",
-            question_type=cls.question_type,
-            model=model,
-        )
-        return applicable_prompts
+    #     applicable_prompts = prompt_lookup(
+    #         component_type="question_instructions",
+    #         question_type=cls.question_type,
+    #         model=model,
+    #     )
+    #     return applicable_prompts
 
     @property
     def model_instructions(self) -> dict:

edsl/questions/QuestionFunctional.py

@@ -50,6 +50,7 @@ class QuestionFunctional(QuestionBase):
         requires_loop: Optional[bool] = False,
         function_source_code: Optional[str] = None,
         function_name: Optional[str] = None,
+        unsafe: Optional[bool] = False,
     ):
         super().__init__()
         if func:
@@ -61,9 +62,12 @@ class QuestionFunctional(QuestionBase):
 
         self.requires_loop = requires_loop
 
-        self.func = create_restricted_function(
-            self.function_name, self.function_source_code
-        )
+        if unsafe:
+            self.func = func
+        else:
+            self.func = create_restricted_function(
+                self.function_name, self.function_source_code
+            )
 
         self.question_name = question_name
         self.question_text = question_text

edsl/questions/descriptors.py

@@ -54,32 +54,32 @@ class BaseDescriptor(ABC):
     def __set__(self, instance, value: Any) -> None:
         """Set the value of the attribute."""
         self.validate(value, instance)
-        from edsl.prompts.registry import get_classes
+        # from edsl.prompts.registry import get_classes
 
         instance.__dict__[self.name] = value
-        if self.name == "_instructions":
-            instructions = value
-            if value is not None:
-                instance.__dict__[self.name] = instructions
-                instance.set_instructions = True
-            else:
-                potential_prompt_classes = get_classes(
-                    question_type=instance.question_type
-                )
-                if len(potential_prompt_classes) > 0:
-                    instructions = potential_prompt_classes[0]().text
-                    instance.__dict__[self.name] = instructions
-                    instance.set_instructions = False
-                else:
-                    if not hasattr(instance, "default_instructions"):
-                        raise Exception(
-                            "No default instructions found and no matching prompts!"
-                        )
-                    instructions = instance.default_instructions
-                    instance.__dict__[self.name] = instructions
-                    instance.set_instructions = False
-
-            # instance.set_instructions = value != instance.default_instructions
+        # if self.name == "_instructions":
+        #     instructions = value
+        #     if value is not None:
+        #         instance.__dict__[self.name] = instructions
+        #         instance.set_instructions = True
+        #     else:
+        #         potential_prompt_classes = get_classes(
+        #             question_type=instance.question_type
+        #         )
+        #         if len(potential_prompt_classes) > 0:
+        #             instructions = potential_prompt_classes[0]().text
+        #             instance.__dict__[self.name] = instructions
+        #             instance.set_instructions = False
+        #         else:
+        #             if not hasattr(instance, "default_instructions"):
+        #                 raise Exception(
+        #                     "No default instructions found and no matching prompts!"
+        #                 )
+        #             instructions = instance.default_instructions
+        #             instance.__dict__[self.name] = instructions
+        #             instance.set_instructions = False
+
+        # instance.set_instructions = value != instance.default_instructions
 
     def __set_name__(self, owner, name: str) -> None:
         """Set the name of the attribute."""

edsl/results/Dataset.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 import random
+import json
 from collections import UserList
 from typing import Any, Union, Optional
 
@@ -110,6 +111,17 @@ class Dataset(UserList, ResultsExportMixin):
                 new_data.append(observation)
         return Dataset(new_data)
 
+    def to_json(self):
+        """Return a JSON representation of the dataset.
+
+        >>> d = Dataset([{'a.b':[1,2,3,4]}])
+        >>> d.to_json()
+        [{'a.b': [1, 2, 3, 4]}]
+        """
+        return json.loads(
+            json.dumps(self.data)
+        )  # janky but I want to make sure it's serializable & deserializable
+
     def _repr_html_(self) -> str:
         """Return an HTML representation of the dataset."""
         from edsl.utilities.utilities import data_to_html