edsl 0.1.40.dev2__py3-none-any.whl → 0.1.42__py3-none-any.whl

edsl/jobs/interviews/InterviewExceptionEntry.py

@@ -1,5 +1,6 @@
 import traceback
 import datetime
+from edsl.agents.InvigilatorBase import InvigilatorBase
 
 
 class InterviewExceptionEntry:
@@ -9,7 +10,7 @@ class InterviewExceptionEntry:
         self,
         *,
         exception: Exception,
-        invigilator: "Invigilator",
+        invigilator: "InvigilatorBase",
         traceback_format="text",
         answers=None,
     ):
@@ -20,6 +21,8 @@ class InterviewExceptionEntry:
         self.traceback_format = traceback_format
         self.answers = answers
 
+        # breakpoint()
+
     @property
     def question_type(self):
         # return self.failed_question.question.question_type
@@ -163,12 +166,16 @@ class InterviewExceptionEntry:
         >>> entry = InterviewExceptionEntry.example()
         >>> _ = entry.to_dict()
         """
-        return {
+        invigilator = (
+            self.invigilator.to_dict() if self.invigilator is not None else None
+        )
+        d = {
             "exception": self.serialize_exception(self.exception),
             "time": self.time,
             "traceback": self.traceback,
-            "invigilator": self.invigilator.to_dict(),
+            "invigilator": invigilator,
         }
+        return d
 
     @classmethod
     def from_dict(cls, data: dict) -> "InterviewExceptionEntry":
@@ -176,7 +183,10 @@ class InterviewExceptionEntry:
         from edsl.agents.Invigilator import InvigilatorAI
 
         exception = cls.deserialize_exception(data["exception"])
-        invigilator = InvigilatorAI.from_dict(data["invigilator"])
+        if data["invigilator"] is None:
+            invigilator = None
+        else:
+            invigilator = InvigilatorAI.from_dict(data["invigilator"])
         return cls(exception=exception, invigilator=invigilator)
 
 

edsl/jobs/loggers/HTMLTableJobLogger.py

@@ -9,7 +9,8 @@ from edsl.jobs.jobs_status_enums import JobsStatus
 class HTMLTableJobLogger(JobLogger):
     def __init__(self, verbose=True, theme="auto", **kwargs):
         super().__init__(verbose=verbose)
-        self.display_handle = display(HTML(""), display_id=True)
+        self.display_handle = display(HTML(""), display_id=True) if verbose else None
+        #self.display_handle = display(HTML(""), display_id=True)
         self.current_message = None
         self.log_id = str(uuid.uuid4())
         self.is_expanded = True
@@ -22,6 +23,9 @@ class HTMLTableJobLogger(JobLogger):
 
     def _init_css(self):
         """Initialize the CSS styles with enhanced theme support"""
+        if not self.verbose:
+            return None
+        
         css = """
         <style>
             /* Base theme variables */
@@ -217,6 +221,7 @@ class HTMLTableJobLogger(JobLogger):
             }});
         </script>
         """
+        
 
         display(HTML(css + init_script))
 

edsl/jobs/results_exceptions_handler.py

@@ -66,9 +66,7 @@ class ResultsExceptionsHandler:
 
     def _generate_error_message(self, indices) -> str:
         """Generate appropriate error message based on number of exceptions."""
-        msg = f"Exceptions were raised in {len(indices)} interviews.\n"
-        if len(indices) > 5:
-            msg += f"Exceptions were raised in the following interviews: {indices}.\n"
+        msg = f"Exceptions were raised.\n" 
         return msg
 
     def handle_exceptions(self) -> None:
@@ -84,7 +82,6 @@ class ResultsExceptionsHandler:
 
         # Generate HTML report
         filepath = self.results.task_history.html(
-            cta="Open report to see details.",
             open_in_browser=self.open_in_browser,
             return_link=True,
         )
@@ -92,7 +89,5 @@ class ResultsExceptionsHandler:
         # Handle remote logging if enabled
         if self.remote_logging:
             filestore = HTMLFileStore(filepath)
-            coop_details = filestore.push(description="Error report")
+            coop_details = filestore.push(description="Exceptions Report")
             print(coop_details)
-
-        print("Also see: https://docs.expectedparrot.com/en/latest/exceptions.html")

edsl/jobs/runners/JobsRunnerAsyncio.py

@@ -44,7 +44,16 @@ class JobsRunnerAsyncio:
             data.append(result)
             task_history.add_interview(interview)
 
-        return Results(survey=self.jobs.survey, task_history=task_history, data=data)
+        results = Results(survey=self.jobs.survey, task_history=task_history, data=data)
+
+        relevant_cache = results.relevant_cache(self.environment.cache)
+
+        return Results(
+            survey=self.jobs.survey,
+            task_history=task_history,
+            data=data,
+            cache=relevant_cache,
+        )
 
     def simple_run(self):
         data = asyncio.run(self.run_async())
@@ -93,16 +102,16 @@ class JobsRunnerAsyncio:
 
             self.completed = True
 
-        def run_progress_bar(stop_event) -> None:
+        def run_progress_bar(stop_event, jobs_runner_status) -> None:
             """Runs the progress bar in a separate thread."""
-            self.jobs_runner_status.update_progress(stop_event)
+            jobs_runner_status.update_progress(stop_event)
 
         def set_up_progress_bar(progress_bar: bool, jobs_runner_status):
             progress_thread = None
             if progress_bar and jobs_runner_status.has_ep_api_key():
                 jobs_runner_status.setup()
                 progress_thread = threading.Thread(
-                    target=run_progress_bar, args=(stop_event,)
+                    target=run_progress_bar, args=(stop_event, jobs_runner_status)
                 )
                 progress_thread.start()
             elif progress_bar:
@@ -115,8 +124,9 @@ class JobsRunnerAsyncio:
             survey=self.jobs.survey,
             data=[],
             task_history=TaskHistory(),
-            cache=self.environment.cache.new_entries_cache(),
+            #           cache=self.environment.cache.new_entries_cache(),
         )
+
         stop_event = threading.Event()
         progress_thread = set_up_progress_bar(
             parameters.progress_bar, run_config.environment.jobs_runner_status
@@ -140,7 +150,9 @@ class JobsRunnerAsyncio:
             if exception_to_raise:
                 raise exception_to_raise
 
-            results.cache = self.environment.cache.new_entries_cache()
+            relevant_cache = results.relevant_cache(self.environment.cache)
+            results.cache = relevant_cache
+            # breakpoint()
             results.bucket_collection = self.environment.bucket_collection
 
             from edsl.jobs.results_exceptions_handler import ResultsExceptionsHandler

edsl/jobs/runners/JobsRunnerStatus.py

@@ -148,7 +148,8 @@ class JobsRunnerStatusBase(ABC):
         }
 
         model_queues = {}
-        for model, bucket in self.jobs_runner.bucket_collection.items():
+        # for model, bucket in self.jobs_runner.bucket_collection.items():
+        for model, bucket in self.jobs_runner.environment.bucket_collection.items():
             model_name = model.model
             model_queues[model_name] = {
                 "language_model_name": model_name,

edsl/jobs/tasks/TaskHistory.py

@@ -264,9 +264,27 @@ class TaskHistory(RepresentationMixin):
         js = env.joinpath("report.js").read_text()
         return js
 
+    @property
+    def exceptions_table(self) -> dict:
+        """Return a dictionary of exceptions organized by type, service, model, and question name."""
+        exceptions_table = {}
+        for interview in self.total_interviews:
+            for question_name, exceptions in interview.exceptions.items():
+                for exception in exceptions:
+                    key = (
+                        exception.exception.__class__.__name__,  # Exception type
+                        interview.model._inference_service_,    # Service
+                        interview.model.model,                  # Model
+                        question_name                           # Question name
+                    )
+                    if key not in exceptions_table:
+                        exceptions_table[key] = 0
+                    exceptions_table[key] += 1
+        return exceptions_table
+    
     @property
     def exceptions_by_type(self) -> dict:
-        """Return a dictionary of exceptions by type."""
+        """Return a dictionary of exceptions tallied by type."""
         exceptions_by_type = {}
         for interview in self.total_interviews:
             for question_name, exceptions in interview.exceptions.items():
@@ -324,6 +342,27 @@ class TaskHistory(RepresentationMixin):
         }
         return sorted_exceptions_by_question_name
 
+    # @property
+    # def exceptions_by_model(self) -> dict:
+    #     """Return a dictionary of exceptions tallied by model and question name."""
+    #     exceptions_by_model = {}
+    #     for interview in self.total_interviews:
+    #         model = interview.model.model
+    #         service = interview.model._inference_service_
+    #         if (service, model) not in exceptions_by_model:
+    #             exceptions_by_model[(service, model)] = 0
+    #         if interview.exceptions != {}:
+    #             exceptions_by_model[(service, model)] += len(interview.exceptions)
+
+    #     # sort the exceptions by model
+    #     sorted_exceptions_by_model = {
+    #         k: v
+    #         for k, v in sorted(
+    #             exceptions_by_model.items(), key=lambda item: item[1], reverse=True
+    #         )
+    #     }
+    #     return sorted_exceptions_by_model
+
     @property
     def exceptions_by_model(self) -> dict:
         """Return a dictionary of exceptions tallied by model and question name."""
@@ -331,19 +370,12 @@ class TaskHistory(RepresentationMixin):
         for interview in self.total_interviews:
             model = interview.model.model
             service = interview.model._inference_service_
-            if (service, model) not in exceptions_by_model:
-                exceptions_by_model[(service, model)] = 0
-            if interview.exceptions != {}:
-                exceptions_by_model[(service, model)] += len(interview.exceptions)
-
-        # sort the exceptions by model
-        sorted_exceptions_by_model = {
-            k: v
-            for k, v in sorted(
-                exceptions_by_model.items(), key=lambda item: item[1], reverse=True
-            )
-        }
-        return sorted_exceptions_by_model
+            for question_name, exceptions in interview.exceptions.items():
+                key = (service, model, question_name)
+                if key not in exceptions_by_model:
+                    exceptions_by_model[key] = 0
+                exceptions_by_model[key] += len(exceptions)
+        return exceptions_by_model
 
     def generate_html_report(self, css: Optional[str], include_plot=False):
         if include_plot:
@@ -372,6 +404,7 @@ class TaskHistory(RepresentationMixin):
             javascript=self.javascript(),
             num_exceptions=len(self.exceptions),
             performance_plot_html=performance_plot_html,
+            exceptions_table=self.exceptions_table,
             exceptions_by_type=self.exceptions_by_type,
             exceptions_by_question_name=self.exceptions_by_question_name,
             exceptions_by_model=self.exceptions_by_model,
@@ -386,11 +419,10 @@ class TaskHistory(RepresentationMixin):
         filename: Optional[str] = None,
         return_link=False,
         css=None,
-        cta="Open Report in New Tab",
+        cta="\nClick to open the report in a new tab\n",
         open_in_browser=False,
     ):
         """Return an HTML report."""
-
         from IPython.display import display, HTML
         import tempfile
         import os
@@ -419,7 +451,7 @@ class TaskHistory(RepresentationMixin):
             html_link = f'<a href="{html_url}" target="_blank">{cta}</a>'
             display(HTML(html_link))
             escaped_output = html.escape(output)
-            iframe = f""""
+            iframe = f"""
             <iframe srcdoc="{ escaped_output }" style="width: 800px; height: 600px;"></iframe>
             """
             display(HTML(iframe))

edsl/language_models/LanguageModel.py

@@ -244,7 +244,7 @@ class LanguageModel(
 
         >>> m = LanguageModel.example()
         >>> hash(m)
-        1811901442659237949
+        325654563661254408
         """
         from edsl.utilities.utilities import dict_hash
 
@@ -495,11 +495,12 @@ class LanguageModel(
 
         >>> m = LanguageModel.example()
         >>> m.to_dict()
-        {'model': '...', 'parameters': {'temperature': ..., 'max_tokens': ..., 'top_p': ..., 'frequency_penalty': ..., 'presence_penalty': ..., 'logprobs': False, 'top_logprobs': ...}, 'edsl_version': '...', 'edsl_class_name': 'LanguageModel'}
+        {'model': '...', 'parameters': {'temperature': ..., 'max_tokens': ..., 'top_p': ..., 'frequency_penalty': ..., 'presence_penalty': ..., 'logprobs': False, 'top_logprobs': ...}, 'inference_service': 'openai', 'edsl_version': '...', 'edsl_class_name': 'LanguageModel'}
         """
         d = {
             "model": self.model,
             "parameters": self.parameters,
+            "inference_service": self._inference_service_,
         }
         if add_edsl_version:
             from edsl import __version__
@@ -511,7 +512,10 @@ class LanguageModel(
     @classmethod
     @remove_edsl_version
     def from_dict(cls, data: dict) -> Type[LanguageModel]:
-        """Convert dictionary to a LanguageModel child instance."""
+        """Convert dictionary to a LanguageModel child instance.
+
+        NB: This method does not use the stores inference_service but rather just fetches a model class based on the name.
+        """
         from edsl.language_models.model import get_model_class
 
         model_class = get_model_class(data["model"])
@@ -558,7 +562,6 @@ class LanguageModel(
         >>> m = LanguageModel.example(test_model = True, canned_response = "WOWZA!", throw_exception = True)
         >>> r = q.by(m).run(cache = False, disable_remote_cache = True, disable_remote_inference = True, print_exceptions = True)
         Exception report saved to ...
-        Also see: ...
         """
         from edsl.language_models.model import Model
 

edsl/language_models/ModelList.py

@@ -60,7 +60,7 @@ class ModelList(Base, UserList):
 
         sl = ScenarioList()
         for model in self:
-            d = {"model": model.model}
+            d = {"model": model.model, "inference_service": model._inference_service_}
             d.update(model.parameters)
             sl.append(Scenario(d))
         return sl

edsl/language_models/key_management/KeyLookupBuilder.py

@@ -36,7 +36,7 @@ class KeyLookupBuilder:
 
     >>> builder = KeyLookupBuilder(fetch_order=("config", "env"))
     >>> builder.DEFAULT_RPM
-    10
+    100
     >>> builder.DEFAULT_TPM
     2000000
     >>> builder.fetch_order
@@ -54,8 +54,12 @@ class KeyLookupBuilder:
     ('openai', 'rpm')
     """
 
-    DEFAULT_RPM = 10
-    DEFAULT_TPM = 2000000
+    # DEFAULT_RPM = 10
+    # DEFAULT_TPM = 2000000
+    from edsl.config import CONFIG
+
+    DEFAULT_RPM = int(CONFIG.get("EDSL_SERVICE_RPM_BASELINE"))
+    DEFAULT_TPM = int(CONFIG.get("EDSL_SERVICE_TPM_BASELINE"))
 
     def __init__(self, fetch_order: Optional[tuple[str]] = None):
         if fetch_order is None:

edsl/language_models/model.py

@@ -233,6 +233,55 @@ class Model(metaclass=Meta):
             print("OK!")
             print("\n")
 
+    @classmethod
+    def check_working_models(
+        cls,
+        service: Optional[str] = None,
+        works_with_text: Optional[bool] = None,
+        works_with_images: Optional[bool] = None,
+    ) -> list[dict]:
+        from edsl.coop import Coop
+
+        c = Coop()
+        working_models = c.fetch_working_models()
+
+        if service is not None:
+            working_models = [m for m in working_models if m["service"] == service]
+        if works_with_text is not None:
+            working_models = [
+                m for m in working_models if m["works_with_text"] == works_with_text
+            ]
+        if works_with_images is not None:
+            working_models = [
+                m for m in working_models if m["works_with_images"] == works_with_images
+            ]
+
+        if len(working_models) == 0:
+            return []
+
+        else:
+            return PrettyList(
+                [
+                    [
+                        m["service"],
+                        m["model"],
+                        m["works_with_text"],
+                        m["works_with_images"],
+                        m["usd_per_1M_input_tokens"],
+                        m["usd_per_1M_output_tokens"],
+                    ]
+                    for m in working_models
+                ],
+                columns=[
+                    "Service",
+                    "Model",
+                    "Works with text",
+                    "Works with images",
+                    "Price per 1M input tokens (USD)",
+                    "Price per 1M output tokens (USD)",
+                ],
+            )
+
     @classmethod
     def example(cls, randomize: bool = False) -> "Model":
         """

edsl/questions/QuestionBudget.py

@@ -8,7 +8,7 @@ from edsl.questions.descriptors import IntegerDescriptor, QuestionOptionsDescrip
 from edsl.questions.response_validator_abc import ResponseValidatorABC
 
 
-class BudgewResponseValidator(ResponseValidatorABC):
+class BudgetResponseValidator(ResponseValidatorABC):
     valid_examples = []
 
     invalid_examples = []
@@ -64,7 +64,7 @@ class QuestionBudget(QuestionBase):
     budget_sum: int = IntegerDescriptor(none_allowed=False)
     question_options: list[str] = QuestionOptionsDescriptor(q_budget=True)
     _response_model = None
-    response_validator_class = BudgewResponseValidator
+    response_validator_class = BudgetResponseValidator
 
     def __init__(
         self,