edsl 0.1.41__py3-none-any.whl → 0.1.43__py3-none-any.whl

edsl/inference_services/AvailableModelFetcher.py

@@ -136,7 +136,10 @@ class AvailableModelFetcher:
         if not service_models:
             import warnings
 
-            warnings.warn(f"No models found for service {service_name}")
+            with warnings.catch_warnings():
+                warnings.simplefilter("ignore")  # Ignores the warning
+                warnings.warn(f"No models found for service {service_name}")
+
             return [], service_name
 
         models_list = AvailableModels(

edsl/inference_services/DeepSeekService.py

@@ -0,0 +1,18 @@
+import aiohttp
+import json
+import requests
+from typing import Any, List
+
+# from edsl.inference_services.InferenceServiceABC import InferenceServiceABC
+from edsl.language_models import LanguageModel
+
+from edsl.inference_services.OpenAIService import OpenAIService
+
+
+class DeepSeekService(OpenAIService):
+    """DeepInfra service class."""
+
+    _inference_service_ = "deepseek"
+    _env_key_name_ = "DEEPSEEK_API_KEY"
+    _base_url_ = "https://api.deepseek.com"
+    _models_list_cache: List[str] = []

edsl/inference_services/registry.py

@@ -13,6 +13,7 @@ from edsl.inference_services.OllamaService import OllamaService
 from edsl.inference_services.TestService import TestService
 from edsl.inference_services.TogetherAIService import TogetherAIService
 from edsl.inference_services.PerplexityService import PerplexityService
+from edsl.inference_services.DeepSeekService import DeepSeekService
 
 try:
     from edsl.inference_services.MistralAIService import MistralAIService
@@ -33,6 +34,7 @@ services = [
     TestService,
     TogetherAIService,
     PerplexityService,
+    DeepSeekService,
 ]
 
 if mistral_available:

edsl/jobs/Jobs.py

@@ -38,6 +38,7 @@ if TYPE_CHECKING:
     from edsl.language_models.ModelList import ModelList
     from edsl.data.Cache import Cache
     from edsl.language_models.key_management.KeyLookup import KeyLookup
+    from edsl.jobs.JobsRemoteInferenceHandler import JobsRemoteInferenceHandler
 
 VisibilityType = Literal["private", "public", "unlisted"]
 
@@ -407,7 +408,13 @@ class Jobs(Base):
         >>> bc
         BucketCollection(...)
         """
-        return BucketCollection.from_models(self.models)
+        bc = BucketCollection.from_models(self.models)
+        
+        if self.run_config.environment.key_lookup is not None:
+            bc.update_from_key_lookup(
+                self.run_config.environment.key_lookup
+            )
+        return bc
 
     def html(self):
         """Return the HTML representations for each scenario"""
@@ -465,22 +472,47 @@ class Jobs(Base):
 
         return False
 
+    def _start_remote_inference_job(
+        self, job_handler: Optional[JobsRemoteInferenceHandler] = None
+    ) -> Union["Results", None]:
+
+        if job_handler is None:
+            job_handler = self._create_remote_inference_handler()
+            
+        job_info = job_handler.create_remote_inference_job(
+                iterations=self.run_config.parameters.n,
+                remote_inference_description=self.run_config.parameters.remote_inference_description,
+                remote_inference_results_visibility=self.run_config.parameters.remote_inference_results_visibility,
+        )
+        return job_info
+    
+    def _create_remote_inference_handler(self) -> JobsRemoteInferenceHandler:
+
+        from edsl.jobs.JobsRemoteInferenceHandler import JobsRemoteInferenceHandler
+        
+        return JobsRemoteInferenceHandler(
+            self, verbose=self.run_config.parameters.verbose
+        )
+        
     def _remote_results(
         self,
+        config: RunConfig,
     ) -> Union["Results", None]:
         from edsl.jobs.JobsRemoteInferenceHandler import JobsRemoteInferenceHandler
+        from edsl.jobs.JobsRemoteInferenceHandler import RemoteJobInfo
 
-        jh = JobsRemoteInferenceHandler(
-            self, verbose=self.run_config.parameters.verbose
-        )
+        background = config.parameters.background
+
+        jh = self._create_remote_inference_handler()
         if jh.use_remote_inference(self.run_config.parameters.disable_remote_inference):
-            job_info = jh.create_remote_inference_job(
-                iterations=self.run_config.parameters.n,
-                remote_inference_description=self.run_config.parameters.remote_inference_description,
-                remote_inference_results_visibility=self.run_config.parameters.remote_inference_results_visibility,
-            )
-            results = jh.poll_remote_inference_job(job_info)
-            return results
+            job_info: RemoteJobInfo = self._start_remote_inference_job(jh)
+            if background:
+                from edsl.results.Results import Results 
+                results = Results.from_job_info(job_info)
+                return results
+            else:
+                results = jh.poll_remote_inference_job(job_info)
+                return results
         else:
             return None
 
@@ -507,13 +539,6 @@ class Jobs(Base):
 
         assert isinstance(self.run_config.environment.cache, Cache)
 
-        # with RemoteCacheSync(
-        #     coop=Coop(),
-        #     cache=self.run_config.environment.cache,
-        #     output_func=self._output,
-        #     remote_cache=use_remote_cache,
-        #     remote_cache_description=self.run_config.parameters.remote_cache_description,
-        # ):
         runner = JobsRunnerAsyncio(self, environment=self.run_config.environment)
         if run_job_async:
             results = await runner.run_async(self.run_config.parameters)
@@ -521,17 +546,6 @@ class Jobs(Base):
             results = runner.run(self.run_config.parameters)
         return results
 
-    def _setup_and_check(self) -> Tuple[RunConfig, Optional[Results]]:
-        self._prepare_to_run()
-        self._check_if_remote_keys_ok()
-
-        # first try to run the job remotely
-        if results := self._remote_results():
-            return results
-
-        self._check_if_local_keys_ok()
-        return None
-
     @property
     def num_interviews(self):
         if self.run_config.parameters.n is None:
@@ -539,7 +553,7 @@ class Jobs(Base):
         else:
             return len(self) * self.run_config.parameters.n
 
-    def _run(self, config: RunConfig):
+    def _run(self, config: RunConfig) -> Union[None, "Results"]:
         "Shared code for run and run_async"
         if config.environment.cache is not None:
             self.run_config.environment.cache = config.environment.cache
@@ -561,7 +575,6 @@ class Jobs(Base):
 
         self.replace_missing_objects()
 
-        # try to run remotely first
         self._prepare_to_run()
         self._check_if_remote_keys_ok()
 
@@ -579,9 +592,9 @@ class Jobs(Base):
             self.run_config.environment.cache = Cache(immediate_write=False)
 
         # first try to run the job remotely
-        if results := self._remote_results():
+        if (results := self._remote_results(config)) is not None:
             return results
-
+        
         self._check_if_local_keys_ok()
 
         if config.environment.bucket_collection is None:
@@ -589,6 +602,16 @@ class Jobs(Base):
                 self.create_bucket_collection()
             )
 
+        if (
+            self.run_config.environment.key_lookup is not None
+            and self.run_config.environment.bucket_collection is not None
+        ):
+            self.run_config.environment.bucket_collection.update_from_key_lookup(
+                self.run_config.environment.key_lookup
+            )
+
+        return None
+
     @with_config
     def run(self, *, config: RunConfig) -> "Results":
         """
@@ -608,7 +631,10 @@ class Jobs(Base):
         :param bucket_collection: A BucketCollection object to track API calls
         :param key_lookup: A KeyLookup object to manage API keys
         """
-        self._run(config)
+        potentially_completed_results = self._run(config)
+
+        if potentially_completed_results is not None:
+            return potentially_completed_results
 
         return asyncio.run(self._execute_with_remote_cache(run_job_async=False))
 

edsl/jobs/JobsPrompts.py

@@ -1,3 +1,5 @@
+import time
+import logging
 from typing import List, TYPE_CHECKING
 
 from edsl.results.Dataset import Dataset
@@ -12,7 +14,9 @@ if TYPE_CHECKING:
     # from edsl.surveys.Survey import Survey
 
 from edsl.jobs.FetchInvigilator import FetchInvigilator
+from edsl.data.CacheEntry import CacheEntry
 
+logger = logging.getLogger(__name__)
 
 class JobsPrompts:
     def __init__(self, jobs: "Jobs"):
@@ -21,6 +25,8 @@ class JobsPrompts:
         self.scenarios = jobs.scenarios
         self.survey = jobs.survey
         self._price_lookup = None
+        self._agent_lookup = {agent: idx for idx, agent in enumerate(self.agents)}
+        self._scenario_lookup = {scenario: idx for idx, scenario in enumerate(self.scenarios)}
 
     @property
     def price_lookup(self):
@@ -47,26 +53,54 @@ class JobsPrompts:
         agent_indices = []
         models = []
         costs = []
-
+        cache_keys = []
+        
         for interview_index, interview in enumerate(interviews):
+            logger.info(f"Processing interview {interview_index} of {len(interviews)}")
+            interview_start = time.time()
+            
+            # Fetch invigilators timing
+            invig_start = time.time()
             invigilators = [
                 FetchInvigilator(interview)(question)
                 for question in interview.survey.questions
             ]
+            invig_end = time.time()
+            logger.debug(f"Time taken to fetch invigilators: {invig_end - invig_start:.4f}s")
+            
+            # Process prompts timing
+            prompts_start = time.time()
             for _, invigilator in enumerate(invigilators):
+                # Get prompts timing
+                get_prompts_start = time.time()
                 prompts = invigilator.get_prompts()
+                get_prompts_end = time.time()
+                logger.debug(f"Time taken to get prompts: {get_prompts_end - get_prompts_start:.4f}s")
+                
                 user_prompt = prompts["user_prompt"]
                 system_prompt = prompts["system_prompt"]
                 user_prompts.append(user_prompt)
                 system_prompts.append(system_prompt)
-                agent_index = self.agents.index(invigilator.agent)
+                
+                # Index lookups timing
+                index_start = time.time()
+                agent_index = self._agent_lookup[invigilator.agent]
                 agent_indices.append(agent_index)
                 interview_indices.append(interview_index)
-                scenario_index = self.scenarios.index(invigilator.scenario)
+                scenario_index = self._scenario_lookup[invigilator.scenario]
                 scenario_indices.append(scenario_index)
+                index_end = time.time()
+                logger.debug(f"Time taken for index lookups: {index_end - index_start:.4f}s")
+                
+                # Model and question name assignment timing
+                assign_start = time.time()
                 models.append(invigilator.model.model)
                 question_names.append(invigilator.question.question_name)
+                assign_end = time.time()
+                logger.debug(f"Time taken for assignments: {assign_end - assign_start:.4f}s")
 
+                # Cost estimation timing
+                cost_start = time.time()
                 prompt_cost = self.estimate_prompt_cost(
                     system_prompt=system_prompt,
                     user_prompt=user_prompt,
@@ -74,8 +108,34 @@ class JobsPrompts:
                     inference_service=invigilator.model._inference_service_,
                     model=invigilator.model.model,
                 )
+                cost_end = time.time()
+                logger.debug(f"Time taken to estimate prompt cost: {cost_end - cost_start:.4f}s")
                 costs.append(prompt_cost["cost_usd"])
 
+                # Cache key generation timing
+                cache_key_gen_start = time.time()
+                cache_key = CacheEntry.gen_key(
+                    model=invigilator.model.model,
+                    parameters=invigilator.model.parameters,
+                    system_prompt=system_prompt,
+                    user_prompt=user_prompt,
+                    iteration=0,
+                )
+                cache_key_gen_end = time.time()
+                cache_keys.append(cache_key)
+                logger.debug(f"Time taken to generate cache key: {cache_key_gen_end - cache_key_gen_start:.4f}s")
+                logger.debug("-" * 50)  # Separator between iterations
+
+            prompts_end = time.time()
+            logger.info(f"Time taken to process prompts: {prompts_end - prompts_start:.4f}s")
+            
+            interview_end = time.time()
+            logger.info(f"Overall time taken for interview: {interview_end - interview_start:.4f}s")
+            logger.info("Time breakdown:")
+            logger.info(f"  Invigilators: {invig_end - invig_start:.4f}s")
+            logger.info(f"  Prompts processing: {prompts_end - prompts_start:.4f}s")
+            logger.info(f"  Other overhead: {(interview_end - interview_start) - ((invig_end - invig_start) + (prompts_end - prompts_start)):.4f}s")
+
         d = Dataset(
             [
                 {"user_prompt": user_prompts},
@@ -86,6 +146,7 @@ class JobsPrompts:
                 {"agent_index": agent_indices},
                 {"model": models},
                 {"estimated_cost": costs},
+                {"cache_key": cache_keys},
             ]
         )
         return d

edsl/jobs/JobsRemoteInferenceHandler.py

@@ -219,6 +219,7 @@ class JobsRemoteInferenceHandler:
         job_info.logger.add_info("results_uuid", results_uuid)
         results = object_fetcher(results_uuid, expected_object_type="results")
         results_url = remote_job_data.get("results_url")
+        job_info.logger.add_info("results_url", results_url)
         job_info.logger.update(
             f"Job completed and Results stored on Coop: {results_url}",
             status=JobsStatus.COMPLETED,
@@ -227,6 +228,40 @@ class JobsRemoteInferenceHandler:
         results.results_uuid = results_uuid
         return results
 
+    def _attempt_fetch_job(
+        self,
+        job_info: RemoteJobInfo,
+        remote_job_data_fetcher: Callable,
+        object_fetcher: Callable,
+    ) -> Union[None, "Results", Literal["continue"]]:
+        """Makes one attempt to fetch and process a remote job's status and results."""
+        remote_job_data = remote_job_data_fetcher(job_info.job_uuid)
+        status = remote_job_data.get("status")
+
+        if status == "cancelled":
+            self._handle_cancelled_job(job_info)
+            return None
+
+        elif status == "failed" or status == "completed":
+            if status == "failed":
+                self._handle_failed_job(job_info, remote_job_data)
+
+            results_uuid = remote_job_data.get("results_uuid")
+            if results_uuid:
+                results = self._fetch_results_and_log(
+                    job_info=job_info,
+                    results_uuid=results_uuid,
+                    remote_job_data=remote_job_data,
+                    object_fetcher=object_fetcher,
+                )
+                return results
+            else:
+                return None
+
+        else:
+            self._sleep_for_a_bit(job_info, status)
+            return "continue"
+
     def poll_remote_inference_job(
         self,
         job_info: RemoteJobInfo,
@@ -241,31 +276,13 @@ class JobsRemoteInferenceHandler:
 
         job_in_queue = True
         while job_in_queue:
-            remote_job_data = remote_job_data_fetcher(job_info.job_uuid)
-            status = remote_job_data.get("status")
-
-            if status == "cancelled":
-                self._handle_cancelled_job(job_info)
-                return None
-
-            elif status == "failed" or status == "completed":
-                if status == "failed":
-                    self._handle_failed_job(job_info, remote_job_data)
-
-                results_uuid = remote_job_data.get("results_uuid")
-                if results_uuid:
-                    results = self._fetch_results_and_log(
-                        job_info=job_info,
-                        results_uuid=results_uuid,
-                        remote_job_data=remote_job_data,
-                        object_fetcher=object_fetcher,
-                    )
-                    return results
-                else:
-                    return None
-
-            else:
-                self._sleep_for_a_bit(job_info, status)
+            result = self._attempt_fetch_job(
+                job_info,
+                remote_job_data_fetcher,
+                object_fetcher
+            )
+            if result != "continue":
+                return result
 
     async def create_and_poll_remote_job(
         self,

edsl/jobs/JobsRemoteInferenceLogger.py

@@ -32,7 +32,7 @@ class JobsInfo:
     pretty_names = {
         "job_uuid": "Job UUID",
         "progress_bar_url": "Progress Bar URL",
-        "error_report_url": "Error Report URL",
+        "error_report_url": "Exceptions Report URL",
         "results_uuid": "Results UUID",
         "results_url": "Results URL",
     }

edsl/jobs/buckets/BucketCollection.py

@@ -96,6 +96,36 @@ class BucketCollection(UserDict):
         else:
             self[model] = self.services_to_buckets[self.models_to_services[model.model]]
 
+    def update_from_key_lookup(self, key_lookup: "KeyLookup") -> None:
+        """Updates the bucket collection rates based on model RPM/TPM from KeyLookup"""
+
+        for model_name, service in self.models_to_services.items():
+            if service in key_lookup and not self.infinity_buckets:
+
+                if key_lookup[service].rpm is not None:
+                    new_rps = key_lookup[service].rpm / 60.0
+                    new_requests_bucket = TokenBucket(
+                        bucket_name=service,
+                        bucket_type="requests",
+                        capacity=new_rps,
+                        refill_rate=new_rps,
+                        remote_url=self.remote_url,
+                    )
+                    self.services_to_buckets[service].requests_bucket = (
+                        new_requests_bucket
+                    )
+
+                if key_lookup[service].tpm is not None:
+                    new_tps = key_lookup[service].tpm / 60.0
+                    new_tokens_bucket = TokenBucket(
+                        bucket_name=service,
+                        bucket_type="tokens",
+                        capacity=new_tps,
+                        refill_rate=new_tps,
+                        remote_url=self.remote_url,
+                    )
+                    self.services_to_buckets[service].tokens_bucket = new_tokens_bucket
+
     def visualize(self) -> dict:
         """Visualize the token and request buckets for each model."""
         plots = {}

edsl/jobs/data_structures.py

@@ -32,6 +32,7 @@ class RunParameters(Base):
     remote_inference_results_visibility: Optional[VisibilityType] = "unlisted"
     skip_retry: bool = False
     raise_validation_errors: bool = False
+    background: bool = False
     disable_remote_cache: bool = False
     disable_remote_inference: bool = False
     job_uuid: Optional[str] = None

edsl/jobs/interviews/Interview.py

@@ -153,7 +153,7 @@ class Interview:
 
         >>> i = Interview.example()
         >>> hash(i)
-        193593189022259693
+        767745459362662063
         """
         d = {
             "agent": self.agent.to_dict(add_edsl_version=add_edsl_version),

edsl/jobs/loggers/HTMLTableJobLogger.py

@@ -9,7 +9,8 @@ from edsl.jobs.jobs_status_enums import JobsStatus
 class HTMLTableJobLogger(JobLogger):
     def __init__(self, verbose=True, theme="auto", **kwargs):
         super().__init__(verbose=verbose)
-        self.display_handle = display(HTML(""), display_id=True)
+        self.display_handle = display(HTML(""), display_id=True) if verbose else None
+        #self.display_handle = display(HTML(""), display_id=True)
         self.current_message = None
         self.log_id = str(uuid.uuid4())
         self.is_expanded = True
@@ -22,6 +23,9 @@ class HTMLTableJobLogger(JobLogger):
 
     def _init_css(self):
         """Initialize the CSS styles with enhanced theme support"""
+        if not self.verbose:
+            return None
+        
         css = """
         <style>
             /* Base theme variables */
@@ -217,6 +221,7 @@ class HTMLTableJobLogger(JobLogger):
             }});
         </script>
         """
+        
 
         display(HTML(css + init_script))
 

edsl/jobs/results_exceptions_handler.py

@@ -66,9 +66,7 @@ class ResultsExceptionsHandler:
 
     def _generate_error_message(self, indices) -> str:
         """Generate appropriate error message based on number of exceptions."""
-        msg = f"Exceptions were raised in {len(indices)} interviews.\n"
-        if len(indices) > 5:
-            msg += f"Exceptions were raised in the following interviews: {indices}.\n"
+        msg = f"Exceptions were raised.\n" 
         return msg
 
     def handle_exceptions(self) -> None:
@@ -84,7 +82,6 @@ class ResultsExceptionsHandler:
 
         # Generate HTML report
         filepath = self.results.task_history.html(
-            cta="Open report to see details.",
             open_in_browser=self.open_in_browser,
             return_link=True,
         )
@@ -92,7 +89,5 @@ class ResultsExceptionsHandler:
         # Handle remote logging if enabled
         if self.remote_logging:
             filestore = HTMLFileStore(filepath)
-            coop_details = filestore.push(description="Error report")
+            coop_details = filestore.push(description="Exceptions Report")
             print(coop_details)
-
-        print("Also see: https://docs.expectedparrot.com/en/latest/exceptions.html")

edsl/jobs/tasks/TaskHistory.py

@@ -264,9 +264,27 @@ class TaskHistory(RepresentationMixin):
         js = env.joinpath("report.js").read_text()
         return js
 
+    @property
+    def exceptions_table(self) -> dict:
+        """Return a dictionary of exceptions organized by type, service, model, and question name."""
+        exceptions_table = {}
+        for interview in self.total_interviews:
+            for question_name, exceptions in interview.exceptions.items():
+                for exception in exceptions:
+                    key = (
+                        exception.exception.__class__.__name__,  # Exception type
+                        interview.model._inference_service_,    # Service
+                        interview.model.model,                  # Model
+                        question_name                           # Question name
+                    )
+                    if key not in exceptions_table:
+                        exceptions_table[key] = 0
+                    exceptions_table[key] += 1
+        return exceptions_table
+    
     @property
     def exceptions_by_type(self) -> dict:
-        """Return a dictionary of exceptions by type."""
+        """Return a dictionary of exceptions tallied by type."""
         exceptions_by_type = {}
         for interview in self.total_interviews:
             for question_name, exceptions in interview.exceptions.items():
@@ -324,6 +342,27 @@ class TaskHistory(RepresentationMixin):
         }
         return sorted_exceptions_by_question_name
 
+    # @property
+    # def exceptions_by_model(self) -> dict:
+    #     """Return a dictionary of exceptions tallied by model and question name."""
+    #     exceptions_by_model = {}
+    #     for interview in self.total_interviews:
+    #         model = interview.model.model
+    #         service = interview.model._inference_service_
+    #         if (service, model) not in exceptions_by_model:
+    #             exceptions_by_model[(service, model)] = 0
+    #         if interview.exceptions != {}:
+    #             exceptions_by_model[(service, model)] += len(interview.exceptions)
+
+    #     # sort the exceptions by model
+    #     sorted_exceptions_by_model = {
+    #         k: v
+    #         for k, v in sorted(
+    #             exceptions_by_model.items(), key=lambda item: item[1], reverse=True
+    #         )
+    #     }
+    #     return sorted_exceptions_by_model
+
     @property
     def exceptions_by_model(self) -> dict:
         """Return a dictionary of exceptions tallied by model and question name."""
@@ -331,19 +370,12 @@ class TaskHistory(RepresentationMixin):
         for interview in self.total_interviews:
             model = interview.model.model
             service = interview.model._inference_service_
-            if (service, model) not in exceptions_by_model:
-                exceptions_by_model[(service, model)] = 0
-            if interview.exceptions != {}:
-                exceptions_by_model[(service, model)] += len(interview.exceptions)
-
-        # sort the exceptions by model
-        sorted_exceptions_by_model = {
-            k: v
-            for k, v in sorted(
-                exceptions_by_model.items(), key=lambda item: item[1], reverse=True
-            )
-        }
-        return sorted_exceptions_by_model
+            for question_name, exceptions in interview.exceptions.items():
+                key = (service, model, question_name)
+                if key not in exceptions_by_model:
+                    exceptions_by_model[key] = 0
+                exceptions_by_model[key] += len(exceptions)
+        return exceptions_by_model
 
     def generate_html_report(self, css: Optional[str], include_plot=False):
         if include_plot:
@@ -372,6 +404,7 @@ class TaskHistory(RepresentationMixin):
             javascript=self.javascript(),
             num_exceptions=len(self.exceptions),
             performance_plot_html=performance_plot_html,
+            exceptions_table=self.exceptions_table,
             exceptions_by_type=self.exceptions_by_type,
             exceptions_by_question_name=self.exceptions_by_question_name,
             exceptions_by_model=self.exceptions_by_model,
@@ -386,11 +419,10 @@ class TaskHistory(RepresentationMixin):
         filename: Optional[str] = None,
         return_link=False,
         css=None,
-        cta="Open Report in New Tab",
+        cta="\nClick to open the report in a new tab\n",
         open_in_browser=False,
     ):
         """Return an HTML report."""
-
         from IPython.display import display, HTML
         import tempfile
         import os
@@ -419,7 +451,7 @@ class TaskHistory(RepresentationMixin):
             html_link = f'<a href="{html_url}" target="_blank">{cta}</a>'
             display(HTML(html_link))
             escaped_output = html.escape(output)
-            iframe = f""""
+            iframe = f"""
             <iframe srcdoc="{ escaped_output }" style="width: 800px; height: 600px;"></iframe>
             """
             display(HTML(iframe))