edsl 0.1.27.dev2__py3-none-any.whl → 0.1.28__py3-none-any.whl

edsl/jobs/runners/JobsRunnerAsyncio.py

@@ -14,7 +14,8 @@ from edsl.results import Results, Result
 
 from edsl.jobs.interviews.Interview import Interview
 from edsl.utilities.decorators import jupyter_nb_handler
-from edsl.jobs.Jobs import Jobs
+
+# from edsl.jobs.Jobs import Jobs
 from edsl.jobs.runners.JobsRunnerStatusMixin import JobsRunnerStatusMixin
 from edsl.language_models import LanguageModel
 from edsl.data.Cache import Cache
@@ -22,6 +23,8 @@ from edsl.data.Cache import Cache
 from edsl.jobs.tasks.TaskHistory import TaskHistory
 from edsl.jobs.buckets.BucketCollection import BucketCollection
 
+import time
+
 
 class JobsRunnerAsyncio(JobsRunnerStatusMixin):
     """A class for running a collection of interviews asynchronously.
@@ -32,12 +35,12 @@ class JobsRunnerAsyncio(JobsRunnerStatusMixin):
 
     def __init__(self, jobs: Jobs):
         self.jobs = jobs
-
+        # this creates the interviews, which can take a while
         self.interviews: List["Interview"] = jobs.interviews()
         self.bucket_collection: "BucketCollection" = jobs.bucket_collection
         self.total_interviews: List["Interview"] = []
 
-    async def run_async(
+    async def run_async_generator(
         self,
         cache: Cache,
         n: int = 1,
@@ -96,6 +99,20 @@ class JobsRunnerAsyncio(JobsRunnerStatusMixin):
                     )  # set the cache for the first interview
                     self.total_interviews.append(interview)
 
+    async def run_async(self, cache=None) -> Results:
+        if cache is None:
+            self.cache = Cache()
+        else:
+            self.cache = cache
+        data = []
+        async for result in self.run_async_generator(cache=self.cache):
+            data.append(result)
+        return Results(survey=self.jobs.survey, data=data)
+
+    def simple_run(self):
+        data = asyncio.run(self.run_async())
+        return Results(survey=self.jobs.survey, data=data)
+
     async def _build_interview_task(
         self,
         *,
@@ -187,39 +204,13 @@ class JobsRunnerAsyncio(JobsRunnerStatusMixin):
         self.cache = cache
         self.sidecar_model = sidecar_model
 
-        def generate_table():
-            return self.status_table(self.results, self.elapsed_time)
-
-        @contextmanager
-        def no_op_cm():
-            """A no-op context manager with a dummy update method."""
-            yield DummyLive()
-
-        class DummyLive:
-            def update(self, *args, **kwargs):
-                """A dummy update method that does nothing."""
-                pass
-
-        progress_bar_context = (
-            Live(generate_table(), console=console, refresh_per_second=5)
-            if progress_bar
-            else no_op_cm()
-        )
-
-        with cache as c:
-            with progress_bar_context as live:
-
-                async def update_progress_bar():
-                    """Updates the progress bar at fixed intervals."""
-                    while True:
-                        live.update(generate_table())
-                        await asyncio.sleep(0.1)  # Update interval
-                        if self.completed:
-                            break
+        if not progress_bar:
+            # print("Running without progress bar")
+            with cache as c:
 
                 async def process_results():
                     """Processes results from interviews."""
-                    async for result in self.run_async(
+                    async for result in self.run_async_generator(
                         n=n,
                         debug=debug,
                         stop_on_exception=stop_on_exception,
@@ -227,25 +218,74 @@ class JobsRunnerAsyncio(JobsRunnerStatusMixin):
                         sidecar_model=sidecar_model,
                     ):
                         self.results.append(result)
-                        live.update(generate_table())
                     self.completed = True
 
-                progress_task = asyncio.create_task(update_progress_bar())
+                await asyncio.gather(process_results())
+
+            results = Results(survey=self.jobs.survey, data=self.results)
+        else:
+            # print("Running with progress bar")
+
+            def generate_table():
+                return self.status_table(self.results, self.elapsed_time)
 
-                try:
-                    await asyncio.gather(process_results(), progress_task)
-                except asyncio.CancelledError:
+            @contextmanager
+            def no_op_cm():
+                """A no-op context manager with a dummy update method."""
+                yield DummyLive()
+
+            class DummyLive:
+                def update(self, *args, **kwargs):
+                    """A dummy update method that does nothing."""
                     pass
-                finally:
-                    progress_task.cancel()  # Cancel the progress_task when process_results is done
-                    await progress_task
 
-                    await asyncio.sleep(1)  # short delay to show the final status
+            progress_bar_context = (
+                Live(generate_table(), console=console, refresh_per_second=5)
+                if progress_bar
+                else no_op_cm()
+            )
 
-                    # one more update
-                    live.update(generate_table())
+            with cache as c:
+                with progress_bar_context as live:
+
+                    async def update_progress_bar():
+                        """Updates the progress bar at fixed intervals."""
+                        while True:
+                            live.update(generate_table())
+                            await asyncio.sleep(0.00001)  # Update interval
+                            if self.completed:
+                                break
+
+                    async def process_results():
+                        """Processes results from interviews."""
+                        async for result in self.run_async_generator(
+                            n=n,
+                            debug=debug,
+                            stop_on_exception=stop_on_exception,
+                            cache=c,
+                            sidecar_model=sidecar_model,
+                        ):
+                            self.results.append(result)
+                            live.update(generate_table())
+                        self.completed = True
+
+                    progress_task = asyncio.create_task(update_progress_bar())
+
+                    try:
+                        await asyncio.gather(process_results(), progress_task)
+                    except asyncio.CancelledError:
+                        pass
+                    finally:
+                        progress_task.cancel()  # Cancel the progress_task when process_results is done
+                        await progress_task
+
+                        await asyncio.sleep(1)  # short delay to show the final status
+
+                        # one more update
+                        live.update(generate_table())
+
+            results = Results(survey=self.jobs.survey, data=self.results)
 
-        results = Results(survey=self.jobs.survey, data=self.results)
         task_history = TaskHistory(self.total_interviews, include_traceback=False)
         results.task_history = task_history
 
@@ -259,6 +299,8 @@ class JobsRunnerAsyncio(JobsRunnerStatusMixin):
                 for interview in self.total_interviews
                 if interview.has_exceptions
             ]
+            from edsl.jobs.Jobs import Jobs
+
             results.failed_jobs = Jobs.from_interviews(
                 [interview for interview in failed_interviews]
             )

edsl/jobs/runners/JobsRunnerStatusData.py

@@ -81,7 +81,7 @@ class JobsRunnerStatusData:
         >>> completed_tasks = []
         >>> elapsed_time = 0
         >>> JobsRunnerStatusData().generate_status_summary(completed_tasks, elapsed_time, interviews)
-        {'Elapsed time': '0.0 sec.', 'Total interviews requested': '1 ', 'Completed interviews': '0 ', 'Percent complete': '0 %', 'Average time per interview': 'NA', 'Task remaining': '1 ', 'Estimated time remaining': 'NA', 'model_queues': [{'model_name': 'gpt-4-1106-preview', 'TPM_limit_k': 1600.0, 'RPM_limit_k': 8.0, 'num_tasks_waiting': 0, 'token_usage_info': [{'cache_status': 'new_token_usage', 'details': [{'type': 'prompt_tokens', 'tokens': 0}, {'type': 'completion_tokens', 'tokens': 0}], 'cost': '$0.00000'}, {'cache_status': 'cached_token_usage', 'details': [{'type': 'prompt_tokens', 'tokens': 0}, {'type': 'completion_tokens', 'tokens': 0}], 'cost': '$0.00000'}]}]}
+        {'Elapsed time': '0.0 sec.', 'Total interviews requested': '1 ', 'Completed interviews': '0 ', 'Percent complete': '0 %', 'Average time per interview': 'NA', 'Task remaining': '1 ', 'Estimated time remaining': 'NA', 'model_queues': [{'model_name': '...', 'TPM_limit_k': ..., 'RPM_limit_k': ..., 'num_tasks_waiting': 0, 'token_usage_info': [{'cache_status': 'new_token_usage', 'details': [{'type': 'prompt_tokens', 'tokens': 0}, {'type': 'completion_tokens', 'tokens': 0}], 'cost': '$0.00000'}, {'cache_status': 'cached_token_usage', 'details': [{'type': 'prompt_tokens', 'tokens': 0}, {'type': 'completion_tokens', 'tokens': 0}], 'cost': '$0.00000'}]}]}
         """
 
         models_to_tokens = defaultdict(InterviewTokenUsage)
@@ -176,7 +176,7 @@ class JobsRunnerStatusData:
         >>> model = interviews[0].model
         >>> num_waiting = 0
         >>> JobsRunnerStatusData()._get_model_info(model, num_waiting, models_to_tokens)
-        {'model_name': 'gpt-4-1106-preview', 'TPM_limit_k': 1600.0, 'RPM_limit_k': 8.0, 'num_tasks_waiting': 0, 'token_usage_info': [{'cache_status': 'new_token_usage', 'details': [{'type': 'prompt_tokens', 'tokens': 0}, {'type': 'completion_tokens', 'tokens': 0}], 'cost': '$0.00000'}, {'cache_status': 'cached_token_usage', 'details': [{'type': 'prompt_tokens', 'tokens': 0}, {'type': 'completion_tokens', 'tokens': 0}], 'cost': '$0.00000'}]}
+        {'model_name': 'gpt-4-1106-preview', 'TPM_limit_k': ..., 'RPM_limit_k': ..., 'num_tasks_waiting': 0, 'token_usage_info': [{'cache_status': 'new_token_usage', 'details': [{'type': 'prompt_tokens', 'tokens': 0}, {'type': 'completion_tokens', 'tokens': 0}], 'cost': '$0.00000'}, {'cache_status': 'cached_token_usage', 'details': [{'type': 'prompt_tokens', 'tokens': 0}, {'type': 'completion_tokens', 'tokens': 0}], 'cost': '$0.00000'}]}
         """
 
         prices = get_token_pricing(model.model)
@@ -234,4 +234,4 @@ class JobsRunnerStatusData:
 if __name__ == "__main__":
     import doctest
 
-    doctest.testmod()
+    doctest.testmod(optionflags=doctest.ELLIPSIS)

edsl/jobs/tasks/QuestionTaskCreator.py

@@ -1,7 +1,7 @@
 import asyncio
 from typing import Callable, Union, List
 from collections import UserList, UserDict
-
+import time
 
 from edsl.jobs.buckets import ModelBuckets
 from edsl.exceptions import InterviewErrorPriorTaskCanceled
@@ -132,7 +132,7 @@ class QuestionTaskCreator(UserList):
             self.waiting = True
             self.task_status = TaskStatus.WAITING_FOR_REQUEST_CAPACITY
 
-        await self.requests_bucket.get_tokens(1)
+        await self.tokens_bucket.get_tokens(1)
 
         self.task_status = TaskStatus.API_CALL_IN_PROGRESS
         try:
@@ -151,6 +151,8 @@ class QuestionTaskCreator(UserList):
                 self.requests_bucket.add_tokens(1)
                 self.from_cache = True
 
+        _ = results.pop("cached_response", None)
+
         tracker = self.cached_token_usage if self.from_cache else self.new_token_usage
 
         # TODO: This is hacky. The 'func' call should return an object that definitely has a 'usage' key.

edsl/language_models/LanguageModel.py

@@ -142,7 +142,7 @@ class LanguageModel(
     def has_valid_api_key(self) -> bool:
         """Check if the model has a valid API key.
 
-        >>> LanguageModel.example().has_valid_api_key()
+        >>> LanguageModel.example().has_valid_api_key() : # doctest: +SKIP
         True
 
         This method is used to check if the model has a valid API key.
@@ -159,7 +159,9 @@ class LanguageModel(
 
     def __hash__(self):
         """Allow the model to be used as a key in a dictionary."""
-        return hash(self.model + str(self.parameters))
+        from edsl.utilities.utilities import dict_hash
+
+        return dict_hash(self.to_dict())
 
     def __eq__(self, other):
         """Check is two models are the same.
@@ -207,8 +209,8 @@ class LanguageModel(
         """Model's tokens-per-minute limit.
 
         >>> m = LanguageModel.example()
-        >>> m.TPM
-        1600000.0
+        >>> m.TPM > 0
+        True
         """
         self._set_rate_limits()
         return self._safety_factor * self.__rate_limits["tpm"]
@@ -285,28 +287,6 @@ class LanguageModel(
         """
         raise NotImplementedError
 
-    def _update_response_with_tracking(
-        self, response: dict, start_time: int, cached_response=False, cache_key=None
-    ):
-        """Update the response with tracking information.
-
-        >>> m = LanguageModel.example()
-        >>> m._update_response_with_tracking(response={"response": "Hello"}, start_time=0, cached_response=False, cache_key=None)
-        {'response': 'Hello', 'elapsed_time': ..., 'timestamp': ..., 'cached_response': False, 'cache_key': None}
-
-
-        """
-        end_time = time.time()
-        response.update(
-            {
-                "elapsed_time": end_time - start_time,
-                "timestamp": end_time,
-                "cached_response": cached_response,
-                "cache_key": cache_key,
-            }
-        )
-        return response
-
     async def async_get_raw_response(
         self,
         user_prompt: str,
@@ -314,7 +294,7 @@ class LanguageModel(
         cache,
         iteration: int = 0,
         encoded_image=None,
-    ) -> dict[str, Any]:
+    ) -> tuple[dict, bool, str]:
         """Handle caching of responses.
 
         :param user_prompt: The user's prompt.
@@ -322,8 +302,7 @@ class LanguageModel(
         :param iteration: The iteration number.
         :param cache: The cache to use.
 
-        If the cache isn't being used, it just returns a 'fresh' call to the LLM,
-        but appends some tracking information to the response (using the _update_response_with_tracking method).
+        If the cache isn't being used, it just returns a 'fresh' call to the LLM.
         But if cache is being used, it first checks the database to see if the response is already there.
         If it is, it returns the cached response, but again appends some tracking information.
         If it isn't, it calls the LLM, saves the response to the database, and returns the response with tracking information.
@@ -334,7 +313,7 @@ class LanguageModel(
         >>> from edsl import Cache
         >>> m = LanguageModel.example(test_model = True)
         >>> m.get_raw_response(user_prompt = "Hello", system_prompt = "hello", cache = Cache())
-        {'message': '{"answer": "Hello world"}', 'elapsed_time': ..., 'timestamp': ..., 'cached_response': False, 'cache_key': '24ff6ac2bc2f1729f817f261e0792577'}
+        ({'message': '{"answer": "Hello world"}'}, False, '24ff6ac2bc2f1729f817f261e0792577')
         """
         start_time = time.time()
 
@@ -379,12 +358,7 @@ class LanguageModel(
             )
             cache_used = False
 
-        return self._update_response_with_tracking(
-            response=response,
-            start_time=start_time,
-            cached_response=cache_used,
-            cache_key=cache_key,
-        )
+        return response, cache_used, cache_key
 
     get_raw_response = sync_wrapper(async_get_raw_response)
 
@@ -427,14 +401,18 @@ class LanguageModel(
         if encoded_image:
             params["encoded_image"] = encoded_image
 
-        raw_response = await self.async_get_raw_response(**params)
+        raw_response, cache_used, cache_key = await self.async_get_raw_response(
+            **params
+        )
         response = self.parse_response(raw_response)
 
         try:
             dict_response = json.loads(response)
         except json.JSONDecodeError as e:
             # TODO: Turn into logs to generate issues
-            dict_response, success = await repair(response, str(e))
+            dict_response, success = await repair(
+                bad_json=response, error_message=str(e), cache=cache
+            )
             if not success:
                 raise Exception(
                     f"""Even the repair failed. The error was: {e}. The response was: {response}."""
@@ -442,7 +420,8 @@ class LanguageModel(
 
         dict_response.update(
             {
-                "cached_response": raw_response["cached_response"],
+                "cached_used": cache_used,
+                "cache_key": cache_key,
                 "usage": raw_response.get("usage", {}),
                 "raw_model_response": raw_response,
             }
@@ -458,15 +437,18 @@ class LanguageModel(
     #######################
     # SERIALIZATION METHODS
     #######################
+    def _to_dict(self) -> dict[str, Any]:
+        return {"model": self.model, "parameters": self.parameters}
+
     @add_edsl_version
     def to_dict(self) -> dict[str, Any]:
         """Convert instance to a dictionary.
 
         >>> m = LanguageModel.example()
         >>> m.to_dict()
-        {'model': 'gpt-4-1106-preview', 'parameters': {'temperature': 0.5, 'max_tokens': 1000, 'top_p': 1, 'frequency_penalty': 0, 'presence_penalty': 0, 'logprobs': False, 'top_logprobs': 3}}
+        {'model': 'gpt-4-1106-preview', 'parameters': {'temperature': 0.5, 'max_tokens': 1000, 'top_p': 1, 'frequency_penalty': 0, 'presence_penalty': 0, 'logprobs': False, 'top_logprobs': 3}, 'edsl_version': '...', 'edsl_class_name': 'LanguageModel'}
         """
-        return {"model": self.model, "parameters": self.parameters}
+        return self._to_dict()
 
     @classmethod
     @remove_edsl_version
@@ -519,8 +501,18 @@ class LanguageModel(
         return table
 
     @classmethod
-    def example(cls, test_model=False):
-        """Return a default instance of the class."""
+    def example(cls, test_model: bool = False, canned_response: str = "Hello world"):
+        """Return a default instance of the class.
+
+        >>> from edsl.language_models import LanguageModel
+        >>> m = LanguageModel.example(test_model = True, canned_response = "WOWZA!")
+        >>> isinstance(m, LanguageModel)
+        True
+        >>> from edsl import QuestionFreeText
+        >>> q = QuestionFreeText(question_text = "What is your name?", question_name = 'example')
+        >>> q.by(m).run(cache = False).select('example').first()
+        'WOWZA!'
+        """
         from edsl import Model
 
         class TestLanguageModelGood(LanguageModel):
@@ -533,7 +525,8 @@ class LanguageModel(
                 self, user_prompt: str, system_prompt: str
             ) -> dict[str, Any]:
                 await asyncio.sleep(0.1)
-                return {"message": """{"answer": "Hello world"}"""}
+                # return {"message": """{"answer": "Hello, world"}"""}
+                return {"message": f'{{"answer": "{canned_response}"}}'}
 
             def parse_response(self, raw_response: dict[str, Any]) -> str:
                 return raw_response["message"]

edsl/language_models/ModelList.py

@@ -0,0 +1,96 @@
+from typing import Optional
+from collections import UserList
+from edsl import Model
+
+from edsl.language_models import LanguageModel
+from edsl.Base import Base
+from edsl.utilities.decorators import add_edsl_version, remove_edsl_version
+from edsl.utilities import is_valid_variable_name
+from edsl.utilities.utilities import dict_hash
+
+
+class ModelList(Base, UserList):
+    def __init__(self, data: Optional[list] = None):
+        """Initialize the ScenarioList class.
+
+        >>> from edsl import Model
+        >>> m = ModelList(Model.available())
+
+        """
+        if data is not None:
+            super().__init__(data)
+        else:
+            super().__init__([])
+
+    @property
+    def names(self):
+        """
+
+        >>> ModelList.example().names
+        {'...'}
+        """
+        return set([model.model for model in self])
+
+    def rich_print(self):
+        pass
+
+    def __repr__(self):
+        return f"ModelList({super().__repr__()})"
+
+    def __hash__(self):
+        """Return a hash of the ModelList. This is used for comparison of ModelLists.
+
+        >>> hash(ModelList.example())
+        1423518243781418961
+
+        """
+        from edsl.utilities.utilities import dict_hash
+
+        return dict_hash(self._to_dict(sort=True))
+
+    def _to_dict(self, sort=False):
+        if sort:
+            model_list = sorted([model for model in self], key=lambda x: hash(x))
+            return {"models": [model._to_dict() for model in model_list]}
+        else:
+            return {"models": [model._to_dict() for model in self]}
+
+    @classmethod
+    def from_names(self, *args):
+        """A a model list from a list of names"""
+        if len(args) == 1 and isinstance(args[0], list):
+            args = args[0]
+        return ModelList([Model(model_name) for model_name in args])
+
+    @add_edsl_version
+    def to_dict(self):
+        """
+        Convert the ModelList to a dictionary.
+        >>> ModelList.example().to_dict()
+        {'models': [...], 'edsl_version': '...', 'edsl_class_name': 'ModelList'}
+        """
+        return self._to_dict()
+
+    @classmethod
+    @remove_edsl_version
+    def from_dict(cls, data):
+        """
+        Create a ModelList from a dictionary.
+
+        >>> newm = ModelList.from_dict(ModelList.example().to_dict())
+        >>> assert ModelList.example() == newm
+        """
+        return cls(data=[LanguageModel.from_dict(model) for model in data["models"]])
+
+    def code(self):
+        pass
+
+    @classmethod
+    def example(cl):
+        return ModelList([LanguageModel.example() for _ in range(3)])
+
+
+if __name__ == "__main__":
+    import doctest
+
+    doctest.testmod(optionflags=doctest.ELLIPSIS)

edsl/language_models/registry.py

@@ -38,6 +38,20 @@ class Model(metaclass=Meta):
         factory = registry.create_model_factory(model_name)
         return factory(*args, **kwargs)
 
+    @classmethod
+    def add_model(cls, service_name, model_name):
+        from edsl.inference_services.registry import default
+
+        registry = default
+        registry.add_model(service_name, model_name)
+
+    @classmethod
+    def services(cls, registry=None):
+        from edsl.inference_services.registry import default
+
+        registry = registry or default
+        return [r._inference_service_ for r in registry.services]
+
     @classmethod
     def available(cls, search_term=None, name_only=False, registry=None):
         from edsl.inference_services.registry import default