edsl 0.1.33__py3-none-any.whl → 0.1.33.dev2__py3-none-any.whl

edsl/jobs/runners/JobsRunnerStatusMixin.py

@@ -0,0 +1,333 @@
+from __future__ import annotations
+from typing import List, DefaultDict
+import asyncio
+from typing import Type
+from collections import defaultdict
+
+from typing import Literal, List, Type, DefaultDict
+from collections import UserDict, defaultdict
+
+from edsl.jobs.interviews.InterviewStatusDictionary import InterviewStatusDictionary
+from edsl.jobs.tokens.InterviewTokenUsage import InterviewTokenUsage
+from edsl.jobs.tokens.TokenUsage import TokenUsage
+from edsl.enums import get_token_pricing
+from edsl.jobs.tasks.task_status_enum import TaskStatus
+
+InterviewTokenUsageMapping = DefaultDict[str, InterviewTokenUsage]
+
+from edsl.jobs.interviews.InterviewStatistic import InterviewStatistic
+from edsl.jobs.interviews.InterviewStatisticsCollection import (
+    InterviewStatisticsCollection,
+)
+from edsl.jobs.tokens.InterviewTokenUsage import InterviewTokenUsage
+
+
+# return {"cache_status": token_usage_type, "details": details, "cost": f"${token_usage.cost(prices):.5f}"}
+
+from dataclasses import dataclass, asdict
+
+from rich.text import Text
+from rich.box import SIMPLE
+from rich.table import Table
+
+
+@dataclass
+class ModelInfo:
+    model_name: str
+    TPM_limit_k: float
+    RPM_limit_k: float
+    num_tasks_waiting: int
+    token_usage_info: dict
+
+
+@dataclass
+class ModelTokenUsageStats:
+    token_usage_type: str
+    details: List[dict]
+    cost: str
+
+
+class Stats:
+    def elapsed_time(self):
+        InterviewStatistic("elapsed_time", value=elapsed_time, digits=1, units="sec.")
+
+
+class JobsRunnerStatusMixin:
+    # @staticmethod
+    # def status_dict(interviews: List[Type["Interview"]]) -> List[Type[InterviewStatusDictionary]]:
+    #     """
+    #     >>> from edsl.jobs.interviews.Interview import Interview
+    #     >>> interviews = [Interview.example()]
+    #     >>> JobsRunnerStatusMixin().status_dict(interviews)
+    #     [InterviewStatusDictionary({<TaskStatus.NOT_STARTED: 1>: 0, <TaskStatus.WAITING_FOR_DEPENDENCIES: 2>: 0, <TaskStatus.CANCELLED: 3>: 0, <TaskStatus.PARENT_FAILED: 4>: 0, <TaskStatus.WAITING_FOR_REQUEST_CAPACITY: 5>: 0, <TaskStatus.WAITING_FOR_TOKEN_CAPACITY: 6>: 0, <TaskStatus.API_CALL_IN_PROGRESS: 7>: 0, <TaskStatus.SUCCESS: 8>: 0, <TaskStatus.FAILED: 9>: 0, 'number_from_cache': 0})]
+    #     """
+    #     return [interview.interview_status for interview in interviews]
+
+    def _compute_statistic(stat_name: str, completed_tasks, elapsed_time, interviews):
+        stat_definitions = {
+            "elapsed_time": lambda: InterviewStatistic(
+                "elapsed_time", value=elapsed_time, digits=1, units="sec."
+            ),
+            "total_interviews_requested": lambda: InterviewStatistic(
+                "total_interviews_requested", value=len(interviews), units=""
+            ),
+            "completed_interviews": lambda: InterviewStatistic(
+                "completed_interviews", value=len(completed_tasks), units=""
+            ),
+            "percent_complete": lambda: InterviewStatistic(
+                "percent_complete",
+                value=(
+                    len(completed_tasks) / len(interviews) * 100
+                    if len(interviews) > 0
+                    else "NA"
+                ),
+                digits=0,
+                units="%",
+            ),
+            "average_time_per_interview": lambda: InterviewStatistic(
+                "average_time_per_interview",
+                value=elapsed_time / len(completed_tasks) if completed_tasks else "NA",
+                digits=1,
+                units="sec.",
+            ),
+            "task_remaining": lambda: InterviewStatistic(
+                "task_remaining", value=len(interviews) - len(completed_tasks), units=""
+            ),
+            "estimated_time_remaining": lambda: InterviewStatistic(
+                "estimated_time_remaining",
+                value=(
+                    (len(interviews) - len(completed_tasks))
+                    * (elapsed_time / len(completed_tasks))
+                    if len(completed_tasks) > 0
+                    else "NA"
+                ),
+                digits=1,
+                units="sec.",
+            ),
+        }
+        if stat_name not in stat_definitions:
+            raise ValueError(
+                f"Invalid stat_name: {stat_name}. The valid stat_names are: {list(stat_definitions.keys())}"
+            )
+        return stat_definitions[stat_name]()
+
+    @staticmethod
+    def _job_level_info(
+        completed_tasks: List[Type[asyncio.Task]],
+        elapsed_time: float,
+        interviews: List[Type["Interview"]],
+    ) -> InterviewStatisticsCollection:
+        interview_statistics = InterviewStatisticsCollection()
+
+        default_statistics = [
+            "elapsed_time",
+            "total_interviews_requested",
+            "completed_interviews",
+            "percent_complete",
+            "average_time_per_interview",
+            "task_remaining",
+            "estimated_time_remaining",
+        ]
+        for stat_name in default_statistics:
+            interview_statistics.add_stat(
+                JobsRunnerStatusMixin._compute_statistic(
+                    stat_name, completed_tasks, elapsed_time, interviews
+                )
+            )
+
+        return interview_statistics
+
+    @staticmethod
+    def _get_model_queues_info(interviews):
+        models_to_tokens = defaultdict(InterviewTokenUsage)
+        model_to_status = defaultdict(InterviewStatusDictionary)
+        waiting_dict = defaultdict(int)
+
+        for interview in interviews:
+            models_to_tokens[interview.model] += interview.token_usage
+            model_to_status[interview.model] += interview.interview_status
+            waiting_dict[interview.model] += interview.interview_status.waiting
+
+        for model, num_waiting in waiting_dict.items():
+            yield JobsRunnerStatusMixin._get_model_info(
+                model, num_waiting, models_to_tokens
+            )
+
+    @staticmethod
+    def generate_status_summary(
+        completed_tasks: List[Type[asyncio.Task]],
+        elapsed_time: float,
+        interviews: List[Type["Interview"]],
+        include_model_queues=False,
+    ) -> InterviewStatisticsCollection:
+        """Generate a summary of the status of the job runner.
+
+        :param completed_tasks: list of completed tasks
+        :param elapsed_time: time elapsed since the start of the job
+        :param interviews: list of interviews to be conducted
+
+        >>> from edsl.jobs.interviews.Interview import Interview
+        >>> interviews = [Interview.example()]
+        >>> completed_tasks = []
+        >>> elapsed_time = 0
+        >>> JobsRunnerStatusMixin().generate_status_summary(completed_tasks, elapsed_time, interviews)
+        {'Elapsed time': '0.0 sec.', 'Total interviews requested': '1 ', 'Completed interviews': '0 ', 'Percent complete': '0 %', 'Average time per interview': 'NA', 'Task remaining': '1 ', 'Estimated time remaining': 'NA'}
+        """
+
+        interview_status_summary: InterviewStatisticsCollection = (
+            JobsRunnerStatusMixin._job_level_info(
+                completed_tasks=completed_tasks,
+                elapsed_time=elapsed_time,
+                interviews=interviews,
+            )
+        )
+        if include_model_queues:
+            interview_status_summary.model_queues = list(
+                JobsRunnerStatusMixin._get_model_queues_info(interviews)
+            )
+        else:
+            interview_status_summary.model_queues = None
+
+        return interview_status_summary
+
+    @staticmethod
+    def _get_model_info(
+        model: str,
+        num_waiting: int,
+        models_to_tokens: InterviewTokenUsageMapping,
+    ) -> dict:
+        """Get the status of a model.
+
+        :param model: the model name
+        :param num_waiting: the number of tasks waiting for capacity
+        :param models_to_tokens: a mapping of models to token usage
+
+        >>> from edsl.jobs.interviews.Interview import Interview
+        >>> interviews = [Interview.example()]
+        >>> models_to_tokens = defaultdict(InterviewTokenUsage)
+        >>> model = interviews[0].model
+        >>> num_waiting = 0
+        >>> JobsRunnerStatusMixin()._get_model_info(model, num_waiting, models_to_tokens)
+        ModelInfo(model_name='...', TPM_limit_k=..., RPM_limit_k=..., num_tasks_waiting=0, token_usage_info=[ModelTokenUsageStats(token_usage_type='new_token_usage', details=[{'type': 'prompt_tokens', 'tokens': 0}, {'type': 'completion_tokens', 'tokens': 0}], cost='$0.00000'), ModelTokenUsageStats(token_usage_type='cached_token_usage', details=[{'type': 'prompt_tokens', 'tokens': 0}, {'type': 'completion_tokens', 'tokens': 0}], cost='$0.00000')])
+        """
+
+        ## TODO: This should probably be a coop method
+        prices = get_token_pricing(model.model)
+
+        token_usage_info = []
+        for token_usage_type in ["new_token_usage", "cached_token_usage"]:
+            token_usage_info.append(
+                JobsRunnerStatusMixin._get_token_usage_info(
+                    token_usage_type, models_to_tokens, model, prices
+                )
+            )
+
+        return ModelInfo(
+            **{
+                "model_name": model.model,
+                "TPM_limit_k": model.TPM / 1000,
+                "RPM_limit_k": model.RPM / 1000,
+                "num_tasks_waiting": num_waiting,
+                "token_usage_info": token_usage_info,
+            }
+        )
+
+    @staticmethod
+    def _get_token_usage_info(
+        token_usage_type: Literal["new_token_usage", "cached_token_usage"],
+        models_to_tokens: InterviewTokenUsageMapping,
+        model: str,
+        prices: "TokenPricing",
+    ) -> ModelTokenUsageStats:
+        """Get the token usage info for a model.
+
+        >>> from edsl.jobs.interviews.Interview import Interview
+        >>> interviews = [Interview.example()]
+        >>> models_to_tokens = defaultdict(InterviewTokenUsage)
+        >>> model = interviews[0].model
+        >>> prices = get_token_pricing(model.model)
+        >>> cache_status = "new_token_usage"
+        >>> JobsRunnerStatusMixin()._get_token_usage_info(cache_status, models_to_tokens, model, prices)
+        ModelTokenUsageStats(token_usage_type='new_token_usage', details=[{'type': 'prompt_tokens', 'tokens': 0}, {'type': 'completion_tokens', 'tokens': 0}], cost='$0.00000')
+
+        """
+        all_token_usage: InterviewTokenUsage = models_to_tokens[model]
+        token_usage: TokenUsage = getattr(all_token_usage, token_usage_type)
+
+        details = [
+            {"type": token_type, "tokens": getattr(token_usage, token_type)}
+            for token_type in ["prompt_tokens", "completion_tokens"]
+        ]
+
+        return ModelTokenUsageStats(
+            token_usage_type=token_usage_type,
+            details=details,
+            cost=f"${token_usage.cost(prices):.5f}",
+        )
+
+    @staticmethod
+    def _add_statistics_to_table(table, status_summary):
+        table.add_column("Statistic", style="dim", no_wrap=True, width=50)
+        table.add_column("Value", width=10)
+
+        for key, value in status_summary.items():
+            if key != "model_queues":
+                table.add_row(key, value)
+
+    @staticmethod
+    def display_status_table(status_summary: InterviewStatisticsCollection) -> "Table":
+        table = Table(
+            title="Job Status",
+            show_header=True,
+            header_style="bold magenta",
+            box=SIMPLE,
+        )
+
+        ### Job-level statistics
+        JobsRunnerStatusMixin._add_statistics_to_table(table, status_summary)
+
+        ## Model-level statistics
+        spacing = " "
+
+        if status_summary.model_queues is not None:
+            table.add_row(Text("Model Queues", style="bold red"), "")
+            for model_info in status_summary.model_queues:
+                model_name = model_info.model_name
+                tpm = f"TPM (k)={model_info.TPM_limit_k}"
+                rpm = f"RPM (k)= {model_info.RPM_limit_k}"
+                pretty_model_name = model_name + ";" + tpm + ";" + rpm
+                table.add_row(Text(pretty_model_name, style="blue"), "")
+                table.add_row(
+                    "Number question tasks waiting for capacity",
+                    str(model_info.num_tasks_waiting),
+                )
+                # Token usage and cost info
+                for token_usage_info in model_info.token_usage_info:
+                    token_usage_type = token_usage_info.token_usage_type
+                    table.add_row(
+                        Text(
+                            spacing + token_usage_type.replace("_", " "), style="bold"
+                        ),
+                        "",
+                    )
+                    for detail in token_usage_info.details:
+                        token_type = detail["type"]
+                        tokens = detail["tokens"]
+                        table.add_row(spacing + f"{token_type}", f"{tokens:,}")
+                    # table.add_row(spacing + "cost", cache_info["cost"])
+
+        return table
+
+    def status_table(self, completed_tasks: List[asyncio.Task], elapsed_time: float):
+        summary_data = JobsRunnerStatusMixin.generate_status_summary(
+            completed_tasks=completed_tasks,
+            elapsed_time=elapsed_time,
+            interviews=self.total_interviews,
+        )
+        return self.display_status_table(summary_data)
+
+
+if __name__ == "__main__":
+    import doctest
+
+    doctest.testmod(optionflags=doctest.ELLIPSIS)

edsl/jobs/tasks/TaskHistory.py

@@ -50,18 +50,6 @@ class TaskHistory:
         """
         return [i.exceptions for k, i in self._interviews.items() if i.exceptions != {}]
 
-    @property
-    def unfixed_exceptions(self):
-        """
-        >>> len(TaskHistory.example().unfixed_exceptions)
-        4
-        """
-        return [
-            i.exceptions
-            for k, i in self._interviews.items()
-            if i.exceptions.num_unfixed() > 0
-        ]
-
     @property
     def indices(self):
         return [k for k, i in self._interviews.items() if i.exceptions != {}]
@@ -90,11 +78,6 @@ class TaskHistory:
         """
         return len(self.exceptions) > 0
 
-    @property
-    def has_unfixed_exceptions(self) -> bool:
-        """Return True if there are any exceptions."""
-        return len(self.unfixed_exceptions) > 0
-
     def _repr_html_(self):
         """Return an HTML representation of the TaskHistory."""
         from edsl.utilities.utilities import data_to_html

edsl/language_models/LanguageModel.py

@@ -164,20 +164,20 @@ class LanguageModel(
         None  # This should be something like ["choices", 0, "message", "content"]
     )
     __rate_limits = None
+    __default_rate_limits = {
+        "rpm": 10_000,
+        "tpm": 2_000_000,
+    }  # TODO: Use the OpenAI Teir 1 rate limits
     _safety_factor = 0.8
 
-    def __init__(
-        self, tpm=None, rpm=None, omit_system_prompt_if_empty_string=True, **kwargs
-    ):
+    def __init__(self, tpm=None, rpm=None, **kwargs):
         """Initialize the LanguageModel."""
         self.model = getattr(self, "_model_", None)
         default_parameters = getattr(self, "_parameters_", None)
         parameters = self._overide_default_parameters(kwargs, default_parameters)
         self.parameters = parameters
         self.remote = False
-        self.omit_system_prompt_if_empty = omit_system_prompt_if_empty_string
 
-        # self._rpm / _tpm comes from the class
         if rpm is not None:
             self._rpm = rpm
 
@@ -286,40 +286,35 @@ class LanguageModel(
         >>> m.RPM
         100
         """
-        if rpm is not None:
-            self._rpm = rpm
-        if tpm is not None:
-            self._tpm = tpm
-        return None
-        # self._set_rate_limits(rpm=rpm, tpm=tpm)
+        self._set_rate_limits(rpm=rpm, tpm=tpm)
 
-    # def _set_rate_limits(self, rpm=None, tpm=None) -> None:
-    #     """Set the rate limits for the model.
+    def _set_rate_limits(self, rpm=None, tpm=None) -> None:
+        """Set the rate limits for the model.
 
-    #     If the model does not have rate limits, use the default rate limits."""
-    #     if rpm is not None and tpm is not None:
-    #         self.__rate_limits = {"rpm": rpm, "tpm": tpm}
-    #         return
+        If the model does not have rate limits, use the default rate limits."""
+        if rpm is not None and tpm is not None:
+            self.__rate_limits = {"rpm": rpm, "tpm": tpm}
+            return
 
-    #     if self.__rate_limits is None:
-    #         if hasattr(self, "get_rate_limits"):
-    #             self.__rate_limits = self.get_rate_limits()
-    #         else:
-    #             self.__rate_limits = self.__default_rate_limits
+        if self.__rate_limits is None:
+            if hasattr(self, "get_rate_limits"):
+                self.__rate_limits = self.get_rate_limits()
+            else:
+                self.__rate_limits = self.__default_rate_limits
 
     @property
     def RPM(self):
         """Model's requests-per-minute limit."""
         # self._set_rate_limits()
         # return self._safety_factor * self.__rate_limits["rpm"]
-        return self._rpm
+        return self.rpm
 
     @property
     def TPM(self):
         """Model's tokens-per-minute limit."""
         # self._set_rate_limits()
         # return self._safety_factor * self.__rate_limits["tpm"]
-        return self._tpm
+        return self.tpm
 
     @property
     def rpm(self):
@@ -337,6 +332,17 @@ class LanguageModel(
     def tpm(self, value):
         self._tpm = value
 
+    @property
+    def TPM(self):
+        """Model's tokens-per-minute limit.
+
+        >>> m = LanguageModel.example()
+        >>> m.TPM > 0
+        True
+        """
+        self._set_rate_limits()
+        return self._safety_factor * self.__rate_limits["tpm"]
+
     @staticmethod
     def _overide_default_parameters(passed_parameter_dict, default_parameter_dict):
         """Return a dictionary of parameters, with passed parameters taking precedence over defaults.
@@ -465,13 +471,12 @@ class LanguageModel(
         if encoded_image:
             # the image has is appended to the user_prompt for hash-lookup purposes
             image_hash = hashlib.md5(encoded_image.encode()).hexdigest()
-            user_prompt += f" {image_hash}"
 
         cache_call_params = {
             "model": str(self.model),
             "parameters": self.parameters,
             "system_prompt": system_prompt,
-            "user_prompt": user_prompt,
+            "user_prompt": user_prompt + "" if not encoded_image else f" {image_hash}",
             "iteration": iteration,
         }
         cached_response, cache_key = cache.fetch(**cache_call_params)

edsl/language_models/registry.py

@@ -2,10 +2,10 @@ import textwrap
 from random import random
 from edsl.config import CONFIG
 
-# if "EDSL_DEFAULT_MODEL" not in CONFIG:
-#     default_model = "test"
-# else:
-#     default_model = CONFIG.get("EDSL_DEFAULT_MODEL")
+if "EDSL_DEFAULT_MODEL" not in CONFIG:
+    default_model = "test"
+else:
+    default_model = CONFIG.get("EDSL_DEFAULT_MODEL")
 
 
 def get_model_class(model_name, registry=None):
@@ -33,24 +33,20 @@ class Meta(type):
 
 
 class Model(metaclass=Meta):
-    default_model = CONFIG.get("EDSL_DEFAULT_MODEL")
+    default_model = default_model
 
-    def __new__(
-        cls, model_name=None, registry=None, service_name=None, *args, **kwargs
-    ):
+    def __new__(cls, model_name=None, registry=None, *args, **kwargs):
         # Map index to the respective subclass
         if model_name is None:
-            model_name = (
-                cls.default_model
-            )  # when model_name is None, use the default model, set in the config file
+            model_name = cls.default_model
         from edsl.inference_services.registry import default
 
         registry = registry or default
 
-        if isinstance(model_name, int):  # can refer to a model by index
+        if isinstance(model_name, int):
             model_name = cls.available(name_only=True)[model_name]
 
-        factory = registry.create_model_factory(model_name, service_name=service_name)
+        factory = registry.create_model_factory(model_name)
         return factory(*args, **kwargs)
 
     @classmethod

edsl/questions/QuestionBase.py

@@ -103,8 +103,13 @@ class QuestionBase(
         """Validate the answer.
         >>> from edsl.exceptions import QuestionAnswerValidationError
         >>> from edsl import QuestionFreeText as Q
-        >>> Q.example()._validate_answer({'answer': 'Hello', 'generated_tokens': 'Hello'})
-        {'answer': 'Hello', 'generated_tokens': 'Hello'}
+        >>> Q.example()._validate_answer({'answer': 'Hello'})
+        {'answer': 'Hello', 'generated_tokens': None}
+        >>> Q.example()._validate_answer({'shmanswer': 1})
+        Traceback (most recent call last):
+        ...
+        edsl.exceptions.questions.QuestionAnswerValidationError:...
+        ...
         """
 
         return self.response_validator.validate(answer)
@@ -466,7 +471,6 @@ class QuestionBase(
         self,
         scenario: Optional[dict] = None,
         agent: Optional[dict] = {},
-        answers: Optional[dict] = None,
         include_question_name: bool = False,
         height: Optional[int] = None,
         width: Optional[int] = None,
@@ -478,17 +482,6 @@ class QuestionBase(
         if scenario is None:
             scenario = {}
 
-        prior_answers_dict = {}
-
-        if isinstance(answers, dict):
-            for key, value in answers.items():
-                if not key.endswith("_comment") and not key.endswith(
-                    "_generated_tokens"
-                ):
-                    prior_answers_dict[key] = {"answer": value}
-
-        # breakpoint()
-
         base_template = """
         <div id="{{ question_name }}" class="survey_question" data-type="{{ question_type }}">
             {% if include_question_name %}
@@ -508,40 +501,13 @@ class QuestionBase(
 
         base_template = Template(base_template)
 
-        context = {
-            "scenario": scenario,
-            "agent": agent,
-        } | prior_answers_dict
-
-        # Render the question text
-        try:
-            question_text = Template(self.question_text).render(context)
-        except Exception as e:
-            print(
-                f"Error rendering question: question_text = {self.question_text}, error = {e}"
-            )
-            question_text = self.question_text
-
-        try:
-            question_content = Template(question_content).render(context)
-        except Exception as e:
-            print(
-                f"Error rendering question: question_content = {question_content}, error = {e}"
-            )
-            question_content = question_content
-
-        try:
-            params = {
-                "question_name": self.question_name,
-                "question_text": question_text,
-                "question_type": self.question_type,
-                "question_content": question_content,
-                "include_question_name": include_question_name,
-            }
-        except Exception as e:
-            raise ValueError(
-                f"Error rendering question: params = {params}, error = {e}"
-            )
+        params = {
+            "question_name": self.question_name,
+            "question_text": Template(self.question_text).render(scenario, agent=agent),
+            "question_type": self.question_type,
+            "question_content": Template(question_content).render(scenario),
+            "include_question_name": include_question_name,
+        }
         rendered_html = base_template.render(**params)
 
         if iframe:
@@ -560,21 +526,6 @@ class QuestionBase(
 
         return rendered_html
 
-    @classmethod
-    def example_model(cls):
-        from edsl import Model
-
-        q = cls.example()
-        m = Model("test", canned_response=cls._simulate_answer(q)["answer"])
-
-        return m
-
-    @classmethod
-    def example_results(cls):
-        m = cls.example_model()
-        q = cls.example()
-        return q.by(m).run(cache=False)
-
     def rich_print(self):
         """Print the question in a rich format."""
         from rich.table import Table