edsl 0.1.33__py3-none-any.whl → 0.1.33.dev1__py3-none-any.whl

edsl/jobs/runners/JobsRunnerStatusMixin.py

@@ -0,0 +1,333 @@
+from __future__ import annotations
+from typing import List, DefaultDict
+import asyncio
+from typing import Type
+from collections import defaultdict
+
+from typing import Literal, List, Type, DefaultDict
+from collections import UserDict, defaultdict
+
+from edsl.jobs.interviews.InterviewStatusDictionary import InterviewStatusDictionary
+from edsl.jobs.tokens.InterviewTokenUsage import InterviewTokenUsage
+from edsl.jobs.tokens.TokenUsage import TokenUsage
+from edsl.enums import get_token_pricing
+from edsl.jobs.tasks.task_status_enum import TaskStatus
+
+InterviewTokenUsageMapping = DefaultDict[str, InterviewTokenUsage]
+
+from edsl.jobs.interviews.InterviewStatistic import InterviewStatistic
+from edsl.jobs.interviews.InterviewStatisticsCollection import (
+    InterviewStatisticsCollection,
+)
+from edsl.jobs.tokens.InterviewTokenUsage import InterviewTokenUsage
+
+
+# return {"cache_status": token_usage_type, "details": details, "cost": f"${token_usage.cost(prices):.5f}"}
+
+from dataclasses import dataclass, asdict
+
+from rich.text import Text
+from rich.box import SIMPLE
+from rich.table import Table
+
+
+@dataclass
+class ModelInfo:
+    model_name: str
+    TPM_limit_k: float
+    RPM_limit_k: float
+    num_tasks_waiting: int
+    token_usage_info: dict
+
+
+@dataclass
+class ModelTokenUsageStats:
+    token_usage_type: str
+    details: List[dict]
+    cost: str
+
+
+class Stats:
+    def elapsed_time(self):
+        InterviewStatistic("elapsed_time", value=elapsed_time, digits=1, units="sec.")
+
+
+class JobsRunnerStatusMixin:
+    # @staticmethod
+    # def status_dict(interviews: List[Type["Interview"]]) -> List[Type[InterviewStatusDictionary]]:
+    #     """
+    #     >>> from edsl.jobs.interviews.Interview import Interview
+    #     >>> interviews = [Interview.example()]
+    #     >>> JobsRunnerStatusMixin().status_dict(interviews)
+    #     [InterviewStatusDictionary({<TaskStatus.NOT_STARTED: 1>: 0, <TaskStatus.WAITING_FOR_DEPENDENCIES: 2>: 0, <TaskStatus.CANCELLED: 3>: 0, <TaskStatus.PARENT_FAILED: 4>: 0, <TaskStatus.WAITING_FOR_REQUEST_CAPACITY: 5>: 0, <TaskStatus.WAITING_FOR_TOKEN_CAPACITY: 6>: 0, <TaskStatus.API_CALL_IN_PROGRESS: 7>: 0, <TaskStatus.SUCCESS: 8>: 0, <TaskStatus.FAILED: 9>: 0, 'number_from_cache': 0})]
+    #     """
+    #     return [interview.interview_status for interview in interviews]
+
+    def _compute_statistic(stat_name: str, completed_tasks, elapsed_time, interviews):
+        stat_definitions = {
+            "elapsed_time": lambda: InterviewStatistic(
+                "elapsed_time", value=elapsed_time, digits=1, units="sec."
+            ),
+            "total_interviews_requested": lambda: InterviewStatistic(
+                "total_interviews_requested", value=len(interviews), units=""
+            ),
+            "completed_interviews": lambda: InterviewStatistic(
+                "completed_interviews", value=len(completed_tasks), units=""
+            ),
+            "percent_complete": lambda: InterviewStatistic(
+                "percent_complete",
+                value=(
+                    len(completed_tasks) / len(interviews) * 100
+                    if len(interviews) > 0
+                    else "NA"
+                ),
+                digits=0,
+                units="%",
+            ),
+            "average_time_per_interview": lambda: InterviewStatistic(
+                "average_time_per_interview",
+                value=elapsed_time / len(completed_tasks) if completed_tasks else "NA",
+                digits=1,
+                units="sec.",
+            ),
+            "task_remaining": lambda: InterviewStatistic(
+                "task_remaining", value=len(interviews) - len(completed_tasks), units=""
+            ),
+            "estimated_time_remaining": lambda: InterviewStatistic(
+                "estimated_time_remaining",
+                value=(
+                    (len(interviews) - len(completed_tasks))
+                    * (elapsed_time / len(completed_tasks))
+                    if len(completed_tasks) > 0
+                    else "NA"
+                ),
+                digits=1,
+                units="sec.",
+            ),
+        }
+        if stat_name not in stat_definitions:
+            raise ValueError(
+                f"Invalid stat_name: {stat_name}. The valid stat_names are: {list(stat_definitions.keys())}"
+            )
+        return stat_definitions[stat_name]()
+
+    @staticmethod
+    def _job_level_info(
+        completed_tasks: List[Type[asyncio.Task]],
+        elapsed_time: float,
+        interviews: List[Type["Interview"]],
+    ) -> InterviewStatisticsCollection:
+        interview_statistics = InterviewStatisticsCollection()
+
+        default_statistics = [
+            "elapsed_time",
+            "total_interviews_requested",
+            "completed_interviews",
+            "percent_complete",
+            "average_time_per_interview",
+            "task_remaining",
+            "estimated_time_remaining",
+        ]
+        for stat_name in default_statistics:
+            interview_statistics.add_stat(
+                JobsRunnerStatusMixin._compute_statistic(
+                    stat_name, completed_tasks, elapsed_time, interviews
+                )
+            )
+
+        return interview_statistics
+
+    @staticmethod
+    def _get_model_queues_info(interviews):
+        models_to_tokens = defaultdict(InterviewTokenUsage)
+        model_to_status = defaultdict(InterviewStatusDictionary)
+        waiting_dict = defaultdict(int)
+
+        for interview in interviews:
+            models_to_tokens[interview.model] += interview.token_usage
+            model_to_status[interview.model] += interview.interview_status
+            waiting_dict[interview.model] += interview.interview_status.waiting
+
+        for model, num_waiting in waiting_dict.items():
+            yield JobsRunnerStatusMixin._get_model_info(
+                model, num_waiting, models_to_tokens
+            )
+
+    @staticmethod
+    def generate_status_summary(
+        completed_tasks: List[Type[asyncio.Task]],
+        elapsed_time: float,
+        interviews: List[Type["Interview"]],
+        include_model_queues=False,
+    ) -> InterviewStatisticsCollection:
+        """Generate a summary of the status of the job runner.
+
+        :param completed_tasks: list of completed tasks
+        :param elapsed_time: time elapsed since the start of the job
+        :param interviews: list of interviews to be conducted
+
+        >>> from edsl.jobs.interviews.Interview import Interview
+        >>> interviews = [Interview.example()]
+        >>> completed_tasks = []
+        >>> elapsed_time = 0
+        >>> JobsRunnerStatusMixin().generate_status_summary(completed_tasks, elapsed_time, interviews)
+        {'Elapsed time': '0.0 sec.', 'Total interviews requested': '1 ', 'Completed interviews': '0 ', 'Percent complete': '0 %', 'Average time per interview': 'NA', 'Task remaining': '1 ', 'Estimated time remaining': 'NA'}
+        """
+
+        interview_status_summary: InterviewStatisticsCollection = (
+            JobsRunnerStatusMixin._job_level_info(
+                completed_tasks=completed_tasks,
+                elapsed_time=elapsed_time,
+                interviews=interviews,
+            )
+        )
+        if include_model_queues:
+            interview_status_summary.model_queues = list(
+                JobsRunnerStatusMixin._get_model_queues_info(interviews)
+            )
+        else:
+            interview_status_summary.model_queues = None
+
+        return interview_status_summary
+
+    @staticmethod
+    def _get_model_info(
+        model: str,
+        num_waiting: int,
+        models_to_tokens: InterviewTokenUsageMapping,
+    ) -> dict:
+        """Get the status of a model.
+
+        :param model: the model name
+        :param num_waiting: the number of tasks waiting for capacity
+        :param models_to_tokens: a mapping of models to token usage
+
+        >>> from edsl.jobs.interviews.Interview import Interview
+        >>> interviews = [Interview.example()]
+        >>> models_to_tokens = defaultdict(InterviewTokenUsage)
+        >>> model = interviews[0].model
+        >>> num_waiting = 0
+        >>> JobsRunnerStatusMixin()._get_model_info(model, num_waiting, models_to_tokens)
+        ModelInfo(model_name='gpt-4-1106-preview', TPM_limit_k=480.0, RPM_limit_k=4.0, num_tasks_waiting=0, token_usage_info=[ModelTokenUsageStats(token_usage_type='new_token_usage', details=[{'type': 'prompt_tokens', 'tokens': 0}, {'type': 'completion_tokens', 'tokens': 0}], cost='$0.00000'), ModelTokenUsageStats(token_usage_type='cached_token_usage', details=[{'type': 'prompt_tokens', 'tokens': 0}, {'type': 'completion_tokens', 'tokens': 0}], cost='$0.00000')])
+        """
+
+        ## TODO: This should probably be a coop method
+        prices = get_token_pricing(model.model)
+
+        token_usage_info = []
+        for token_usage_type in ["new_token_usage", "cached_token_usage"]:
+            token_usage_info.append(
+                JobsRunnerStatusMixin._get_token_usage_info(
+                    token_usage_type, models_to_tokens, model, prices
+                )
+            )
+
+        return ModelInfo(
+            **{
+                "model_name": model.model,
+                "TPM_limit_k": model.TPM / 1000,
+                "RPM_limit_k": model.RPM / 1000,
+                "num_tasks_waiting": num_waiting,
+                "token_usage_info": token_usage_info,
+            }
+        )
+
+    @staticmethod
+    def _get_token_usage_info(
+        token_usage_type: Literal["new_token_usage", "cached_token_usage"],
+        models_to_tokens: InterviewTokenUsageMapping,
+        model: str,
+        prices: "TokenPricing",
+    ) -> ModelTokenUsageStats:
+        """Get the token usage info for a model.
+
+        >>> from edsl.jobs.interviews.Interview import Interview
+        >>> interviews = [Interview.example()]
+        >>> models_to_tokens = defaultdict(InterviewTokenUsage)
+        >>> model = interviews[0].model
+        >>> prices = get_token_pricing(model.model)
+        >>> cache_status = "new_token_usage"
+        >>> JobsRunnerStatusMixin()._get_token_usage_info(cache_status, models_to_tokens, model, prices)
+        ModelTokenUsageStats(token_usage_type='new_token_usage', details=[{'type': 'prompt_tokens', 'tokens': 0}, {'type': 'completion_tokens', 'tokens': 0}], cost='$0.00000')
+
+        """
+        all_token_usage: InterviewTokenUsage = models_to_tokens[model]
+        token_usage: TokenUsage = getattr(all_token_usage, token_usage_type)
+
+        details = [
+            {"type": token_type, "tokens": getattr(token_usage, token_type)}
+            for token_type in ["prompt_tokens", "completion_tokens"]
+        ]
+
+        return ModelTokenUsageStats(
+            token_usage_type=token_usage_type,
+            details=details,
+            cost=f"${token_usage.cost(prices):.5f}",
+        )
+
+    @staticmethod
+    def _add_statistics_to_table(table, status_summary):
+        table.add_column("Statistic", style="dim", no_wrap=True, width=50)
+        table.add_column("Value", width=10)
+
+        for key, value in status_summary.items():
+            if key != "model_queues":
+                table.add_row(key, value)
+
+    @staticmethod
+    def display_status_table(status_summary: InterviewStatisticsCollection) -> "Table":
+        table = Table(
+            title="Job Status",
+            show_header=True,
+            header_style="bold magenta",
+            box=SIMPLE,
+        )
+
+        ### Job-level statistics
+        JobsRunnerStatusMixin._add_statistics_to_table(table, status_summary)
+
+        ## Model-level statistics
+        spacing = " "
+
+        if status_summary.model_queues is not None:
+            table.add_row(Text("Model Queues", style="bold red"), "")
+            for model_info in status_summary.model_queues:
+                model_name = model_info.model_name
+                tpm = f"TPM (k)={model_info.TPM_limit_k}"
+                rpm = f"RPM (k)= {model_info.RPM_limit_k}"
+                pretty_model_name = model_name + ";" + tpm + ";" + rpm
+                table.add_row(Text(pretty_model_name, style="blue"), "")
+                table.add_row(
+                    "Number question tasks waiting for capacity",
+                    str(model_info.num_tasks_waiting),
+                )
+                # Token usage and cost info
+                for token_usage_info in model_info.token_usage_info:
+                    token_usage_type = token_usage_info.token_usage_type
+                    table.add_row(
+                        Text(
+                            spacing + token_usage_type.replace("_", " "), style="bold"
+                        ),
+                        "",
+                    )
+                    for detail in token_usage_info.details:
+                        token_type = detail["type"]
+                        tokens = detail["tokens"]
+                        table.add_row(spacing + f"{token_type}", f"{tokens:,}")
+                    # table.add_row(spacing + "cost", cache_info["cost"])
+
+        return table
+
+    def status_table(self, completed_tasks: List[asyncio.Task], elapsed_time: float):
+        summary_data = JobsRunnerStatusMixin.generate_status_summary(
+            completed_tasks=completed_tasks,
+            elapsed_time=elapsed_time,
+            interviews=self.total_interviews,
+        )
+        return self.display_status_table(summary_data)
+
+
+if __name__ == "__main__":
+    import doctest
+
+    doctest.testmod(optionflags=doctest.ELLIPSIS)

edsl/jobs/tasks/QuestionTaskCreator.py

@@ -55,7 +55,6 @@ class QuestionTaskCreator(UserList):
 
         """
         super().__init__([])
-        # answer_question_func is the 'interview.answer_question_and_record_task" method
         self.answer_question_func = answer_question_func
         self.question = question
         self.iteration = iteration
@@ -88,10 +87,10 @@ class QuestionTaskCreator(UserList):
         """
         self.append(task)
 
-    def generate_task(self) -> asyncio.Task:
+    def generate_task(self, debug: bool) -> asyncio.Task:
         """Create a task that depends on the passed-in dependencies."""
         task = asyncio.create_task(
-            self._run_task_async(), name=self.question.question_name
+            self._run_task_async(debug), name=self.question.question_name
         )
         task.depends_on = [t.get_name() for t in self]
         return task
@@ -104,7 +103,7 @@ class QuestionTaskCreator(UserList):
         """Returns the token usage for the task.
 
         >>> qt = QuestionTaskCreator.example()
-        >>> answers = asyncio.run(qt._run_focal_task())
+        >>> answers = asyncio.run(qt._run_focal_task(debug=False))
         >>> qt.token_usage()
         {'cached_tokens': TokenUsage(from_cache=True, prompt_tokens=0, completion_tokens=0), 'new_tokens': TokenUsage(from_cache=False, prompt_tokens=0, completion_tokens=0)}
         """
@@ -112,15 +111,15 @@ class QuestionTaskCreator(UserList):
             cached_tokens=self.cached_token_usage, new_tokens=self.new_token_usage
         )
 
-    async def _run_focal_task(self) -> Answers:
+    async def _run_focal_task(self, debug: bool) -> Answers:
         """Run the focal task i.e., the question that we are interested in answering.
 
         It is only called after all the dependency tasks are completed.
 
         >>> qt = QuestionTaskCreator.example()
-        >>> answers = asyncio.run(qt._run_focal_task())
-        >>> answers.answer
-        'This is an example answer'
+        >>> answers = asyncio.run(qt._run_focal_task(debug=False))
+        >>> answers["answer"]
+        'Yo!'
         """
 
         requested_tokens = self.estimated_tokens()
@@ -133,19 +132,19 @@ class QuestionTaskCreator(UserList):
             self.waiting = True
             self.task_status = TaskStatus.WAITING_FOR_REQUEST_CAPACITY
 
-        await self.requests_bucket.get_tokens(1, cheat_bucket_capacity=True)
+        await self.tokens_bucket.get_tokens(1)
 
         self.task_status = TaskStatus.API_CALL_IN_PROGRESS
         try:
             results = await self.answer_question_func(
-                question=self.question, task=None  # self
+                question=self.question, debug=debug, task=None  # self
             )
             self.task_status = TaskStatus.SUCCESS
         except Exception as e:
             self.task_status = TaskStatus.FAILED
             raise e
 
-        if results.cache_used:
+        if results.get("cache_used", False):
             self.tokens_bucket.add_tokens(requested_tokens)
             self.requests_bucket.add_tokens(1)
             self.from_cache = True
@@ -156,18 +155,17 @@ class QuestionTaskCreator(UserList):
             self.tokens_bucket.turbo_mode_off()
             self.requests_bucket.turbo_mode_off()
 
-        # breakpoint()
-        # _ = results.pop("cached_response", None)
+        _ = results.pop("cached_response", None)
 
-        # tracker = self.cached_token_usage if self.from_cache else self.new_token_usage
+        tracker = self.cached_token_usage if self.from_cache else self.new_token_usage
 
         # TODO: This is hacky. The 'func' call should return an object that definitely has a 'usage' key.
-        # usage = results.get("usage", {"prompt_tokens": 0, "completion_tokens": 0})
-        # prompt_tokens = usage.get("prompt_tokens", 0)
-        # completion_tokens = usage.get("completion_tokens", 0)
-        # tracker.add_tokens(
-        #    prompt_tokens=prompt_tokens, completion_tokens=completion_tokens
-        # )
+        usage = results.get("usage", {"prompt_tokens": 0, "completion_tokens": 0})
+        prompt_tokens = usage.get("prompt_tokens", 0)
+        completion_tokens = usage.get("completion_tokens", 0)
+        tracker.add_tokens(
+            prompt_tokens=prompt_tokens, completion_tokens=completion_tokens
+        )
 
         return results
 
@@ -179,13 +177,8 @@ class QuestionTaskCreator(UserList):
 
         m = ModelBuckets.infinity_bucket()
 
-        from collections import namedtuple
-
-        AnswerDict = namedtuple("AnswerDict", ["answer", "cache_used"])
-        answer = AnswerDict(answer="This is an example answer", cache_used=False)
-
-        async def answer_question_func(question, task):
-            return answer
+        async def answer_question_func(question, debug, task):
+            return {"answer": "Yo!"}
 
         return cls(
             question=QuestionFreeText.example(),
@@ -195,7 +188,7 @@ class QuestionTaskCreator(UserList):
             iteration=0,
         )
 
-    async def _run_task_async(self) -> None:
+    async def _run_task_async(self, debug) -> None:
         """Run the task asynchronously, awaiting the tasks that must be completed before this one can be run.
 
         >>> qt1 = QuestionTaskCreator.example()
@@ -238,6 +231,8 @@ class QuestionTaskCreator(UserList):
                 if isinstance(result, Exception):
                     raise result
 
+            return await self._run_focal_task(debug)
+
         except asyncio.CancelledError:
             self.task_status = TaskStatus.CANCELLED
             raise
@@ -249,8 +244,6 @@ class QuestionTaskCreator(UserList):
                 f"Required tasks failed for {self.question.question_name}"
             ) from e
 
-        return await self._run_focal_task()
-
 
 if __name__ == "__main__":
     import doctest