edsl 0.1.33.dev2__py3-none-any.whl → 0.1.34__py3-none-any.whl

edsl/jobs/runners/JobsRunnerStatus.py

@@ -0,0 +1,332 @@
+from __future__ import annotations
+
+import time
+from dataclasses import dataclass, asdict
+
+from typing import List, DefaultDict, Optional, Type, Literal
+from collections import UserDict, defaultdict
+
+from rich.text import Text
+from rich.box import SIMPLE
+from rich.table import Table
+from rich.live import Live
+from rich.panel import Panel
+from rich.progress import Progress, TextColumn, BarColumn, TaskProgressColumn
+from rich.layout import Layout
+from rich.console import Group
+from rich import box
+
+from edsl.jobs.interviews.InterviewStatusDictionary import InterviewStatusDictionary
+from edsl.jobs.tokens.InterviewTokenUsage import InterviewTokenUsage
+from edsl.jobs.tokens.TokenUsage import TokenUsage
+from edsl.enums import get_token_pricing
+from edsl.jobs.tasks.task_status_enum import TaskStatus
+
+InterviewTokenUsageMapping = DefaultDict[str, InterviewTokenUsage]
+
+from edsl.jobs.interviews.InterviewStatistic import InterviewStatistic
+from edsl.jobs.interviews.InterviewStatisticsCollection import (
+    InterviewStatisticsCollection,
+)
+from edsl.jobs.tokens.InterviewTokenUsage import InterviewTokenUsage
+
+
+@dataclass
+class ModelInfo:
+    model_name: str
+    TPM_limit_k: float
+    RPM_limit_k: float
+    num_tasks_waiting: int
+    token_usage_info: dict
+
+
+@dataclass
+class ModelTokenUsageStats:
+    token_usage_type: str
+    details: List[dict]
+    cost: str
+
+
+class Stats:
+    def elapsed_time(self):
+        InterviewStatistic("elapsed_time", value=elapsed_time, digits=1, units="sec.")
+
+
+class JobsRunnerStatus:
+    def __init__(
+        self, jobs_runner: "JobsRunnerAsyncio", n: int, refresh_rate: float = 0.25
+    ):
+        self.jobs_runner = jobs_runner
+        self.start_time = time.time()
+        self.completed_interviews = []
+        self.refresh_rate = refresh_rate
+        self.statistics = [
+            "elapsed_time",
+            "total_interviews_requested",
+            "completed_interviews",
+            #            "percent_complete",
+            "average_time_per_interview",
+            #            "task_remaining",
+            "estimated_time_remaining",
+            "exceptions",
+            "unfixed_exceptions",
+            "throughput",
+        ]
+        self.num_total_interviews = n * len(self.jobs_runner.interviews)
+
+        self.distinct_models = list(
+            set(i.model.model for i in self.jobs_runner.interviews)
+        )
+
+        self.completed_interview_by_model = defaultdict(list)
+
+    def add_completed_interview(self, result):
+        self.completed_interviews.append(result.interview_hash)
+
+        relevant_model = result.model.model
+        self.completed_interview_by_model[relevant_model].append(result.interview_hash)
+
+    def _compute_statistic(self, stat_name: str):
+        completed_tasks = self.completed_interviews
+        elapsed_time = time.time() - self.start_time
+        interviews = self.jobs_runner.total_interviews
+
+        stat_definitions = {
+            "elapsed_time": lambda: InterviewStatistic(
+                "elapsed_time", value=elapsed_time, digits=1, units="sec."
+            ),
+            "total_interviews_requested": lambda: InterviewStatistic(
+                "total_interviews_requested", value=len(interviews), units=""
+            ),
+            "completed_interviews": lambda: InterviewStatistic(
+                "completed_interviews", value=len(completed_tasks), units=""
+            ),
+            "percent_complete": lambda: InterviewStatistic(
+                "percent_complete",
+                value=(
+                    len(completed_tasks) / len(interviews) * 100
+                    if len(interviews) > 0
+                    else 0
+                ),
+                digits=1,
+                units="%",
+            ),
+            "average_time_per_interview": lambda: InterviewStatistic(
+                "average_time_per_interview",
+                value=elapsed_time / len(completed_tasks) if completed_tasks else 0,
+                digits=2,
+                units="sec.",
+            ),
+            "task_remaining": lambda: InterviewStatistic(
+                "task_remaining", value=len(interviews) - len(completed_tasks), units=""
+            ),
+            "estimated_time_remaining": lambda: InterviewStatistic(
+                "estimated_time_remaining",
+                value=(
+                    (len(interviews) - len(completed_tasks))
+                    * (elapsed_time / len(completed_tasks))
+                    if len(completed_tasks) > 0
+                    else 0
+                ),
+                digits=1,
+                units="sec.",
+            ),
+            "exceptions": lambda: InterviewStatistic(
+                "exceptions",
+                value=sum(len(i.exceptions) for i in interviews),
+                units="",
+            ),
+            "unfixed_exceptions": lambda: InterviewStatistic(
+                "unfixed_exceptions",
+                value=sum(i.exceptions.num_unfixed() for i in interviews),
+                units="",
+            ),
+            "throughput": lambda: InterviewStatistic(
+                "throughput",
+                value=len(completed_tasks) / elapsed_time if elapsed_time > 0 else 0,
+                digits=2,
+                units="interviews/sec.",
+            ),
+        }
+        return stat_definitions[stat_name]()
+
+    def create_progress_bar(self):
+        return Progress(
+            TextColumn("[progress.description]{task.description}"),
+            BarColumn(),
+            TaskProgressColumn(),
+            TextColumn("{task.completed}/{task.total}"),
+        )
+
+    def generate_model_queues_table(self):
+        table = Table(show_header=False, box=box.SIMPLE)
+        table.add_column("Info", style="cyan")
+        table.add_column("Value", style="magenta")
+        # table.add_row("Bucket collection", str(self.jobs_runner.bucket_collection))
+        for model, bucket in self.jobs_runner.bucket_collection.items():
+            table.add_row(Text(model.model, style="bold blue"), "")
+            bucket_types = ["requests_bucket", "tokens_bucket"]
+            for bucket_type in bucket_types:
+                table.add_row(Text(" " + bucket_type, style="green"), "")
+                # table.add_row(
+                #     f"  Current level (capacity = {round(getattr(bucket, bucket_type).capacity, 3)})",
+                #     str(round(getattr(bucket, bucket_type).tokens, 3)),
+                # )
+                num_requests = getattr(bucket, bucket_type).num_requests
+                num_released = getattr(bucket, bucket_type).num_released
+                tokens_returned = getattr(bucket, bucket_type).tokens_returned
+                # table.add_row(
+                #     f"  Requested",
+                #     str(num_requests),
+                # )
+                # table.add_row(
+                #     f"  Completed",
+                #     str(num_released),
+                # )
+                table.add_row(
+                    "  Completed vs. Requested", f"{num_released} vs. {num_requests}"
+                )
+                table.add_row(
+                    "  Added tokens (from cache)",
+                    str(tokens_returned),
+                )
+                if bucket_type == "tokens_bucket":
+                    rate_name = "TPM"
+                else:
+                    rate_name = "RPM"
+                target_rate = round(getattr(bucket, bucket_type).target_rate, 1)
+                table.add_row(
+                    f"  Empirical {rate_name} (target = {target_rate})",
+                    str(round(getattr(bucket, bucket_type).get_throughput(), 0)),
+                )
+
+        return table
+
+    def generate_layout(self):
+        progress = self.create_progress_bar()
+        task_ids = []
+        for model in self.distinct_models:
+            task_id = progress.add_task(
+                f"[cyan]{model}...",
+                total=int(self.num_total_interviews / len(self.distinct_models)),
+            )
+            task_ids.append((model, task_id))
+
+        progress_height = min(5, 2 + len(self.distinct_models))
+        layout = Layout()
+
+        # Create the top row with only the progress panel
+        layout.split_column(
+            Layout(
+                Panel(
+                    progress,
+                    title="Interview Progress",
+                    border_style="cyan",
+                    box=box.ROUNDED,
+                ),
+                name="progress",
+                size=progress_height,  # Adjusted size
+            ),
+            Layout(name="bottom_row"),  # Adjusted size
+        )
+
+        # Split the bottom row into two columns for metrics and model queues
+        layout["bottom_row"].split_row(
+            Layout(
+                Panel(
+                    self.generate_metrics_table(),
+                    title="Metrics",
+                    border_style="magenta",
+                    box=box.ROUNDED,
+                ),
+                name="metrics",
+            ),
+            Layout(
+                Panel(
+                    self.generate_model_queues_table(),
+                    title="Model Queues",
+                    border_style="yellow",
+                    box=box.ROUNDED,
+                ),
+                name="model_queues",
+            ),
+        )
+
+        return layout, progress, task_ids
+
+    def generate_metrics_table(self):
+        table = Table(show_header=True, header_style="bold magenta", box=box.SIMPLE)
+        table.add_column("Metric", style="cyan", no_wrap=True)
+        table.add_column("Value", justify="right")
+
+        for stat_name in self.statistics:
+            pretty_name, value = list(self._compute_statistic(stat_name).items())[0]
+            # breakpoint()
+            table.add_row(pretty_name, value)
+        return table
+
+    def update_progress(self, stop_event):
+        layout, progress, task_ids = self.generate_layout()
+
+        with Live(
+            layout, refresh_per_second=int(1 / self.refresh_rate), transient=True
+        ) as live:
+            while (
+                len(self.completed_interviews) < len(self.jobs_runner.total_interviews)
+                and not stop_event.is_set()
+            ):
+                completed_tasks = len(self.completed_interviews)
+                total_tasks = len(self.jobs_runner.total_interviews)
+
+                for model, task_id in task_ids:
+                    completed_tasks = len(self.completed_interview_by_model[model])
+                    progress.update(
+                        task_id,
+                        completed=completed_tasks,
+                        description=f"[cyan]Conducting interviews for {model}...",
+                    )
+
+                layout["metrics"].update(
+                    Panel(
+                        self.generate_metrics_table(),
+                        title="Metrics",
+                        border_style="magenta",
+                        box=box.ROUNDED,
+                    )
+                )
+                layout["model_queues"].update(
+                    Panel(
+                        self.generate_model_queues_table(),
+                        title="Final Model Queues",
+                        border_style="yellow",
+                        box=box.ROUNDED,
+                    )
+                )
+
+                time.sleep(self.refresh_rate)
+
+            # Final update
+            for model, task_id in task_ids:
+                completed_tasks = len(self.completed_interview_by_model[model])
+                progress.update(
+                    task_id,
+                    completed=completed_tasks,
+                    description=f"[cyan]Conducting interviews for {model}...",
+                )
+
+            layout["metrics"].update(
+                Panel(
+                    self.generate_metrics_table(),
+                    title="Final Metrics",
+                    border_style="magenta",
+                    box=box.ROUNDED,
+                )
+            )
+            live.update(layout)
+            time.sleep(1)  # Show final state for 1 second
+
+
+if __name__ == "__main__":
+    import doctest
+
+    doctest.testmod(optionflags=doctest.ELLIPSIS)

edsl/jobs/tasks/QuestionTaskCreator.py

@@ -156,19 +156,6 @@ class QuestionTaskCreator(UserList):
             self.tokens_bucket.turbo_mode_off()
             self.requests_bucket.turbo_mode_off()
 
-        # breakpoint()
-        # _ = results.pop("cached_response", None)
-
-        # tracker = self.cached_token_usage if self.from_cache else self.new_token_usage
-
-        # TODO: This is hacky. The 'func' call should return an object that definitely has a 'usage' key.
-        # usage = results.get("usage", {"prompt_tokens": 0, "completion_tokens": 0})
-        # prompt_tokens = usage.get("prompt_tokens", 0)
-        # completion_tokens = usage.get("completion_tokens", 0)
-        # tracker.add_tokens(
-        #    prompt_tokens=prompt_tokens, completion_tokens=completion_tokens
-        # )
-
         return results
 
     @classmethod
@@ -249,6 +236,7 @@ class QuestionTaskCreator(UserList):
                 f"Required tasks failed for {self.question.question_name}"
             ) from e
 
+        # this only runs if all the dependencies are successful
         return await self._run_focal_task()
 
 

edsl/jobs/tasks/TaskHistory.py

@@ -50,6 +50,18 @@ class TaskHistory:
         """
         return [i.exceptions for k, i in self._interviews.items() if i.exceptions != {}]
 
+    @property
+    def unfixed_exceptions(self):
+        """
+        >>> len(TaskHistory.example().unfixed_exceptions)
+        4
+        """
+        return [
+            i.exceptions
+            for k, i in self._interviews.items()
+            if i.exceptions.num_unfixed() > 0
+        ]
+
     @property
     def indices(self):
         return [k for k, i in self._interviews.items() if i.exceptions != {}]
@@ -78,6 +90,11 @@ class TaskHistory:
         """
         return len(self.exceptions) > 0
 
+    @property
+    def has_unfixed_exceptions(self) -> bool:
+        """Return True if there are any exceptions."""
+        return len(self.unfixed_exceptions) > 0
+
     def _repr_html_(self):
         """Return an HTML representation of the TaskHistory."""
         from edsl.utilities.utilities import data_to_html

edsl/language_models/LanguageModel.py

@@ -164,20 +164,20 @@ class LanguageModel(
         None  # This should be something like ["choices", 0, "message", "content"]
     )
     __rate_limits = None
-    __default_rate_limits = {
-        "rpm": 10_000,
-        "tpm": 2_000_000,
-    }  # TODO: Use the OpenAI Teir 1 rate limits
     _safety_factor = 0.8
 
-    def __init__(self, tpm=None, rpm=None, **kwargs):
+    def __init__(
+        self, tpm=None, rpm=None, omit_system_prompt_if_empty_string=True, **kwargs
+    ):
         """Initialize the LanguageModel."""
         self.model = getattr(self, "_model_", None)
         default_parameters = getattr(self, "_parameters_", None)
         parameters = self._overide_default_parameters(kwargs, default_parameters)
         self.parameters = parameters
         self.remote = False
+        self.omit_system_prompt_if_empty = omit_system_prompt_if_empty_string
 
+        # self._rpm / _tpm comes from the class
         if rpm is not None:
             self._rpm = rpm
 
@@ -286,35 +286,40 @@ class LanguageModel(
         >>> m.RPM
         100
         """
-        self._set_rate_limits(rpm=rpm, tpm=tpm)
+        if rpm is not None:
+            self._rpm = rpm
+        if tpm is not None:
+            self._tpm = tpm
+        return None
+        # self._set_rate_limits(rpm=rpm, tpm=tpm)
 
-    def _set_rate_limits(self, rpm=None, tpm=None) -> None:
-        """Set the rate limits for the model.
+    # def _set_rate_limits(self, rpm=None, tpm=None) -> None:
+    #     """Set the rate limits for the model.
 
-        If the model does not have rate limits, use the default rate limits."""
-        if rpm is not None and tpm is not None:
-            self.__rate_limits = {"rpm": rpm, "tpm": tpm}
-            return
+    #     If the model does not have rate limits, use the default rate limits."""
+    #     if rpm is not None and tpm is not None:
+    #         self.__rate_limits = {"rpm": rpm, "tpm": tpm}
+    #         return
 
-        if self.__rate_limits is None:
-            if hasattr(self, "get_rate_limits"):
-                self.__rate_limits = self.get_rate_limits()
-            else:
-                self.__rate_limits = self.__default_rate_limits
+    #     if self.__rate_limits is None:
+    #         if hasattr(self, "get_rate_limits"):
+    #             self.__rate_limits = self.get_rate_limits()
+    #         else:
+    #             self.__rate_limits = self.__default_rate_limits
 
     @property
     def RPM(self):
         """Model's requests-per-minute limit."""
         # self._set_rate_limits()
         # return self._safety_factor * self.__rate_limits["rpm"]
-        return self.rpm
+        return self._rpm
 
     @property
     def TPM(self):
         """Model's tokens-per-minute limit."""
         # self._set_rate_limits()
         # return self._safety_factor * self.__rate_limits["tpm"]
-        return self.tpm
+        return self._tpm
 
     @property
     def rpm(self):
@@ -332,17 +337,6 @@ class LanguageModel(
     def tpm(self, value):
         self._tpm = value
 
-    @property
-    def TPM(self):
-        """Model's tokens-per-minute limit.
-
-        >>> m = LanguageModel.example()
-        >>> m.TPM > 0
-        True
-        """
-        self._set_rate_limits()
-        return self._safety_factor * self.__rate_limits["tpm"]
-
     @staticmethod
     def _overide_default_parameters(passed_parameter_dict, default_parameter_dict):
         """Return a dictionary of parameters, with passed parameters taking precedence over defaults.
@@ -446,7 +440,7 @@ class LanguageModel(
         system_prompt: str,
         cache: "Cache",
         iteration: int = 0,
-        encoded_image=None,
+        files_list=None,
     ) -> ModelResponse:
         """Handle caching of responses.
 
@@ -468,15 +462,18 @@ class LanguageModel(
         >>> m._get_intended_model_call_outcome(user_prompt = "Hello", system_prompt = "hello", cache = Cache())
         ModelResponse(...)"""
 
-        if encoded_image:
-            # the image has is appended to the user_prompt for hash-lookup purposes
-            image_hash = hashlib.md5(encoded_image.encode()).hexdigest()
+        if files_list:
+            files_hash = "+".join([str(hash(file)) for file in files_list])
+            # print(f"Files hash: {files_hash}")
+            user_prompt_with_hashes = user_prompt + f" {files_hash}"
+        else:
+            user_prompt_with_hashes = user_prompt
 
         cache_call_params = {
             "model": str(self.model),
             "parameters": self.parameters,
             "system_prompt": system_prompt,
-            "user_prompt": user_prompt + "" if not encoded_image else f" {image_hash}",
+            "user_prompt": user_prompt_with_hashes,
             "iteration": iteration,
         }
         cached_response, cache_key = cache.fetch(**cache_call_params)
@@ -492,7 +489,8 @@ class LanguageModel(
             params = {
                 "user_prompt": user_prompt,
                 "system_prompt": system_prompt,
-                **({"encoded_image": encoded_image} if encoded_image else {}),
+                "files_list": files_list
+                # **({"encoded_image": encoded_image} if encoded_image else {}),
             }
             # response = await f(**params)
             response = await asyncio.wait_for(f(**params), timeout=TIMEOUT)
@@ -536,7 +534,7 @@ class LanguageModel(
         system_prompt: str,
         cache: "Cache",
         iteration: int = 1,
-        encoded_image=None,
+        files_list: Optional[List["File"]] = None,
     ) -> dict:
         """Get response, parse, and return as string.
 
@@ -552,7 +550,7 @@ class LanguageModel(
             "system_prompt": system_prompt,
             "iteration": iteration,
             "cache": cache,
-            **({"encoded_image": encoded_image} if encoded_image else {}),
+            "files_list": files_list,
         }
         model_inputs = ModelInputs(user_prompt=user_prompt, system_prompt=system_prompt)
         model_outputs = await self._async_get_intended_model_call_outcome(**params)

edsl/language_models/registry.py

@@ -2,10 +2,10 @@ import textwrap
 from random import random
 from edsl.config import CONFIG
 
-if "EDSL_DEFAULT_MODEL" not in CONFIG:
-    default_model = "test"
-else:
-    default_model = CONFIG.get("EDSL_DEFAULT_MODEL")
+# if "EDSL_DEFAULT_MODEL" not in CONFIG:
+#     default_model = "test"
+# else:
+#     default_model = CONFIG.get("EDSL_DEFAULT_MODEL")
 
 
 def get_model_class(model_name, registry=None):
@@ -33,20 +33,24 @@ class Meta(type):
 
 
 class Model(metaclass=Meta):
-    default_model = default_model
+    default_model = CONFIG.get("EDSL_DEFAULT_MODEL")
 
-    def __new__(cls, model_name=None, registry=None, *args, **kwargs):
+    def __new__(
+        cls, model_name=None, registry=None, service_name=None, *args, **kwargs
+    ):
         # Map index to the respective subclass
         if model_name is None:
-            model_name = cls.default_model
+            model_name = (
+                cls.default_model
+            )  # when model_name is None, use the default model, set in the config file
         from edsl.inference_services.registry import default
 
         registry = registry or default
 
-        if isinstance(model_name, int):
+        if isinstance(model_name, int):  # can refer to a model by index
             model_name = cls.available(name_only=True)[model_name]
 
-        factory = registry.create_model_factory(model_name)
+        factory = registry.create_model_factory(model_name, service_name=service_name)
         return factory(*args, **kwargs)
 
     @classmethod

edsl/language_models/utilities.py

@@ -1,5 +1,5 @@
 import asyncio
-from typing import Any
+from typing import Any, Optional, List
 from edsl import Survey
 from edsl.config import CONFIG
 from edsl.enums import InferenceServiceType
@@ -40,7 +40,10 @@ def create_language_model(
         _tpm = 1000000000000
 
         async def async_execute_model_call(
-            self, user_prompt: str, system_prompt: str
+            self,
+            user_prompt: str,
+            system_prompt: str,
+            files_list: Optional[List[Any]] = None,
         ) -> dict[str, Any]:
             question_number = int(
                 user_prompt.split("XX")[1]