edsl 0.1.33.dev2__py3-none-any.whl → 0.1.33.dev3__py3-none-any.whl

edsl/jobs/runners/JobsRunnerStatus.py

@@ -0,0 +1,331 @@
+from __future__ import annotations
+
+import time
+from dataclasses import dataclass, asdict
+
+from typing import List, DefaultDict, Optional, Type, Literal
+from collections import UserDict, defaultdict
+
+from rich.text import Text
+from rich.box import SIMPLE
+from rich.table import Table
+from rich.live import Live
+from rich.panel import Panel
+from rich.progress import Progress, TextColumn, BarColumn, TaskProgressColumn
+from rich.layout import Layout
+from rich.console import Group
+from rich import box
+
+from edsl.jobs.interviews.InterviewStatusDictionary import InterviewStatusDictionary
+from edsl.jobs.tokens.InterviewTokenUsage import InterviewTokenUsage
+from edsl.jobs.tokens.TokenUsage import TokenUsage
+from edsl.enums import get_token_pricing
+from edsl.jobs.tasks.task_status_enum import TaskStatus
+
+InterviewTokenUsageMapping = DefaultDict[str, InterviewTokenUsage]
+
+from edsl.jobs.interviews.InterviewStatistic import InterviewStatistic
+from edsl.jobs.interviews.InterviewStatisticsCollection import (
+    InterviewStatisticsCollection,
+)
+from edsl.jobs.tokens.InterviewTokenUsage import InterviewTokenUsage
+
+
+@dataclass
+class ModelInfo:
+    model_name: str
+    TPM_limit_k: float
+    RPM_limit_k: float
+    num_tasks_waiting: int
+    token_usage_info: dict
+
+
+@dataclass
+class ModelTokenUsageStats:
+    token_usage_type: str
+    details: List[dict]
+    cost: str
+
+
+class Stats:
+    def elapsed_time(self):
+        InterviewStatistic("elapsed_time", value=elapsed_time, digits=1, units="sec.")
+
+
+class JobsRunnerStatus:
+    def __init__(
+        self, jobs_runner: "JobsRunnerAsyncio", n: int, refresh_rate: float = 0.25
+    ):
+        self.jobs_runner = jobs_runner
+        self.start_time = time.time()
+        self.completed_interviews = []
+        self.refresh_rate = refresh_rate
+        self.statistics = [
+            "elapsed_time",
+            "total_interviews_requested",
+            "completed_interviews",
+            #            "percent_complete",
+            "average_time_per_interview",
+            #            "task_remaining",
+            "estimated_time_remaining",
+            "exceptions",
+            "unfixed_exceptions",
+            "throughput",
+        ]
+        self.num_total_interviews = n * len(self.jobs_runner.interviews)
+
+        self.distinct_models = list(
+            set(i.model.model for i in self.jobs_runner.interviews)
+        )
+
+        self.completed_interview_by_model = defaultdict(list)
+
+    def add_completed_interview(self, result):
+        self.completed_interviews.append(result.interview_hash)
+
+        relevant_model = result.model.model
+        self.completed_interview_by_model[relevant_model].append(result.interview_hash)
+
+    def _compute_statistic(self, stat_name: str):
+        completed_tasks = self.completed_interviews
+        elapsed_time = time.time() - self.start_time
+        interviews = self.jobs_runner.total_interviews
+
+        stat_definitions = {
+            "elapsed_time": lambda: InterviewStatistic(
+                "elapsed_time", value=elapsed_time, digits=1, units="sec."
+            ),
+            "total_interviews_requested": lambda: InterviewStatistic(
+                "total_interviews_requested", value=len(interviews), units=""
+            ),
+            "completed_interviews": lambda: InterviewStatistic(
+                "completed_interviews", value=len(completed_tasks), units=""
+            ),
+            "percent_complete": lambda: InterviewStatistic(
+                "percent_complete",
+                value=(
+                    len(completed_tasks) / len(interviews) * 100
+                    if len(interviews) > 0
+                    else 0
+                ),
+                digits=1,
+                units="%",
+            ),
+            "average_time_per_interview": lambda: InterviewStatistic(
+                "average_time_per_interview",
+                value=elapsed_time / len(completed_tasks) if completed_tasks else 0,
+                digits=2,
+                units="sec.",
+            ),
+            "task_remaining": lambda: InterviewStatistic(
+                "task_remaining", value=len(interviews) - len(completed_tasks), units=""
+            ),
+            "estimated_time_remaining": lambda: InterviewStatistic(
+                "estimated_time_remaining",
+                value=(
+                    (len(interviews) - len(completed_tasks))
+                    * (elapsed_time / len(completed_tasks))
+                    if len(completed_tasks) > 0
+                    else 0
+                ),
+                digits=1,
+                units="sec.",
+            ),
+            "exceptions": lambda: InterviewStatistic(
+                "exceptions",
+                value=sum(len(i.exceptions) for i in interviews),
+                units="",
+            ),
+            "unfixed_exceptions": lambda: InterviewStatistic(
+                "unfixed_exceptions",
+                value=sum(i.exceptions.num_unfixed() for i in interviews),
+                units="",
+            ),
+            "throughput": lambda: InterviewStatistic(
+                "throughput",
+                value=len(completed_tasks) / elapsed_time if elapsed_time > 0 else 0,
+                digits=2,
+                units="interviews/sec.",
+            ),
+        }
+        return stat_definitions[stat_name]()
+
+    def create_progress_bar(self):
+        return Progress(
+            TextColumn("[progress.description]{task.description}"),
+            BarColumn(),
+            TaskProgressColumn(),
+            TextColumn("{task.completed}/{task.total}"),
+        )
+
+    def generate_model_queues_table(self):
+        table = Table(show_header=False, box=box.SIMPLE)
+        table.add_column("Info", style="cyan")
+        table.add_column("Value", style="magenta")
+        # table.add_row("Bucket collection", str(self.jobs_runner.bucket_collection))
+        for model, bucket in self.jobs_runner.bucket_collection.items():
+            table.add_row(Text(model.model, style="bold blue"), "")
+            bucket_types = ["requests_bucket", "tokens_bucket"]
+            for bucket_type in bucket_types:
+                table.add_row(Text(" " + bucket_type, style="green"), "")
+                # table.add_row(
+                #     f"  Current level (capacity = {round(getattr(bucket, bucket_type).capacity, 3)})",
+                #     str(round(getattr(bucket, bucket_type).tokens, 3)),
+                # )
+                num_requests = getattr(bucket, bucket_type).num_requests
+                num_released = getattr(bucket, bucket_type).num_released
+                tokens_returned = getattr(bucket, bucket_type).tokens_returned
+                # table.add_row(
+                #     f"  Requested",
+                #     str(num_requests),
+                # )
+                # table.add_row(
+                #     f"  Completed",
+                #     str(num_released),
+                # )
+                table.add_row(
+                    "  Completed vs. Requested", f"{num_released} vs. {num_requests}"
+                )
+                table.add_row(
+                    "  Added tokens (from cache)",
+                    str(tokens_returned),
+                )
+                if bucket_type == "tokens_bucket":
+                    rate_name = "TPM"
+                else:
+                    rate_name = "RPM"
+                target_rate = round(getattr(bucket, bucket_type).target_rate, 1)
+                table.add_row(
+                    f"  Empirical {rate_name} (target = {target_rate})",
+                    str(round(getattr(bucket, bucket_type).get_throughput(), 0)),
+                )
+
+        return table
+
+    def generate_layout(self):
+        progress = self.create_progress_bar()
+        task_ids = []
+        for model in self.distinct_models:
+            task_id = progress.add_task(
+                f"[cyan]{model}...",
+                total=int(self.num_total_interviews / len(self.distinct_models)),
+            )
+            task_ids.append((model, task_id))
+
+        progress_height = min(5, 2 + len(self.distinct_models))
+        layout = Layout()
+
+        # Create the top row with only the progress panel
+        layout.split_column(
+            Layout(
+                Panel(
+                    progress,
+                    title="Interview Progress",
+                    border_style="cyan",
+                    box=box.ROUNDED,
+                ),
+                name="progress",
+                size=progress_height,  # Adjusted size
+            ),
+            Layout(name="bottom_row"),  # Adjusted size
+        )
+
+        # Split the bottom row into two columns for metrics and model queues
+        layout["bottom_row"].split_row(
+            Layout(
+                Panel(
+                    self.generate_metrics_table(),
+                    title="Metrics",
+                    border_style="magenta",
+                    box=box.ROUNDED,
+                ),
+                name="metrics",
+            ),
+            Layout(
+                Panel(
+                    self.generate_model_queues_table(),
+                    title="Model Queues",
+                    border_style="yellow",
+                    box=box.ROUNDED,
+                ),
+                name="model_queues",
+            ),
+        )
+
+        return layout, progress, task_ids
+
+    def generate_metrics_table(self):
+        table = Table(show_header=True, header_style="bold magenta", box=box.SIMPLE)
+        table.add_column("Metric", style="cyan", no_wrap=True)
+        table.add_column("Value", justify="right")
+
+        for stat_name in self.statistics:
+            pretty_name, value = list(self._compute_statistic(stat_name).items())[0]
+            # breakpoint()
+            table.add_row(pretty_name, value)
+        return table
+
+    def update_progress(self):
+        layout, progress, task_ids = self.generate_layout()
+
+        with Live(
+            layout, refresh_per_second=int(1 / self.refresh_rate), transient=True
+        ) as live:
+            while len(self.completed_interviews) < len(
+                self.jobs_runner.total_interviews
+            ):
+                completed_tasks = len(self.completed_interviews)
+                total_tasks = len(self.jobs_runner.total_interviews)
+
+                for model, task_id in task_ids:
+                    completed_tasks = len(self.completed_interview_by_model[model])
+                    progress.update(
+                        task_id,
+                        completed=completed_tasks,
+                        description=f"[cyan]Conducting interviews for {model}...",
+                    )
+
+                layout["metrics"].update(
+                    Panel(
+                        self.generate_metrics_table(),
+                        title="Metrics",
+                        border_style="magenta",
+                        box=box.ROUNDED,
+                    )
+                )
+                layout["model_queues"].update(
+                    Panel(
+                        self.generate_model_queues_table(),
+                        title="Final Model Queues",
+                        border_style="yellow",
+                        box=box.ROUNDED,
+                    )
+                )
+
+                time.sleep(self.refresh_rate)
+
+            # Final update
+            for model, task_id in task_ids:
+                completed_tasks = len(self.completed_interview_by_model[model])
+                progress.update(
+                    task_id,
+                    completed=completed_tasks,
+                    description=f"[cyan]Conducting interviews for {model}...",
+                )
+
+            layout["metrics"].update(
+                Panel(
+                    self.generate_metrics_table(),
+                    title="Final Metrics",
+                    border_style="magenta",
+                    box=box.ROUNDED,
+                )
+            )
+            live.update(layout)
+            time.sleep(1)  # Show final state for 1 second
+
+
+if __name__ == "__main__":
+    import doctest
+
+    doctest.testmod(optionflags=doctest.ELLIPSIS)

edsl/jobs/tasks/TaskHistory.py

@@ -50,6 +50,18 @@ class TaskHistory:
         """
         return [i.exceptions for k, i in self._interviews.items() if i.exceptions != {}]
 
+    @property
+    def unfixed_exceptions(self):
+        """
+        >>> len(TaskHistory.example().unfixed_exceptions)
+        4
+        """
+        return [
+            i.exceptions
+            for k, i in self._interviews.items()
+            if i.exceptions.num_unfixed() > 0
+        ]
+
     @property
     def indices(self):
         return [k for k, i in self._interviews.items() if i.exceptions != {}]
@@ -78,6 +90,11 @@ class TaskHistory:
         """
         return len(self.exceptions) > 0
 
+    @property
+    def has_unfixed_exceptions(self) -> bool:
+        """Return True if there are any exceptions."""
+        return len(self.unfixed_exceptions) > 0
+
     def _repr_html_(self):
         """Return an HTML representation of the TaskHistory."""
         from edsl.utilities.utilities import data_to_html

edsl/language_models/LanguageModel.py

@@ -164,20 +164,20 @@ class LanguageModel(
         None  # This should be something like ["choices", 0, "message", "content"]
     )
     __rate_limits = None
-    __default_rate_limits = {
-        "rpm": 10_000,
-        "tpm": 2_000_000,
-    }  # TODO: Use the OpenAI Teir 1 rate limits
     _safety_factor = 0.8
 
-    def __init__(self, tpm=None, rpm=None, **kwargs):
+    def __init__(
+        self, tpm=None, rpm=None, omit_system_prompt_if_empty_string=True, **kwargs
+    ):
         """Initialize the LanguageModel."""
         self.model = getattr(self, "_model_", None)
         default_parameters = getattr(self, "_parameters_", None)
         parameters = self._overide_default_parameters(kwargs, default_parameters)
         self.parameters = parameters
         self.remote = False
+        self.omit_system_prompt_if_empty = omit_system_prompt_if_empty_string
 
+        # self._rpm / _tpm comes from the class
         if rpm is not None:
             self._rpm = rpm
 
@@ -286,35 +286,40 @@ class LanguageModel(
         >>> m.RPM
         100
         """
-        self._set_rate_limits(rpm=rpm, tpm=tpm)
+        if rpm is not None:
+            self._rpm = rpm
+        if tpm is not None:
+            self._tpm = tpm
+        return None
+        # self._set_rate_limits(rpm=rpm, tpm=tpm)
 
-    def _set_rate_limits(self, rpm=None, tpm=None) -> None:
-        """Set the rate limits for the model.
+    # def _set_rate_limits(self, rpm=None, tpm=None) -> None:
+    #     """Set the rate limits for the model.
 
-        If the model does not have rate limits, use the default rate limits."""
-        if rpm is not None and tpm is not None:
-            self.__rate_limits = {"rpm": rpm, "tpm": tpm}
-            return
+    #     If the model does not have rate limits, use the default rate limits."""
+    #     if rpm is not None and tpm is not None:
+    #         self.__rate_limits = {"rpm": rpm, "tpm": tpm}
+    #         return
 
-        if self.__rate_limits is None:
-            if hasattr(self, "get_rate_limits"):
-                self.__rate_limits = self.get_rate_limits()
-            else:
-                self.__rate_limits = self.__default_rate_limits
+    #     if self.__rate_limits is None:
+    #         if hasattr(self, "get_rate_limits"):
+    #             self.__rate_limits = self.get_rate_limits()
+    #         else:
+    #             self.__rate_limits = self.__default_rate_limits
 
     @property
     def RPM(self):
         """Model's requests-per-minute limit."""
         # self._set_rate_limits()
         # return self._safety_factor * self.__rate_limits["rpm"]
-        return self.rpm
+        return self._rpm
 
     @property
     def TPM(self):
         """Model's tokens-per-minute limit."""
         # self._set_rate_limits()
         # return self._safety_factor * self.__rate_limits["tpm"]
-        return self.tpm
+        return self._tpm
 
     @property
     def rpm(self):
@@ -332,17 +337,6 @@ class LanguageModel(
     def tpm(self, value):
         self._tpm = value
 
-    @property
-    def TPM(self):
-        """Model's tokens-per-minute limit.
-
-        >>> m = LanguageModel.example()
-        >>> m.TPM > 0
-        True
-        """
-        self._set_rate_limits()
-        return self._safety_factor * self.__rate_limits["tpm"]
-
     @staticmethod
     def _overide_default_parameters(passed_parameter_dict, default_parameter_dict):
         """Return a dictionary of parameters, with passed parameters taking precedence over defaults.
@@ -471,12 +465,13 @@ class LanguageModel(
         if encoded_image:
             # the image has is appended to the user_prompt for hash-lookup purposes
             image_hash = hashlib.md5(encoded_image.encode()).hexdigest()
+            user_prompt += f" {image_hash}"
 
         cache_call_params = {
             "model": str(self.model),
             "parameters": self.parameters,
             "system_prompt": system_prompt,
-            "user_prompt": user_prompt + "" if not encoded_image else f" {image_hash}",
+            "user_prompt": user_prompt,
             "iteration": iteration,
         }
         cached_response, cache_key = cache.fetch(**cache_call_params)

edsl/language_models/registry.py

@@ -2,10 +2,10 @@ import textwrap
 from random import random
 from edsl.config import CONFIG
 
-if "EDSL_DEFAULT_MODEL" not in CONFIG:
-    default_model = "test"
-else:
-    default_model = CONFIG.get("EDSL_DEFAULT_MODEL")
+# if "EDSL_DEFAULT_MODEL" not in CONFIG:
+#     default_model = "test"
+# else:
+#     default_model = CONFIG.get("EDSL_DEFAULT_MODEL")
 
 
 def get_model_class(model_name, registry=None):
@@ -33,20 +33,24 @@ class Meta(type):
 
 
 class Model(metaclass=Meta):
-    default_model = default_model
+    default_model = CONFIG.get("EDSL_DEFAULT_MODEL")
 
-    def __new__(cls, model_name=None, registry=None, *args, **kwargs):
+    def __new__(
+        cls, model_name=None, registry=None, service_name=None, *args, **kwargs
+    ):
         # Map index to the respective subclass
         if model_name is None:
-            model_name = cls.default_model
+            model_name = (
+                cls.default_model
+            )  # when model_name is None, use the default model, set in the config file
         from edsl.inference_services.registry import default
 
         registry = registry or default
 
-        if isinstance(model_name, int):
+        if isinstance(model_name, int):  # can refer to a model by index
             model_name = cls.available(name_only=True)[model_name]
 
-        factory = registry.create_model_factory(model_name)
+        factory = registry.create_model_factory(model_name, service_name=service_name)
         return factory(*args, **kwargs)
 
     @classmethod

edsl/questions/QuestionBase.py

@@ -75,8 +75,7 @@ class QuestionBase(
         if not hasattr(self, "_fake_data_factory"):
             from polyfactory.factories.pydantic_factory import ModelFactory
 
-            class FakeData(ModelFactory[self.response_model]):
-                ...
+            class FakeData(ModelFactory[self.response_model]): ...
 
             self._fake_data_factory = FakeData
         return self._fake_data_factory
@@ -103,13 +102,8 @@ class QuestionBase(
         """Validate the answer.
         >>> from edsl.exceptions import QuestionAnswerValidationError
         >>> from edsl import QuestionFreeText as Q
-        >>> Q.example()._validate_answer({'answer': 'Hello'})
-        {'answer': 'Hello', 'generated_tokens': None}
-        >>> Q.example()._validate_answer({'shmanswer': 1})
-        Traceback (most recent call last):
-        ...
-        edsl.exceptions.questions.QuestionAnswerValidationError:...
-        ...
+        >>> Q.example()._validate_answer({'answer': 'Hello', 'generated_tokens': 'Hello'})
+        {'answer': 'Hello', 'generated_tokens': 'Hello'}
         """
 
         return self.response_validator.validate(answer)
@@ -471,6 +465,7 @@ class QuestionBase(
         self,
         scenario: Optional[dict] = None,
         agent: Optional[dict] = {},
+        answers: Optional[dict] = None,
         include_question_name: bool = False,
         height: Optional[int] = None,
         width: Optional[int] = None,
@@ -482,6 +477,17 @@ class QuestionBase(
         if scenario is None:
             scenario = {}
 
+        prior_answers_dict = {}
+
+        if isinstance(answers, dict):
+            for key, value in answers.items():
+                if not key.endswith("_comment") and not key.endswith(
+                    "_generated_tokens"
+                ):
+                    prior_answers_dict[key] = {"answer": value}
+
+        # breakpoint()
+
         base_template = """
         <div id="{{ question_name }}" class="survey_question" data-type="{{ question_type }}">
             {% if include_question_name %}
@@ -501,13 +507,40 @@ class QuestionBase(
 
         base_template = Template(base_template)
 
-        params = {
-            "question_name": self.question_name,
-            "question_text": Template(self.question_text).render(scenario, agent=agent),
-            "question_type": self.question_type,
-            "question_content": Template(question_content).render(scenario),
-            "include_question_name": include_question_name,
-        }
+        context = {
+            "scenario": scenario,
+            "agent": agent,
+        } | prior_answers_dict
+
+        # Render the question text
+        try:
+            question_text = Template(self.question_text).render(context)
+        except Exception as e:
+            print(
+                f"Error rendering question: question_text = {self.question_text}, error = {e}"
+            )
+            question_text = self.question_text
+
+        try:
+            question_content = Template(question_content).render(context)
+        except Exception as e:
+            print(
+                f"Error rendering question: question_content = {question_content}, error = {e}"
+            )
+            question_content = question_content
+
+        try:
+            params = {
+                "question_name": self.question_name,
+                "question_text": question_text,
+                "question_type": self.question_type,
+                "question_content": question_content,
+                "include_question_name": include_question_name,
+            }
+        except Exception as e:
+            raise ValueError(
+                f"Error rendering question: params = {params}, error = {e}"
+            )
         rendered_html = base_template.render(**params)
 
         if iframe:
@@ -526,6 +559,21 @@ class QuestionBase(
 
         return rendered_html
 
+    @classmethod
+    def example_model(cls):
+        from edsl import Model
+
+        q = cls.example()
+        m = Model("test", canned_response=cls._simulate_answer(q)["answer"])
+
+        return m
+
+    @classmethod
+    def example_results(cls):
+        m = cls.example_model()
+        q = cls.example()
+        return q.by(m).run(cache=False)
+
     def rich_print(self):
         """Print the question in a rich format."""
         from rich.table import Table