edsl 0.1.33__py3-none-any.whl → 0.1.33.dev1__py3-none-any.whl

edsl/inference_services/TestService.py

@@ -1,80 +0,0 @@
-from typing import Any, List
-import os
-import asyncio
-from edsl.inference_services.InferenceServiceABC import InferenceServiceABC
-from edsl.language_models import LanguageModel
-from edsl.inference_services.rate_limits_cache import rate_limits
-from edsl.utilities.utilities import fix_partial_correct_response
-
-from edsl.enums import InferenceServiceType
-import random
-
-
-class TestService(InferenceServiceABC):
-    """OpenAI service class."""
-
-    _inference_service_ = "test"
-    _env_key_name_ = None
-    _base_url_ = None
-
-    _sync_client_ = None
-    _async_client_ = None
-
-    _sync_client_instance = None
-    _async_client_instance = None
-
-    key_sequence = None
-    usage_sequence = None
-    model_exclude_list = []
-    input_token_name = "prompt_tokens"
-    output_token_name = "completion_tokens"
-
-    @classmethod
-    def available(cls) -> list[str]:
-        return ["test"]
-
-    @classmethod
-    def create_model(cls, model_name, model_class_name=None) -> LanguageModel:
-        throw_exception = False
-
-        class TestServiceLanguageModel(LanguageModel):
-            _model_ = "test"
-            _parameters_ = {"temperature": 0.5}
-            _inference_service_ = InferenceServiceType.TEST.value
-            usage_sequence = ["usage"]
-            key_sequence = ["message", 0, "text"]
-            input_token_name = cls.input_token_name
-            output_token_name = cls.output_token_name
-            _rpm = 1000
-            _tpm = 100000
-
-            @property
-            def _canned_response(self):
-                if hasattr(self, "canned_response"):
-                    return self.canned_response
-                else:
-                    return "Hello, world"
-
-            async def async_execute_model_call(
-                self,
-                user_prompt: str,
-                system_prompt: str,
-                encoded_image=None,
-            ) -> dict[str, Any]:
-                await asyncio.sleep(0.1)
-                # return {"message": """{"answer": "Hello, world"}"""}
-
-                if hasattr(self, "throw_exception") and self.throw_exception:
-                    if hasattr(self, "exception_probability"):
-                        p = self.exception_probability
-                    else:
-                        p = 1
-
-                    if random.random() < p:
-                        raise Exception("This is a test error")
-                return {
-                    "message": [{"text": f"{self._canned_response}"}],
-                    "usage": {"prompt_tokens": 1, "completion_tokens": 1},
-                }
-
-        return TestServiceLanguageModel

edsl/inference_services/TogetherAIService.py

@@ -1,170 +0,0 @@
-import aiohttp
-import json
-import requests
-from typing import Any, List
-
-# from edsl.inference_services.InferenceServiceABC import InferenceServiceABC
-from edsl.language_models import LanguageModel
-
-from edsl.inference_services.OpenAIService import OpenAIService
-import openai
-
-
-class TogetherAIService(OpenAIService):
-    """DeepInfra service class."""
-
-    _inference_service_ = "together"
-    _env_key_name_ = "TOGETHER_API_KEY"
-    _base_url_ = "https://api.together.xyz/v1"
-    _models_list_cache: List[str] = []
-
-    # These are non-serverless models. There was no api param to filter them
-    model_exclude_list = [
-        "EleutherAI/llemma_7b",
-        "HuggingFaceH4/zephyr-7b-beta",
-        "Nexusflow/NexusRaven-V2-13B",
-        "NousResearch/Hermes-2-Theta-Llama-3-70B",
-        "NousResearch/Nous-Capybara-7B-V1p9",
-        "NousResearch/Nous-Hermes-13b",
-        "NousResearch/Nous-Hermes-2-Mistral-7B-DPO",
-        "NousResearch/Nous-Hermes-2-Mixtral-8x7B-SFT",
-        "NousResearch/Nous-Hermes-Llama2-13b",
-        "NousResearch/Nous-Hermes-Llama2-70b",
-        "NousResearch/Nous-Hermes-llama-2-7b",
-        "NumbersStation/nsql-llama-2-7B",
-        "Open-Orca/Mistral-7B-OpenOrca",
-        "Phind/Phind-CodeLlama-34B-Python-v1",
-        "Phind/Phind-CodeLlama-34B-v2",
-        "Qwen/Qwen1.5-0.5B",
-        "Qwen/Qwen1.5-0.5B-Chat",
-        "Qwen/Qwen1.5-1.8B",
-        "Qwen/Qwen1.5-1.8B-Chat",
-        "Qwen/Qwen1.5-14B",
-        "Qwen/Qwen1.5-14B-Chat",
-        "Qwen/Qwen1.5-32B",
-        "Qwen/Qwen1.5-32B-Chat",
-        "Qwen/Qwen1.5-4B",
-        "Qwen/Qwen1.5-4B-Chat",
-        "Qwen/Qwen1.5-72B",
-        "Qwen/Qwen1.5-7B",
-        "Qwen/Qwen1.5-7B-Chat",
-        "Qwen/Qwen2-1.5B",
-        "Qwen/Qwen2-1.5B-Instruct",
-        "Qwen/Qwen2-72B",
-        "Qwen/Qwen2-7B",
-        "Qwen/Qwen2-7B-Instruct",
-        "SG161222/Realistic_Vision_V3.0_VAE",
-        "Snowflake/snowflake-arctic-instruct",
-        "Undi95/ReMM-SLERP-L2-13B",
-        "Undi95/Toppy-M-7B",
-        "WizardLM/WizardCoder-Python-34B-V1.0",
-        "WizardLM/WizardLM-13B-V1.2",
-        "WizardLM/WizardLM-70B-V1.0",
-        "allenai/OLMo-7B",
-        "allenai/OLMo-7B-Instruct",
-        "bert-base-uncased",
-        "codellama/CodeLlama-13b-Instruct-hf",
-        "codellama/CodeLlama-13b-Python-hf",
-        "codellama/CodeLlama-13b-hf",
-        "codellama/CodeLlama-34b-Python-hf",
-        "codellama/CodeLlama-34b-hf",
-        "codellama/CodeLlama-70b-Instruct-hf",
-        "codellama/CodeLlama-70b-Python-hf",
-        "codellama/CodeLlama-70b-hf",
-        "codellama/CodeLlama-7b-Instruct-hf",
-        "codellama/CodeLlama-7b-Python-hf",
-        "codellama/CodeLlama-7b-hf",
-        "cognitivecomputations/dolphin-2.5-mixtral-8x7b",
-        "deepseek-ai/deepseek-coder-33b-instruct",
-        "garage-bAInd/Platypus2-70B-instruct",
-        "google/gemma-2b",
-        "google/gemma-7b",
-        "google/gemma-7b-it",
-        "gradientai/Llama-3-70B-Instruct-Gradient-1048k",
-        "hazyresearch/M2-BERT-2k-Retrieval-Encoder-V1",
-        "huggyllama/llama-13b",
-        "huggyllama/llama-30b",
-        "huggyllama/llama-65b",
-        "huggyllama/llama-7b",
-        "lmsys/vicuna-13b-v1.3",
-        "lmsys/vicuna-13b-v1.5",
-        "lmsys/vicuna-13b-v1.5-16k",
-        "lmsys/vicuna-7b-v1.3",
-        "lmsys/vicuna-7b-v1.5",
-        "meta-llama/Llama-2-13b-hf",
-        "meta-llama/Llama-2-70b-chat-hf",
-        "meta-llama/Llama-2-7b-hf",
-        "meta-llama/Llama-3-70b-hf",
-        "meta-llama/Llama-3-8b-hf",
-        "meta-llama/Meta-Llama-3-70B",
-        "meta-llama/Meta-Llama-3-70B-Instruct",
-        "meta-llama/Meta-Llama-3-8B-Instruct",
-        "meta-llama/Meta-Llama-3.1-70B-Instruct-Reference",
-        "meta-llama/Meta-Llama-3.1-70B-Reference",
-        "meta-llama/Meta-Llama-3.1-8B-Reference",
-        "microsoft/phi-2",
-        "mistralai/Mixtral-8x22B",
-        "openchat/openchat-3.5-1210",
-        "prompthero/openjourney",
-        "runwayml/stable-diffusion-v1-5",
-        "sentence-transformers/msmarco-bert-base-dot-v5",
-        "snorkelai/Snorkel-Mistral-PairRM-DPO",
-        "stabilityai/stable-diffusion-2-1",
-        "teknium/OpenHermes-2-Mistral-7B",
-        "teknium/OpenHermes-2p5-Mistral-7B",
-        "togethercomputer/CodeLlama-13b-Instruct",
-        "togethercomputer/CodeLlama-13b-Python",
-        "togethercomputer/CodeLlama-34b",
-        "togethercomputer/CodeLlama-34b-Python",
-        "togethercomputer/CodeLlama-7b-Instruct",
-        "togethercomputer/CodeLlama-7b-Python",
-        "togethercomputer/Koala-13B",
-        "togethercomputer/Koala-7B",
-        "togethercomputer/LLaMA-2-7B-32K",
-        "togethercomputer/SOLAR-10.7B-Instruct-v1.0-int4",
-        "togethercomputer/StripedHyena-Hessian-7B",
-        "togethercomputer/alpaca-7b",
-        "togethercomputer/evo-1-131k-base",
-        "togethercomputer/evo-1-8k-base",
-        "togethercomputer/guanaco-13b",
-        "togethercomputer/guanaco-33b",
-        "togethercomputer/guanaco-65b",
-        "togethercomputer/guanaco-7b",
-        "togethercomputer/llama-2-13b",
-        "togethercomputer/llama-2-70b-chat",
-        "togethercomputer/llama-2-7b",
-        "wavymulder/Analog-Diffusion",
-        "zero-one-ai/Yi-34B",
-        "zero-one-ai/Yi-34B-Chat",
-        "zero-one-ai/Yi-6B",
-    ]
-
-    _sync_client_ = openai.OpenAI
-    _async_client_ = openai.AsyncOpenAI
-
-    @classmethod
-    def get_model_list(cls):
-        # Togheter.ai has a different response in model list then openai
-        # and the OpenAI class returns an error when calling .models.list()
-        import requests
-        import os
-
-        url = "https://api.together.xyz/v1/models?filter=serverless"
-        token = os.getenv(cls._env_key_name_)
-        headers = {"accept": "application/json", "authorization": f"Bearer {token}"}
-
-        response = requests.get(url, headers=headers)
-        return response.json()
-
-    @classmethod
-    def available(cls) -> List[str]:
-        if not cls._models_list_cache:
-            try:
-                cls._models_list_cache = [
-                    m["id"]
-                    for m in cls.get_model_list()
-                    if m["id"] not in cls.model_exclude_list
-                ]
-            except Exception as e:
-                raise
-        return cls._models_list_cache

edsl/jobs/FailedQuestion.py

@@ -1,78 +0,0 @@
-from edsl.questions import QuestionBase
-from edsl import Question, Scenario, Model, Agent
-
-from edsl.language_models.LanguageModel import LanguageModel
-
-
-class FailedQuestion:
-    # tests/jobs/test_Interview.py::test_handle_model_exceptions
-
-    # (Pdb) dir(self.exception.__traceback__)
-    # ['tb_frame', 'tb_lasti', 'tb_lineno', 'tb_next']
-
-    def __init__(
-        self, question, scenario, model, agent, raw_model_response, exception, prompts
-    ):
-        self.question = question
-        self.scenario = scenario
-        self.model = model
-        self.agent = agent
-        self.raw_model_response = raw_model_response  # JSON
-        self.exception = exception
-        self.prompts = prompts
-
-    def to_dict(self):
-        return {
-            "question": self.question._to_dict(),
-            "scenario": self.scenario._to_dict(),
-            "model": self.model._to_dict(),
-            "agent": self.agent._to_dict(),
-            "raw_model_response": self.raw_model_response,
-            "exception": self.exception.__class__.__name__,  # self.exception,
-            "prompts": self.prompts,
-        }
-
-    @classmethod
-    def from_dict(cls, data):
-        question = QuestionBase.from_dict(data["question"])
-        scenario = Scenario.from_dict(data["scenario"])
-        model = LanguageModel.from_dict(data["model"])
-        agent = Agent.from_dict(data["agent"])
-        raw_model_response = data["raw_model_response"]
-        exception = data["exception"]
-        prompts = data["prompts"]
-        return cls(
-            question, scenario, model, agent, raw_model_response, exception, prompts
-        )
-
-    def __repr__(self):
-        return f"{self.__class__.__name__}(question={repr(self.question)}, scenario={repr(self.scenario)}, model={repr(self.model)}, agent={repr(self.agent)}, raw_model_response={repr(self.raw_model_response)}, exception={repr(self.exception)})"
-
-    @property
-    def jobs(self):
-        return self.question.by(self.scenario).by(self.agent).by(self.model)
-
-    def rerun(self):
-        results = self.jobs.run()
-        return results
-
-    def help(self):
-        pass
-
-    @classmethod
-    def example(cls):
-        from edsl.language_models.utilities import create_language_model
-        from edsl.language_models.utilities import create_survey
-
-        survey = create_survey(2, chained=False, take_scenario=False)
-        fail_at_number = 1
-        model = create_language_model(ValueError, fail_at_number)()
-        from edsl import Survey
-
-        results = survey.by(model).run()
-        return results.failed_questions[0][0]
-
-
-if __name__ == "__main__":
-    fq = FailedQuestion.example()
-    new_fq = FailedQuestion.from_dict(fq.to_dict())

edsl/jobs/runners/JobsRunnerStatus.py

@@ -1,331 +0,0 @@
-from __future__ import annotations
-
-import time
-from dataclasses import dataclass, asdict
-
-from typing import List, DefaultDict, Optional, Type, Literal
-from collections import UserDict, defaultdict
-
-from rich.text import Text
-from rich.box import SIMPLE
-from rich.table import Table
-from rich.live import Live
-from rich.panel import Panel
-from rich.progress import Progress, TextColumn, BarColumn, TaskProgressColumn
-from rich.layout import Layout
-from rich.console import Group
-from rich import box
-
-from edsl.jobs.interviews.InterviewStatusDictionary import InterviewStatusDictionary
-from edsl.jobs.tokens.InterviewTokenUsage import InterviewTokenUsage
-from edsl.jobs.tokens.TokenUsage import TokenUsage
-from edsl.enums import get_token_pricing
-from edsl.jobs.tasks.task_status_enum import TaskStatus
-
-InterviewTokenUsageMapping = DefaultDict[str, InterviewTokenUsage]
-
-from edsl.jobs.interviews.InterviewStatistic import InterviewStatistic
-from edsl.jobs.interviews.InterviewStatisticsCollection import (
-    InterviewStatisticsCollection,
-)
-from edsl.jobs.tokens.InterviewTokenUsage import InterviewTokenUsage
-
-
-@dataclass
-class ModelInfo:
-    model_name: str
-    TPM_limit_k: float
-    RPM_limit_k: float
-    num_tasks_waiting: int
-    token_usage_info: dict
-
-
-@dataclass
-class ModelTokenUsageStats:
-    token_usage_type: str
-    details: List[dict]
-    cost: str
-
-
-class Stats:
-    def elapsed_time(self):
-        InterviewStatistic("elapsed_time", value=elapsed_time, digits=1, units="sec.")
-
-
-class JobsRunnerStatus:
-    def __init__(
-        self, jobs_runner: "JobsRunnerAsyncio", n: int, refresh_rate: float = 0.25
-    ):
-        self.jobs_runner = jobs_runner
-        self.start_time = time.time()
-        self.completed_interviews = []
-        self.refresh_rate = refresh_rate
-        self.statistics = [
-            "elapsed_time",
-            "total_interviews_requested",
-            "completed_interviews",
-            #            "percent_complete",
-            "average_time_per_interview",
-            #            "task_remaining",
-            "estimated_time_remaining",
-            "exceptions",
-            "unfixed_exceptions",
-            "throughput",
-        ]
-        self.num_total_interviews = n * len(self.jobs_runner.interviews)
-
-        self.distinct_models = list(
-            set(i.model.model for i in self.jobs_runner.interviews)
-        )
-
-        self.completed_interview_by_model = defaultdict(list)
-
-    def add_completed_interview(self, result):
-        self.completed_interviews.append(result.interview_hash)
-
-        relevant_model = result.model.model
-        self.completed_interview_by_model[relevant_model].append(result.interview_hash)
-
-    def _compute_statistic(self, stat_name: str):
-        completed_tasks = self.completed_interviews
-        elapsed_time = time.time() - self.start_time
-        interviews = self.jobs_runner.total_interviews
-
-        stat_definitions = {
-            "elapsed_time": lambda: InterviewStatistic(
-                "elapsed_time", value=elapsed_time, digits=1, units="sec."
-            ),
-            "total_interviews_requested": lambda: InterviewStatistic(
-                "total_interviews_requested", value=len(interviews), units=""
-            ),
-            "completed_interviews": lambda: InterviewStatistic(
-                "completed_interviews", value=len(completed_tasks), units=""
-            ),
-            "percent_complete": lambda: InterviewStatistic(
-                "percent_complete",
-                value=(
-                    len(completed_tasks) / len(interviews) * 100
-                    if len(interviews) > 0
-                    else 0
-                ),
-                digits=1,
-                units="%",
-            ),
-            "average_time_per_interview": lambda: InterviewStatistic(
-                "average_time_per_interview",
-                value=elapsed_time / len(completed_tasks) if completed_tasks else 0,
-                digits=2,
-                units="sec.",
-            ),
-            "task_remaining": lambda: InterviewStatistic(
-                "task_remaining", value=len(interviews) - len(completed_tasks), units=""
-            ),
-            "estimated_time_remaining": lambda: InterviewStatistic(
-                "estimated_time_remaining",
-                value=(
-                    (len(interviews) - len(completed_tasks))
-                    * (elapsed_time / len(completed_tasks))
-                    if len(completed_tasks) > 0
-                    else 0
-                ),
-                digits=1,
-                units="sec.",
-            ),
-            "exceptions": lambda: InterviewStatistic(
-                "exceptions",
-                value=sum(len(i.exceptions) for i in interviews),
-                units="",
-            ),
-            "unfixed_exceptions": lambda: InterviewStatistic(
-                "unfixed_exceptions",
-                value=sum(i.exceptions.num_unfixed() for i in interviews),
-                units="",
-            ),
-            "throughput": lambda: InterviewStatistic(
-                "throughput",
-                value=len(completed_tasks) / elapsed_time if elapsed_time > 0 else 0,
-                digits=2,
-                units="interviews/sec.",
-            ),
-        }
-        return stat_definitions[stat_name]()
-
-    def create_progress_bar(self):
-        return Progress(
-            TextColumn("[progress.description]{task.description}"),
-            BarColumn(),
-            TaskProgressColumn(),
-            TextColumn("{task.completed}/{task.total}"),
-        )
-
-    def generate_model_queues_table(self):
-        table = Table(show_header=False, box=box.SIMPLE)
-        table.add_column("Info", style="cyan")
-        table.add_column("Value", style="magenta")
-        # table.add_row("Bucket collection", str(self.jobs_runner.bucket_collection))
-        for model, bucket in self.jobs_runner.bucket_collection.items():
-            table.add_row(Text(model.model, style="bold blue"), "")
-            bucket_types = ["requests_bucket", "tokens_bucket"]
-            for bucket_type in bucket_types:
-                table.add_row(Text(" " + bucket_type, style="green"), "")
-                # table.add_row(
-                #     f"  Current level (capacity = {round(getattr(bucket, bucket_type).capacity, 3)})",
-                #     str(round(getattr(bucket, bucket_type).tokens, 3)),
-                # )
-                num_requests = getattr(bucket, bucket_type).num_requests
-                num_released = getattr(bucket, bucket_type).num_released
-                tokens_returned = getattr(bucket, bucket_type).tokens_returned
-                # table.add_row(
-                #     f"  Requested",
-                #     str(num_requests),
-                # )
-                # table.add_row(
-                #     f"  Completed",
-                #     str(num_released),
-                # )
-                table.add_row(
-                    "  Completed vs. Requested", f"{num_released} vs. {num_requests}"
-                )
-                table.add_row(
-                    "  Added tokens (from cache)",
-                    str(tokens_returned),
-                )
-                if bucket_type == "tokens_bucket":
-                    rate_name = "TPM"
-                else:
-                    rate_name = "RPM"
-                target_rate = round(getattr(bucket, bucket_type).target_rate, 1)
-                table.add_row(
-                    f"  Empirical {rate_name} (target = {target_rate})",
-                    str(round(getattr(bucket, bucket_type).get_throughput(), 0)),
-                )
-
-        return table
-
-    def generate_layout(self):
-        progress = self.create_progress_bar()
-        task_ids = []
-        for model in self.distinct_models:
-            task_id = progress.add_task(
-                f"[cyan]{model}...",
-                total=int(self.num_total_interviews / len(self.distinct_models)),
-            )
-            task_ids.append((model, task_id))
-
-        progress_height = min(5, 2 + len(self.distinct_models))
-        layout = Layout()
-
-        # Create the top row with only the progress panel
-        layout.split_column(
-            Layout(
-                Panel(
-                    progress,
-                    title="Interview Progress",
-                    border_style="cyan",
-                    box=box.ROUNDED,
-                ),
-                name="progress",
-                size=progress_height,  # Adjusted size
-            ),
-            Layout(name="bottom_row"),  # Adjusted size
-        )
-
-        # Split the bottom row into two columns for metrics and model queues
-        layout["bottom_row"].split_row(
-            Layout(
-                Panel(
-                    self.generate_metrics_table(),
-                    title="Metrics",
-                    border_style="magenta",
-                    box=box.ROUNDED,
-                ),
-                name="metrics",
-            ),
-            Layout(
-                Panel(
-                    self.generate_model_queues_table(),
-                    title="Model Queues",
-                    border_style="yellow",
-                    box=box.ROUNDED,
-                ),
-                name="model_queues",
-            ),
-        )
-
-        return layout, progress, task_ids
-
-    def generate_metrics_table(self):
-        table = Table(show_header=True, header_style="bold magenta", box=box.SIMPLE)
-        table.add_column("Metric", style="cyan", no_wrap=True)
-        table.add_column("Value", justify="right")
-
-        for stat_name in self.statistics:
-            pretty_name, value = list(self._compute_statistic(stat_name).items())[0]
-            # breakpoint()
-            table.add_row(pretty_name, value)
-        return table
-
-    def update_progress(self):
-        layout, progress, task_ids = self.generate_layout()
-
-        with Live(
-            layout, refresh_per_second=int(1 / self.refresh_rate), transient=True
-        ) as live:
-            while len(self.completed_interviews) < len(
-                self.jobs_runner.total_interviews
-            ):
-                completed_tasks = len(self.completed_interviews)
-                total_tasks = len(self.jobs_runner.total_interviews)
-
-                for model, task_id in task_ids:
-                    completed_tasks = len(self.completed_interview_by_model[model])
-                    progress.update(
-                        task_id,
-                        completed=completed_tasks,
-                        description=f"[cyan]Conducting interviews for {model}...",
-                    )
-
-                layout["metrics"].update(
-                    Panel(
-                        self.generate_metrics_table(),
-                        title="Metrics",
-                        border_style="magenta",
-                        box=box.ROUNDED,
-                    )
-                )
-                layout["model_queues"].update(
-                    Panel(
-                        self.generate_model_queues_table(),
-                        title="Final Model Queues",
-                        border_style="yellow",
-                        box=box.ROUNDED,
-                    )
-                )
-
-                time.sleep(self.refresh_rate)
-
-            # Final update
-            for model, task_id in task_ids:
-                completed_tasks = len(self.completed_interview_by_model[model])
-                progress.update(
-                    task_id,
-                    completed=completed_tasks,
-                    description=f"[cyan]Conducting interviews for {model}...",
-                )
-
-            layout["metrics"].update(
-                Panel(
-                    self.generate_metrics_table(),
-                    title="Final Metrics",
-                    border_style="magenta",
-                    box=box.ROUNDED,
-                )
-            )
-            live.update(layout)
-            time.sleep(1)  # Show final state for 1 second
-
-
-if __name__ == "__main__":
-    import doctest
-
-    doctest.testmod(optionflags=doctest.ELLIPSIS)

edsl/language_models/fake_openai_call.py

@@ -1,15 +0,0 @@
-from openai import AsyncOpenAI
-import asyncio
-
-client = AsyncOpenAI(base_url="http://127.0.0.1:8000/v1", api_key="fake_key")
-
-
-async def main():
-    response = await client.chat.completions.create(
-        model="gpt-3.5-turbo", messages=[{"role": "user", "content": "Question XX42"}]
-    )
-    print(response)
-
-
-if __name__ == "__main__":
-    asyncio.run(main())