edsl 0.1.33__py3-none-any.whl → 0.1.33.dev1__py3-none-any.whl

edsl/language_models/fake_openai_service.py

@@ -1,61 +0,0 @@
-import threading
-import asyncio
-from fastapi import FastAPI, Request
-from fastapi.responses import JSONResponse
-import uvicorn
-import json
-from typing import Any
-
-app = FastAPI()
-
-
-async def generate_response(question_number: int) -> dict:
-    # Simulate some asynchronous work
-    await asyncio.sleep(1)
-    return {
-        "id": "chatcmpl-123",
-        "object": "chat.completion",
-        "created": 1677652288,
-        "model": "gpt-3.5-turbo-0613",
-        "choices": [
-            {
-                "index": 0,
-                "message": {
-                    "role": "assistant",
-                    "content": json.dumps(
-                        {"answer": f"SPAM for question {question_number}!"}
-                    ),
-                },
-                "finish_reason": "stop",
-            }
-        ],
-        "usage": {"prompt_tokens": 9, "completion_tokens": 12, "total_tokens": 21},
-    }
-
-
-@app.post("/v1/chat/completions")
-async def chat_completions(request: Request):
-    body = await request.json()
-    user_prompt = body["messages"][-1]["content"]
-    question_number = int(user_prompt.split("XX")[1])
-
-    response = await generate_response(question_number)
-    return JSONResponse(content=response)
-
-
-def run_server():
-    uvicorn.run(app, host="127.0.0.1", port=8000)
-
-
-if __name__ == "__main__":
-    # Start the server in a separate thread
-    server_thread = threading.Thread(target=run_server)
-    server_thread.start()
-
-# Your main code here
-# ...
-
-# To use this with the OpenAI SDK:
-# from openai import AsyncOpenAI
-# client = AsyncOpenAI(base_url="http://127.0.0.1:8000/v1", api_key="fake_key")
-# response = await client.chat.completions.create(model="gpt-3.5-turbo", messages=[...])

edsl/language_models/utilities.py

@@ -1,61 +0,0 @@
-import asyncio
-from typing import Any
-from edsl import Survey
-from edsl.config import CONFIG
-from edsl.enums import InferenceServiceType
-from edsl.language_models.LanguageModel import LanguageModel
-from edsl.questions import QuestionFreeText
-
-
-def create_survey(num_questions: int, chained: bool = True, take_scenario=False):
-    survey = Survey()
-    for i in range(num_questions):
-        if take_scenario:
-            q = QuestionFreeText(
-                question_text=f"XX{i}XX and {{scenario_value }}",
-                question_name=f"question_{i}",
-            )
-        else:
-            q = QuestionFreeText(
-                question_text=f"XX{i}XX", question_name=f"question_{i}"
-            )
-        survey.add_question(q)
-        if i > 0 and chained:
-            survey.add_targeted_memory(f"question_{i}", f"question_{i-1}")
-    return survey
-
-
-def create_language_model(
-    exception: Exception, fail_at_number: int, never_ending=False
-):
-    class LanguageModelFromUtilities(LanguageModel):
-        _model_ = "test"
-        _parameters_ = {"temperature": 0.5}
-        _inference_service_ = InferenceServiceType.TEST.value
-        key_sequence = ["message", 0, "text"]
-        usage_sequence = ["usage"]
-        input_token_name = "prompt_tokens"
-        output_token_name = "completion_tokens"
-        _rpm = 1000000000000
-        _tpm = 1000000000000
-
-        async def async_execute_model_call(
-            self, user_prompt: str, system_prompt: str
-        ) -> dict[str, Any]:
-            question_number = int(
-                user_prompt.split("XX")[1]
-            )  ## grabs the question number from the prompt
-            await asyncio.sleep(0.1)
-            if never_ending:  ## you're not going anywhere buddy
-                await asyncio.sleep(float("inf"))
-            if question_number == fail_at_number:
-                if asyncio.iscoroutinefunction(exception):
-                    await exception()
-                else:
-                    raise exception
-            return {
-                "message": [{"text": "SPAM!"}],
-                "usage": {"prompt_tokens": 1, "completion_tokens": 1},
-            }
-
-    return LanguageModelFromUtilities

edsl/questions/QuestionBaseGenMixin.py

@@ -1,133 +0,0 @@
-from __future__ import annotations
-import copy
-import itertools
-from typing import Optional, List, Callable, Type
-from typing import TypeVar
-
-
-class QuestionBaseGenMixin:
-    def copy(self) -> QuestionBase:
-        """Return a deep copy of the question.
-
-        >>> from edsl.questions import QuestionFreeText
-        >>> q = QuestionFreeText(question_name = "color", question_text = "What is your favorite color?")
-        >>> q2 = q.copy()
-        >>> q2.question_name
-        'color'
-
-        """
-        return copy.deepcopy(self)
-
-    def option_permutations(self) -> list[QuestionBase]:
-        """Return a list of questions with all possible permutations of the options.
-
-        >>> from edsl import QuestionMultipleChoice as Q
-        >>> len(Q.example().option_permutations())
-        24
-        """
-
-        if not hasattr(self, "question_options"):
-            return [self]
-
-        questions = []
-        for index, permutation in enumerate(
-            itertools.permutations(self.question_options)
-        ):
-            question = copy.deepcopy(self)
-            question.question_options = list(permutation)
-            question.question_name = f"{self.question_name}_{index}"
-            questions.append(question)
-        return questions
-
-    def loop(self, scenario_list: ScenarioList) -> List[QuestionBase]:
-        """Return a list of questions with the question name modified for each scenario.
-
-        :param scenario_list: The list of scenarios to loop through.
-
-        >>> from edsl import QuestionFreeText
-        >>> from edsl import ScenarioList
-        >>> q = QuestionFreeText(question_text = "What are your thoughts on: {{ subject}}?", question_name = "base_{{subject}}")
-        >>> len(q.loop(ScenarioList.from_list("subject", ["Math", "Economics", "Chemistry"])))
-        3
-
-        """
-        from jinja2 import Environment
-        from edsl.questions.QuestionBase import QuestionBase
-
-        starting_name = self.question_name
-        questions = []
-        for index, scenario in enumerate(scenario_list):
-            env = Environment()
-            new_data = self.to_dict().copy()
-            for key, value in [(k, v) for k, v in new_data.items() if v is not None]:
-                if (
-                    isinstance(value, str) or isinstance(value, int)
-                ) and key != "question_options":
-                    new_data[key] = env.from_string(value).render(scenario)
-                elif isinstance(value, list):
-                    new_data[key] = [
-                        env.from_string(v).render(scenario) if isinstance(v, str) else v
-                        for v in value
-                    ]
-                elif isinstance(value, dict):
-                    new_data[key] = {
-                        (
-                            env.from_string(k).render(scenario)
-                            if isinstance(k, str)
-                            else k
-                        ): (
-                            env.from_string(v).render(scenario)
-                            if isinstance(v, str)
-                            else v
-                        )
-                        for k, v in value.items()
-                    }
-                elif key == "question_options" and isinstance(value, str):
-                    new_data[key] = value
-                else:
-                    raise ValueError(
-                        f"Unexpected value type: {type(value)} for key '{key}'"
-                    )
-
-            if new_data["question_name"] == starting_name:
-                new_data["question_name"] = new_data["question_name"] + f"_{index}"
-
-            questions.append(QuestionBase.from_dict(new_data))
-        return questions
-
-    def apply_function(self, func: Callable, exclude_components=None) -> QuestionBase:
-        """Apply a function to the question parts
-
-        >>> from edsl.questions import QuestionFreeText
-        >>> q = QuestionFreeText(question_name = "color", question_text = "What is your favorite color?")
-        >>> shouting = lambda x: x.upper()
-        >>> q.apply_function(shouting)
-        Question('free_text', question_name = \"""color\""", question_text = \"""WHAT IS YOUR FAVORITE COLOR?\""")
-
-        """
-        from edsl.questions.QuestionBase import QuestionBase
-
-        if exclude_components is None:
-            exclude_components = ["question_name", "question_type"]
-
-        d = copy.deepcopy(self._to_dict())
-        for key, value in d.items():
-            if key in exclude_components:
-                continue
-            if isinstance(value, dict):
-                for k, v in value.items():
-                    value[k] = func(v)
-                d[key] = value
-                continue
-            if isinstance(value, list):
-                value = [func(v) for v in value]
-                d[key] = value
-                continue
-            d[key] = func(value)
-        return QuestionBase.from_dict(d)
-
-
-if __name__ == "__main__":
-    import doctest
-
-    doctest.testmod()

edsl/questions/QuestionBasePromptsMixin.py

@@ -1,266 +0,0 @@
-from importlib import resources
-from typing import Optional
-from edsl.prompts import Prompt
-from edsl.exceptions.questions import QuestionAnswerValidationError
-
-from functools import lru_cache
-
-
-class TemplateManager:
-    _instance = None
-
-    def __new__(cls):
-        if cls._instance is None:
-            cls._instance = super().__new__(cls)
-            cls._instance._template_cache = {}
-        return cls._instance
-
-    @lru_cache(maxsize=None)
-    def get_template(self, question_type, template_name):
-        if (question_type, template_name) not in self._template_cache:
-            with resources.open_text(
-                f"edsl.questions.templates.{question_type}", template_name
-            ) as file:
-                self._template_cache[(question_type, template_name)] = file.read()
-        return self._template_cache[(question_type, template_name)]
-
-
-# Global instance
-template_manager = TemplateManager()
-
-
-class QuestionBasePromptsMixin:
-    # @classmethod
-    # @lru_cache(maxsize=1)
-    # def _read_template(cls, template_name):
-    #     with resources.open_text(
-    #         f"edsl.questions.templates.{cls.question_type}", template_name
-    #     ) as file:
-    #         return file.read()
-
-    @classmethod
-    def applicable_prompts(
-        cls, model: Optional[str] = None
-    ) -> list[type["PromptBase"]]:
-        """Get the prompts that are applicable to the question type.
-
-        :param model: The language model to use.
-
-        >>> from edsl.questions import QuestionFreeText
-        >>> QuestionFreeText.applicable_prompts()
-        [<class 'edsl.prompts.library.question_freetext.FreeText'>]
-
-        :param model: The language model to use. If None, assumes does not matter.
-
-        """
-        from edsl.prompts.registry import get_classes as prompt_lookup
-
-        applicable_prompts = prompt_lookup(
-            component_type="question_instructions",
-            question_type=cls.question_type,
-            model=model,
-        )
-        return applicable_prompts
-
-    @property
-    def model_instructions(self) -> dict:
-        """Get the model-specific instructions for the question."""
-        if not hasattr(self, "_model_instructions"):
-            self._model_instructions = {}
-        return self._model_instructions
-
-    def _all_text(self) -> str:
-        """Return the question text.
-
-        >>> from edsl import QuestionMultipleChoice as Q
-        >>> Q.example()._all_text()
-        "how_feelingHow are you?['Good', 'Great', 'OK', 'Bad']"
-        """
-        txt = ""
-        for key, value in self.data.items():
-            if isinstance(value, str):
-                txt += value
-            elif isinstance(value, list):
-                txt += "".join(str(value))
-        return txt
-
-    @model_instructions.setter
-    def model_instructions(self, data: dict):
-        """Set the model-specific instructions for the question."""
-        self._model_instructions = data
-
-    def add_model_instructions(
-        self, *, instructions: str, model: Optional[str] = None
-    ) -> None:
-        """Add model-specific instructions for the question that override the default instructions.
-
-        :param instructions: The instructions to add. This is typically a jinja2 template.
-        :param model: The language model for this instruction.
-
-        >>> from edsl.questions import QuestionFreeText
-        >>> q = QuestionFreeText(question_name = "color", question_text = "What is your favorite color?")
-        >>> q.add_model_instructions(instructions = "{{question_text}}. Answer in valid JSON like so {'answer': 'comment: <>}", model = "gpt3")
-        >>> q.get_instructions(model = "gpt3")
-        Prompt(text=\"""{{question_text}}. Answer in valid JSON like so {'answer': 'comment: <>}\""")
-        """
-        from edsl import Model
-
-        if not hasattr(self, "_model_instructions"):
-            self._model_instructions = {}
-        if model is None:
-            # if not model is passed, all the models are mapped to this instruction, including 'None'
-            self._model_instructions = {
-                model_name: instructions
-                for model_name in Model.available(name_only=True)
-            }
-            self._model_instructions.update({model: instructions})
-        else:
-            self._model_instructions.update({model: instructions})
-
-    @classmethod
-    def path_to_folder(cls) -> str:
-        return resources.files(f"edsl.questions.templates", cls.question_type)
-
-    @property
-    def response_model(self) -> type["BaseModel"]:
-        if self._response_model is not None:
-            return self._response_model
-        else:
-            return self.create_response_model()
-
-    @property
-    def use_code(self) -> bool:
-        if hasattr(self, "_use_code"):
-            return self._use_code
-        return True
-
-    @use_code.setter
-    def use_code(self, value: bool) -> None:
-        self._use_code = value
-
-    @property
-    def include_comment(self) -> bool:
-        if hasattr(self, "_include_comment"):
-            return self._include_comment
-        return True
-
-    @include_comment.setter
-    def include_comment(self, value: bool) -> None:
-        self._include_comment = value
-
-    @classmethod
-    def default_answering_instructions(cls) -> str:
-        # template_text = cls._read_template("answering_instructions.jinja")
-        template_text = template_manager.get_template(
-            cls.question_type, "answering_instructions.jinja"
-        )
-        return Prompt(text=template_text)
-
-    @classmethod
-    def default_question_presentation(cls):
-        # template_text = cls._read_template("question_presentation.jinja")
-        template_text = template_manager.get_template(
-            cls.question_type, "question_presentation.jinja"
-        )
-        return Prompt(text=template_text)
-
-    @property
-    def answering_instructions(self) -> str:
-        if self._answering_instructions is None:
-            return self.default_answering_instructions()
-        return self._answering_instructions
-
-    @answering_instructions.setter
-    def answering_instructions(self, value) -> None:
-        self._answering_instructions = value
-
-    # @classmethod
-    # def default_answering_instructions(cls) -> str:
-    #     with resources.open_text(
-    #         f"edsl.questions.templates.{cls.question_type}",
-    #         "answering_instructions.jinja",
-    #     ) as file:
-    #         return Prompt(text=file.read())
-
-    # @classmethod
-    # def default_question_presentation(cls):
-    #     with resources.open_text(
-    #         f"edsl.questions.templates.{cls.question_type}",
-    #         "question_presentation.jinja",
-    #     ) as file:
-    #         return Prompt(text=file.read())
-
-    @property
-    def question_presentation(self):
-        if self._question_presentation is None:
-            return self.default_question_presentation()
-        return self._question_presentation
-
-    @question_presentation.setter
-    def question_presentation(self, value):
-        self._question_presentation = value
-
-    def prompt_preview(self, scenario=None, agent=None):
-        return self.new_default_instructions.render(
-            self.data
-            | {
-                "include_comment": getattr(self, "_include_comment", True),
-                "use_code": getattr(self, "_use_code", True),
-            }
-            | ({"scenario": scenario} or {})
-            | ({"agent": agent} or {})
-        )
-
-    @classmethod
-    def self_check(cls):
-        q = cls.example()
-        for answer, params in q.response_validator.valid_examples:
-            for key, value in params.items():
-                setattr(q, key, value)
-            q._validate_answer(answer)
-        for answer, params, reason in q.response_validator.invalid_examples:
-            for key, value in params.items():
-                setattr(q, key, value)
-            try:
-                q._validate_answer(answer)
-            except QuestionAnswerValidationError:
-                pass
-            else:
-                raise ValueError(f"Example {answer} should have failed for {reason}.")
-
-    @property
-    def new_default_instructions(self) -> "Prompt":
-        "This is set up as a property because there are mutable question values that determine how it is rendered."
-        return self.question_presentation + self.answering_instructions
-
-    @property
-    def parameters(self) -> set[str]:
-        """Return the parameters of the question."""
-        from jinja2 import Environment, meta
-
-        env = Environment()
-        # Parse the template
-        txt = self._all_text()
-        # txt = self.question_text
-        # if hasattr(self, "question_options"):
-        #    txt += " ".join(self.question_options)
-        parsed_content = env.parse(txt)
-        # Extract undeclared variables
-        variables = meta.find_undeclared_variables(parsed_content)
-        # Return as a list
-        return set(variables)
-
-    def get_instructions(self, model: Optional[str] = None) -> type["PromptBase"]:
-        """Get the mathcing question-answering instructions for the question.
-
-        :param model: The language model to use.
-        """
-        from edsl.prompts.Prompt import Prompt
-
-        if model in self.model_instructions:
-            return Prompt(text=self.model_instructions[model])
-        else:
-            if hasattr(self, "new_default_instructions"):
-                return self.new_default_instructions
-            else:
-                return self.applicable_prompts(model)[0]()

edsl/questions/Quick.py

@@ -1,41 +0,0 @@
-from edsl import (
-    QuestionFreeText,
-    QuestionMultipleChoice,
-    Survey,
-    QuestionList,
-    Question,
-)
-
-
-def Quick(question_text):
-    q_type = QuestionMultipleChoice(
-        question_text=f"A researcher is asking a language model this: {question_text}. What is the most appropriate type of question to ask?",
-        question_name="potential_question_type",
-        question_options=["multiple_choice", "list", "free_text"],
-    )
-
-    q_name = QuestionFreeText(
-        question_text=f"A researcher is asking a language model this: {question_text}. What is a good name for this question that's a valid python identifier? Just return the proposed identifer",
-        question_name="potential_question_name",
-    )
-
-    q_options = QuestionList(
-        question_text=f"A research is asking this question: { question_text }. What are the possible options for this question?",
-        question_name="potential_question_options",
-    )
-
-    survey = Survey([q_type, q_name, q_options]).add_skip_rule(
-        q_options, "{{ potential_question_type }} != 'multiple_choice'"
-    )
-    return survey
-    # results = survey.run()
-    # question_type = results.select("potential_question_type").first()
-    # question_options = results.select("potential_question_options").first()
-    # question_name = results.select("potential_question_name").first()
-    # print("Question Type: ", question_type)
-    # print("Question Name: ", question_name)
-    # print("Question Options: ", question_options)
-    # if question_options == None:
-    #     return Question(question_type, question_name = question_name)
-    # else:
-    #     return Question(question_type, question_name = question_name, question_options = question_options)