edsl 0.1.32__py3-none-any.whl → 0.1.33__py3-none-any.whl

edsl/language_models/fake_openai_service.py

@@ -0,0 +1,61 @@
+import threading
+import asyncio
+from fastapi import FastAPI, Request
+from fastapi.responses import JSONResponse
+import uvicorn
+import json
+from typing import Any
+
+app = FastAPI()
+
+
+async def generate_response(question_number: int) -> dict:
+    # Simulate some asynchronous work
+    await asyncio.sleep(1)
+    return {
+        "id": "chatcmpl-123",
+        "object": "chat.completion",
+        "created": 1677652288,
+        "model": "gpt-3.5-turbo-0613",
+        "choices": [
+            {
+                "index": 0,
+                "message": {
+                    "role": "assistant",
+                    "content": json.dumps(
+                        {"answer": f"SPAM for question {question_number}!"}
+                    ),
+                },
+                "finish_reason": "stop",
+            }
+        ],
+        "usage": {"prompt_tokens": 9, "completion_tokens": 12, "total_tokens": 21},
+    }
+
+
+@app.post("/v1/chat/completions")
+async def chat_completions(request: Request):
+    body = await request.json()
+    user_prompt = body["messages"][-1]["content"]
+    question_number = int(user_prompt.split("XX")[1])
+
+    response = await generate_response(question_number)
+    return JSONResponse(content=response)
+
+
+def run_server():
+    uvicorn.run(app, host="127.0.0.1", port=8000)
+
+
+if __name__ == "__main__":
+    # Start the server in a separate thread
+    server_thread = threading.Thread(target=run_server)
+    server_thread.start()
+
+# Your main code here
+# ...
+
+# To use this with the OpenAI SDK:
+# from openai import AsyncOpenAI
+# client = AsyncOpenAI(base_url="http://127.0.0.1:8000/v1", api_key="fake_key")
+# response = await client.chat.completions.create(model="gpt-3.5-turbo", messages=[...])

edsl/language_models/registry.py

@@ -1,5 +1,11 @@
 import textwrap
 from random import random
+from edsl.config import CONFIG
+
+# if "EDSL_DEFAULT_MODEL" not in CONFIG:
+#     default_model = "test"
+# else:
+#     default_model = CONFIG.get("EDSL_DEFAULT_MODEL")
 
 
 def get_model_class(model_name, registry=None):
@@ -27,20 +33,24 @@ class Meta(type):
 
 
 class Model(metaclass=Meta):
-    default_model = "gpt-4-1106-preview"
+    default_model = CONFIG.get("EDSL_DEFAULT_MODEL")
 
-    def __new__(cls, model_name=None, registry=None, *args, **kwargs):
+    def __new__(
+        cls, model_name=None, registry=None, service_name=None, *args, **kwargs
+    ):
         # Map index to the respective subclass
         if model_name is None:
-            model_name = cls.default_model
+            model_name = (
+                cls.default_model
+            )  # when model_name is None, use the default model, set in the config file
         from edsl.inference_services.registry import default
 
         registry = registry or default
 
-        if isinstance(model_name, int):
+        if isinstance(model_name, int):  # can refer to a model by index
             model_name = cls.available(name_only=True)[model_name]
 
-        factory = registry.create_model_factory(model_name)
+        factory = registry.create_model_factory(model_name, service_name=service_name)
         return factory(*args, **kwargs)
 
     @classmethod
@@ -58,11 +68,18 @@ class Model(metaclass=Meta):
         return [r._inference_service_ for r in registry.services]
 
     @classmethod
-    def available(cls, search_term=None, name_only=False, registry=None):
+    def available(cls, search_term=None, name_only=False, registry=None, service=None):
         from edsl.inference_services.registry import default
 
         registry = registry or default
         full_list = registry.available()
+
+        if service is not None:
+            if service not in cls.services(registry=registry):
+                raise ValueError(f"Service {service} not found in available services.")
+
+            full_list = [m for m in full_list if m[1] == service]
+
         if search_term is None:
             if name_only:
                 return [m[0] for m in full_list]

edsl/language_models/repair.py

@@ -141,25 +141,6 @@ def repair(
     return repair_wrapper(bad_json, error_message, user_prompt, system_prompt, cache)
 
 
-# Example usage:
-# result, success = repair_wrapper('{"name": "John Doe", "age": 30,}')  # example bad JSON
-
-
-# def repair_wrapper(bad_json, error_message=""):
-#     loop = asyncio.get_event_loop()
-#     if loop.is_running():
-#         # Add repair as a task to the running loop
-#         task = loop.create_task(repair(bad_json, error_message))
-#         return task
-#     else:
-#         # Run a new event loop for repair
-#         return loop.run_until_complete(repair(bad_json, error_message))
-
-
-# Example usage:
-# result, success = repair_wrapper('{"name": "John Doe", "age": 30,}')  # example bad JSON
-
-
 if __name__ == "__main__":
     bad_json = """
     {

edsl/language_models/utilities.py

@@ -0,0 +1,61 @@
+import asyncio
+from typing import Any
+from edsl import Survey
+from edsl.config import CONFIG
+from edsl.enums import InferenceServiceType
+from edsl.language_models.LanguageModel import LanguageModel
+from edsl.questions import QuestionFreeText
+
+
+def create_survey(num_questions: int, chained: bool = True, take_scenario=False):
+    survey = Survey()
+    for i in range(num_questions):
+        if take_scenario:
+            q = QuestionFreeText(
+                question_text=f"XX{i}XX and {{scenario_value }}",
+                question_name=f"question_{i}",
+            )
+        else:
+            q = QuestionFreeText(
+                question_text=f"XX{i}XX", question_name=f"question_{i}"
+            )
+        survey.add_question(q)
+        if i > 0 and chained:
+            survey.add_targeted_memory(f"question_{i}", f"question_{i-1}")
+    return survey
+
+
+def create_language_model(
+    exception: Exception, fail_at_number: int, never_ending=False
+):
+    class LanguageModelFromUtilities(LanguageModel):
+        _model_ = "test"
+        _parameters_ = {"temperature": 0.5}
+        _inference_service_ = InferenceServiceType.TEST.value
+        key_sequence = ["message", 0, "text"]
+        usage_sequence = ["usage"]
+        input_token_name = "prompt_tokens"
+        output_token_name = "completion_tokens"
+        _rpm = 1000000000000
+        _tpm = 1000000000000
+
+        async def async_execute_model_call(
+            self, user_prompt: str, system_prompt: str
+        ) -> dict[str, Any]:
+            question_number = int(
+                user_prompt.split("XX")[1]
+            )  ## grabs the question number from the prompt
+            await asyncio.sleep(0.1)
+            if never_ending:  ## you're not going anywhere buddy
+                await asyncio.sleep(float("inf"))
+            if question_number == fail_at_number:
+                if asyncio.iscoroutinefunction(exception):
+                    await exception()
+                else:
+                    raise exception
+            return {
+                "message": [{"text": "SPAM!"}],
+                "usage": {"prompt_tokens": 1, "completion_tokens": 1},
+            }
+
+    return LanguageModelFromUtilities

edsl/notebooks/Notebook.py

@@ -2,7 +2,7 @@
 
 from __future__ import annotations
 import json
-from typing import Dict, List, Optional
+from typing import Dict, List, Optional, Union
 from uuid import uuid4
 from edsl.Base import Base
 from edsl.utilities.decorators import add_edsl_version, remove_edsl_version
@@ -56,6 +56,8 @@ class Notebook(Base):
 
     @classmethod
     def from_script(cls, path: str, name: Optional[str] = None) -> "Notebook":
+        import nbformat
+
         # Read the script file
         with open(path, "r") as script_file:
             script_content = script_file.read()
@@ -92,12 +94,26 @@ class Notebook(Base):
         """
         return self.data == other.data
 
+    def __hash__(self) -> int:
+        """
+        Allow the model to be used as a key in a dictionary.
+        """
+        from edsl.utilities.utilities import dict_hash
+
+        return dict_hash(self.data["cells"])
+
+    def _to_dict(self) -> dict:
+        """
+        Serialize to a dictionary.
+        """
+        return {"name": self.name, "data": self.data}
+
     @add_edsl_version
     def to_dict(self) -> dict:
         """
         Convert a Notebook to a dictionary.
         """
-        return {"name": self.name, "data": self.data}
+        return self._to_dict()
 
     @classmethod
     @remove_edsl_version
@@ -111,6 +127,8 @@ class Notebook(Base):
         """
         Save the notebook at the specified filepath.
         """
+        import nbformat
+
         nbformat.write(nbformat.from_dict(self.data), fp=path)
 
     def print(self):

edsl/prompts/Prompt.py

@@ -3,13 +3,17 @@ from typing import Optional
 from abc import ABC
 from typing import Any, List
 
+from jinja2 import Environment, FileSystemLoader
+from typing import Union, Dict
+from pathlib import Path
+
 from rich.table import Table
 from jinja2 import Template, Environment, meta, TemplateSyntaxError, Undefined
 
 
 class PreserveUndefined(Undefined):
     def __str__(self):
-        return "{{ " + self._undefined_name + " }}"
+        return "{{ " + str(self._undefined_name) + " }}"
 
 
 from edsl.exceptions.prompts import TemplateRenderError
@@ -70,6 +74,50 @@ class PromptBase(
             text = f.read()
         return cls(text=text)
 
+    @classmethod
+    def from_template(
+        cls,
+        file_name: str,
+        path_to_folder: Optional[Union[str, Path]] = None,
+        **kwargs: Dict[str, Any],
+    ) -> "PromptBase":
+        """Create a `PromptBase` from a Jinja template.
+
+        Args:
+            file_name (str): The name of the Jinja template file.
+            path_to_folder (Union[str, Path]): The path to the folder containing the template.
+                            Can be absolute or relative.
+            **kwargs: Variables to be passed to the template for rendering.
+
+        Returns:
+            PromptBase: An instance of PromptBase with the rendered template as text.
+        """
+        # if file_name lacks the .j2 extension, add it
+        if not file_name.endswith(".jinja"):
+            file_name += ".jinja"
+
+        # Convert path_to_folder to a Path object if it's a string
+        if path_to_folder is None:
+            from importlib import resources
+            import os
+
+            path_to_folder = resources.path("edsl.questions", "prompt_templates")
+
+        try:
+            folder_path = Path(path_to_folder)
+        except Exception as e:
+            raise ValueError(f"Invalid path: {path_to_folder}. Error: {e}")
+
+        with open(folder_path.joinpath(file_name), "r") as f:
+            text = f.read()
+        return cls(text=text)
+        # Resolve the path to get the absolute path
+        # absolute_path = folder_path.resolve()
+        # env = Environment(loader=FileSystemLoader(absolute_path))
+        # template = env.get_template(file_name)
+        # rendered_text = template.render({})
+        # return cls(text=rendered_text)
+
     @property
     def text(self):
         """Return the `Prompt` text."""
@@ -247,7 +295,9 @@ class PromptBase(
                 "Too much nesting - you created an infinite loop here, pal"
             )
         except TemplateSyntaxError as e:
-            raise TemplateRenderError(f"Template syntax error: {e}")
+            raise TemplateRenderError(
+                f"Template syntax error: {e}. Bad template: {text}"
+            )
 
     def to_dict(self) -> dict[str, Any]:
         """Return the `Prompt` as a dictionary.

edsl/questions/AnswerValidatorMixin.py

@@ -16,19 +16,27 @@ class AnswerValidatorMixin:
     - Question specific validation: validators for specific question types
     """
 
+    def failing_job(self):
+        from edsl import Agent
+
+        a = Agent()
+
+        def f(self, question, scenario):
+            return []
+
+        a.add_direct_question_answering_method(f, validate_response=True)
+        from edsl import QuestionNumerical
+
+        q = QuestionNumerical.example()
+        results = q.by(a).run()
+        return results
+
     #####################
     # TEMPLATE VALIDATION
     #####################
     def _validate_answer_template_basic(self, answer: Any) -> None:
         """Check that the answer (i) is a dictionary (ii) has an 'answer' key.
 
-        >>> avm = AnswerValidatorMixin()
-        >>> avm._validate_answer_template_basic({'answer': 1})
-        >>> avm._validate_answer_template_basic([])
-        Traceback (most recent call last):
-        ...
-        edsl.exceptions.questions.QuestionAnswerValidationError: Answer must be a dictionary (got []).
-
         - E.g., both {'answer': 1} and {'answer': {'a': 1}, 'other_key'=[1,2,3]} are valid
         """
         if not isinstance(answer, dict):
@@ -56,14 +64,9 @@ class AnswerValidatorMixin:
     def _validate_answer_key_value_numeric(
         self, answer: dict[str, Any], key: str
     ) -> None:
-        """Check that the value of a key is numeric (int or float).
-
-        >>> avm = AnswerValidatorMixin()
-        >>> avm._validate_answer_key_value_numeric({'answer': 1}, 'answer')
-        >>> avm._validate_answer_key_value_numeric({'answer': 'poo'}, 'answer')
-        Traceback (most recent call last):
-        ...
-        edsl.exceptions.questions.QuestionAnswerValidationError: Answer should be numerical (int or float). Got 'poo'
+        """Check that the value is numeric (int or float).
+        Can also deal with strings that contain commas and other characters.
+
         """
         value = answer.get(key)
         initial_value = value
@@ -128,15 +131,6 @@ class AnswerValidatorMixin:
 
         :param answer: Answer to validate
 
-        >>> avm = AnswerValidatorMixin()
-        >>> avm.question_options = ["a", "b", "c"]
-        >>> avm.min_selections = 1
-        >>> avm.max_selections = 2
-        >>> avm._validate_answer_checkbox({"answer": ["0", "1"]})
-        >>> avm._validate_answer_checkbox({"answer": []})
-        Traceback (most recent call last):
-        ...
-        edsl.exceptions.questions.QuestionAnswerValidationError:...
 
         Check that answer["answer"]:
         - has elements that are strings, bytes-like objects or real numbers evaluating to integers
@@ -287,6 +281,9 @@ class AnswerValidatorMixin:
 
 
 if __name__ == "__main__":
-    import doctest
+    pass
+    # import doctest
+
+    # doctest.testmod(optionflags=doctest.ELLIPSIS)
 
-    doctest.testmod(optionflags=doctest.ELLIPSIS)
+    # results = AnswerValidatorMixin().failing_job()