edsl 0.1.33.dev1__py3-none-any.whl → 0.1.33.dev2__py3-none-any.whl

edsl/language_models/registry.py

@@ -1,5 +1,11 @@
 import textwrap
 from random import random
+from edsl.config import CONFIG
+
+if "EDSL_DEFAULT_MODEL" not in CONFIG:
+    default_model = "test"
+else:
+    default_model = CONFIG.get("EDSL_DEFAULT_MODEL")
 
 
 def get_model_class(model_name, registry=None):
@@ -27,7 +33,7 @@ class Meta(type):
 
 
 class Model(metaclass=Meta):
-    default_model = "gpt-4-1106-preview"
+    default_model = default_model
 
     def __new__(cls, model_name=None, registry=None, *args, **kwargs):
         # Map index to the respective subclass
@@ -58,11 +64,18 @@ class Model(metaclass=Meta):
         return [r._inference_service_ for r in registry.services]
 
     @classmethod
-    def available(cls, search_term=None, name_only=False, registry=None):
+    def available(cls, search_term=None, name_only=False, registry=None, service=None):
         from edsl.inference_services.registry import default
 
         registry = registry or default
         full_list = registry.available()
+
+        if service is not None:
+            if service not in cls.services(registry=registry):
+                raise ValueError(f"Service {service} not found in available services.")
+
+            full_list = [m for m in full_list if m[1] == service]
+
         if search_term is None:
             if name_only:
                 return [m[0] for m in full_list]

edsl/language_models/repair.py

@@ -141,25 +141,6 @@ def repair(
     return repair_wrapper(bad_json, error_message, user_prompt, system_prompt, cache)
 
 
-# Example usage:
-# result, success = repair_wrapper('{"name": "John Doe", "age": 30,}')  # example bad JSON
-
-
-# def repair_wrapper(bad_json, error_message=""):
-#     loop = asyncio.get_event_loop()
-#     if loop.is_running():
-#         # Add repair as a task to the running loop
-#         task = loop.create_task(repair(bad_json, error_message))
-#         return task
-#     else:
-#         # Run a new event loop for repair
-#         return loop.run_until_complete(repair(bad_json, error_message))
-
-
-# Example usage:
-# result, success = repair_wrapper('{"name": "John Doe", "age": 30,}')  # example bad JSON
-
-
 if __name__ == "__main__":
     bad_json = """
     {

edsl/language_models/utilities.py

@@ -0,0 +1,61 @@
+import asyncio
+from typing import Any
+from edsl import Survey
+from edsl.config import CONFIG
+from edsl.enums import InferenceServiceType
+from edsl.language_models.LanguageModel import LanguageModel
+from edsl.questions import QuestionFreeText
+
+
+def create_survey(num_questions: int, chained: bool = True, take_scenario=False):
+    survey = Survey()
+    for i in range(num_questions):
+        if take_scenario:
+            q = QuestionFreeText(
+                question_text=f"XX{i}XX and {{scenario_value }}",
+                question_name=f"question_{i}",
+            )
+        else:
+            q = QuestionFreeText(
+                question_text=f"XX{i}XX", question_name=f"question_{i}"
+            )
+        survey.add_question(q)
+        if i > 0 and chained:
+            survey.add_targeted_memory(f"question_{i}", f"question_{i-1}")
+    return survey
+
+
+def create_language_model(
+    exception: Exception, fail_at_number: int, never_ending=False
+):
+    class LanguageModelFromUtilities(LanguageModel):
+        _model_ = "test"
+        _parameters_ = {"temperature": 0.5}
+        _inference_service_ = InferenceServiceType.TEST.value
+        key_sequence = ["message", 0, "text"]
+        usage_sequence = ["usage"]
+        input_token_name = "prompt_tokens"
+        output_token_name = "completion_tokens"
+        _rpm = 1000000000000
+        _tpm = 1000000000000
+
+        async def async_execute_model_call(
+            self, user_prompt: str, system_prompt: str
+        ) -> dict[str, Any]:
+            question_number = int(
+                user_prompt.split("XX")[1]
+            )  ## grabs the question number from the prompt
+            await asyncio.sleep(0.1)
+            if never_ending:  ## you're not going anywhere buddy
+                await asyncio.sleep(float("inf"))
+            if question_number == fail_at_number:
+                if asyncio.iscoroutinefunction(exception):
+                    await exception()
+                else:
+                    raise exception
+            return {
+                "message": [{"text": "SPAM!"}],
+                "usage": {"prompt_tokens": 1, "completion_tokens": 1},
+            }
+
+    return LanguageModelFromUtilities

edsl/prompts/Prompt.py

@@ -3,13 +3,17 @@ from typing import Optional
 from abc import ABC
 from typing import Any, List
 
+from jinja2 import Environment, FileSystemLoader
+from typing import Union, Dict
+from pathlib import Path
+
 from rich.table import Table
 from jinja2 import Template, Environment, meta, TemplateSyntaxError, Undefined
 
 
 class PreserveUndefined(Undefined):
     def __str__(self):
-        return "{{ " + self._undefined_name + " }}"
+        return "{{ " + str(self._undefined_name) + " }}"
 
 
 from edsl.exceptions.prompts import TemplateRenderError
@@ -70,6 +74,50 @@ class PromptBase(
             text = f.read()
         return cls(text=text)
 
+    @classmethod
+    def from_template(
+        cls,
+        file_name: str,
+        path_to_folder: Optional[Union[str, Path]] = None,
+        **kwargs: Dict[str, Any],
+    ) -> "PromptBase":
+        """Create a `PromptBase` from a Jinja template.
+
+        Args:
+            file_name (str): The name of the Jinja template file.
+            path_to_folder (Union[str, Path]): The path to the folder containing the template.
+                            Can be absolute or relative.
+            **kwargs: Variables to be passed to the template for rendering.
+
+        Returns:
+            PromptBase: An instance of PromptBase with the rendered template as text.
+        """
+        # if file_name lacks the .j2 extension, add it
+        if not file_name.endswith(".jinja"):
+            file_name += ".jinja"
+
+        # Convert path_to_folder to a Path object if it's a string
+        if path_to_folder is None:
+            from importlib import resources
+            import os
+
+            path_to_folder = resources.path("edsl.questions", "prompt_templates")
+
+        try:
+            folder_path = Path(path_to_folder)
+        except Exception as e:
+            raise ValueError(f"Invalid path: {path_to_folder}. Error: {e}")
+
+        with open(folder_path.joinpath(file_name), "r") as f:
+            text = f.read()
+        return cls(text=text)
+        # Resolve the path to get the absolute path
+        # absolute_path = folder_path.resolve()
+        # env = Environment(loader=FileSystemLoader(absolute_path))
+        # template = env.get_template(file_name)
+        # rendered_text = template.render({})
+        # return cls(text=rendered_text)
+
     @property
     def text(self):
         """Return the `Prompt` text."""
@@ -247,7 +295,9 @@ class PromptBase(
                 "Too much nesting - you created an infinite loop here, pal"
             )
         except TemplateSyntaxError as e:
-            raise TemplateRenderError(f"Template syntax error: {e}")
+            raise TemplateRenderError(
+                f"Template syntax error: {e}. Bad template: {text}"
+            )
 
     def to_dict(self) -> dict[str, Any]:
         """Return the `Prompt` as a dictionary.

edsl/questions/AnswerValidatorMixin.py

@@ -16,19 +16,27 @@ class AnswerValidatorMixin:
     - Question specific validation: validators for specific question types
     """
 
+    def failing_job(self):
+        from edsl import Agent
+
+        a = Agent()
+
+        def f(self, question, scenario):
+            return []
+
+        a.add_direct_question_answering_method(f, validate_response=True)
+        from edsl import QuestionNumerical
+
+        q = QuestionNumerical.example()
+        results = q.by(a).run()
+        return results
+
     #####################
     # TEMPLATE VALIDATION
     #####################
     def _validate_answer_template_basic(self, answer: Any) -> None:
         """Check that the answer (i) is a dictionary (ii) has an 'answer' key.
 
-        >>> avm = AnswerValidatorMixin()
-        >>> avm._validate_answer_template_basic({'answer': 1})
-        >>> avm._validate_answer_template_basic([])
-        Traceback (most recent call last):
-        ...
-        edsl.exceptions.questions.QuestionAnswerValidationError: Answer must be a dictionary (got []).
-
         - E.g., both {'answer': 1} and {'answer': {'a': 1}, 'other_key'=[1,2,3]} are valid
         """
         if not isinstance(answer, dict):
@@ -56,14 +64,9 @@ class AnswerValidatorMixin:
     def _validate_answer_key_value_numeric(
         self, answer: dict[str, Any], key: str
     ) -> None:
-        """Check that the value of a key is numeric (int or float).
-
-        >>> avm = AnswerValidatorMixin()
-        >>> avm._validate_answer_key_value_numeric({'answer': 1}, 'answer')
-        >>> avm._validate_answer_key_value_numeric({'answer': 'poo'}, 'answer')
-        Traceback (most recent call last):
-        ...
-        edsl.exceptions.questions.QuestionAnswerValidationError: Answer should be numerical (int or float). Got 'poo'
+        """Check that the value is numeric (int or float).
+        Can also deal with strings that contain commas and other characters.
+
         """
         value = answer.get(key)
         initial_value = value
@@ -128,15 +131,6 @@ class AnswerValidatorMixin:
 
         :param answer: Answer to validate
 
-        >>> avm = AnswerValidatorMixin()
-        >>> avm.question_options = ["a", "b", "c"]
-        >>> avm.min_selections = 1
-        >>> avm.max_selections = 2
-        >>> avm._validate_answer_checkbox({"answer": ["0", "1"]})
-        >>> avm._validate_answer_checkbox({"answer": []})
-        Traceback (most recent call last):
-        ...
-        edsl.exceptions.questions.QuestionAnswerValidationError:...
 
         Check that answer["answer"]:
         - has elements that are strings, bytes-like objects or real numbers evaluating to integers
@@ -287,6 +281,9 @@ class AnswerValidatorMixin:
 
 
 if __name__ == "__main__":
-    import doctest
+    pass
+    # import doctest
+
+    # doctest.testmod(optionflags=doctest.ELLIPSIS)
 
-    doctest.testmod(optionflags=doctest.ELLIPSIS)
+    # results = AnswerValidatorMixin().failing_job()