edsl 0.1.43__py3-none-any.whl → 0.1.45__py3-none-any.whl

edsl/language_models/model.py

@@ -1,6 +1,6 @@
 import textwrap
 from random import random
-from typing import Optional, TYPE_CHECKING, List
+from typing import Optional, TYPE_CHECKING, List, Callable
 
 from edsl.utilities.PrettyList import PrettyList
 from edsl.config import CONFIG
@@ -11,17 +11,25 @@ from edsl.inference_services.InferenceServicesCollection import (
 from edsl.inference_services.data_structures import AvailableModels
 from edsl.inference_services.InferenceServiceABC import InferenceServiceABC
 from edsl.enums import InferenceServiceLiteral
+from edsl.exceptions.inference_services import InferenceServiceError
 
 if TYPE_CHECKING:
     from edsl.results.Dataset import Dataset
 
 
-def get_model_class(model_name, registry: Optional[InferenceServicesCollection] = None):
+def get_model_class(
+    model_name,
+    registry: Optional[InferenceServicesCollection] = None,
+    service_name: Optional[InferenceServiceLiteral] = None,
+):
     from edsl.inference_services.registry import default
 
     registry = registry or default
-    factory = registry.create_model_factory(model_name)
-    return factory
+    try:
+        factory = registry.create_model_factory(model_name, service_name=service_name)
+        return factory
+    except (InferenceServiceError, Exception) as e:
+        return Model._handle_model_error(model_name, e)
 
 
 class Meta(type):
@@ -36,6 +44,9 @@ class Meta(type):
         To get the default model, you can leave out the model name. 
         To see the available models, you can do:
         >>> Model.available()
+
+        Or to see the models for a specific service, you can do:
+        >>> Model.available(service='openai')
         """
         )
 
@@ -58,6 +69,33 @@ class Model(metaclass=Meta):
         """Set a new registry"""
         cls._registry = registry
 
+    @classmethod
+    def _handle_model_error(cls, model_name: str, error: Exception):
+        """Handle errors from model creation and execution with notebook-aware behavior."""
+        if isinstance(error, InferenceServiceError):
+            services = [s._inference_service_ for s in cls.get_registry().services]
+            message = (
+                f"Model '{model_name}' not found in any services.\n"
+                "It is likely that our registry is just out of date.\n"
+                "Simply adding the service name to your model call should fix this.\n"
+                f"Available services are: {services}\n"
+                f"To specify a model with a service, use:\n"
+                f'Model("{model_name}", service_name="<service_name>")'
+            )
+        else:
+            message = f"An error occurred: {str(error)}"
+
+        # Check if we're in a notebook environment
+        try:
+            get_ipython()
+            print(message)
+            return None
+        except NameError:
+            # Not in a notebook, raise the exception
+            if isinstance(error, InferenceServiceError):
+                raise InferenceServiceError(message)
+            raise error
+
     def __new__(
         cls,
         model_name: Optional[str] = None,
@@ -66,12 +104,13 @@ class Model(metaclass=Meta):
         *args,
         **kwargs,
     ):
-        "Instantiate a new language model."
+        """Instantiate a new language model.
+        >>> Model()
+        Model(...)
+        """
         # Map index to the respective subclass
         if model_name is None:
-            model_name = (
-                cls.default_model
-            )  # when model_name is None, use the default model, set in the config file
+            model_name = cls.default_model
 
         if registry is not None:
             cls.set_registry(registry)
@@ -79,10 +118,13 @@ class Model(metaclass=Meta):
         if isinstance(model_name, int):  # can refer to a model by index
             model_name = cls.available(name_only=True)[model_name]
 
-        factory = cls.get_registry().create_model_factory(
-            model_name, service_name=service_name
-        )
-        return factory(*args, **kwargs)
+        try:
+            factory = cls.get_registry().create_model_factory(
+                model_name, service_name=service_name
+            )
+            return factory(*args, **kwargs)
+        except (InferenceServiceError, Exception) as e:
+            return cls._handle_model_error(model_name, e)
 
     @classmethod
     def add_model(cls, service_name, model_name) -> None:
@@ -95,28 +137,25 @@ class Model(metaclass=Meta):
         >>> Model.service_classes()
         [...]
         """
-        return [r for r in cls.services(name_only=True)]
+        return [r for r in cls.services()]
 
     @classmethod
     def services(cls, name_only: bool = False) -> List[str]:
-        """Returns a list of services, annotated with whether the user has local keys for them."""
-        services_with_local_keys = set(cls.key_info().select("service").to_list())
-        f = lambda service_name: (
-            "yes" if service_name in services_with_local_keys else " "
-        )
-        if name_only:
-            return PrettyList(
-                [r._inference_service_ for r in cls.get_registry().services],
-                columns=["Service Name"],
-            )
-        else:
-            return PrettyList(
+        """Returns a list of services excluding 'test', sorted alphabetically.
+
+        >>> Model.services()
+        [...]
+        """
+        return PrettyList(
+            sorted(
                 [
-                    (r._inference_service_, f(r._inference_service_))
+                    [r._inference_service_]
                     for r in cls.get_registry().services
-                ],
-                columns=["Service Name", "Local key?"],
-            )
+                    if r._inference_service_.lower() != "test"
+                ]
+            ),
+            columns=["Service Name"],
+        )
 
     @classmethod
     def services_with_local_keys(cls) -> set:
@@ -166,7 +205,15 @@ class Model(metaclass=Meta):
         search_term: str = None,
         name_only: bool = False,
         service: Optional[str] = None,
+        force_refresh: bool = False,
     ):
+        """Get available models
+
+        >>> Model.available()
+        [...]
+        >>> Model.available(service='openai')
+        [...]
+        """
         # if search_term is None and service is None:
         #     print("Getting available models...")
         #     print("You have local keys for the following services:")
@@ -177,13 +224,16 @@ class Model(metaclass=Meta):
         #     return None
 
         if service is not None:
-            if service not in cls.services(name_only=True):
+            known_services = [x[0] for x in cls.services(name_only=True)]
+            if service not in known_services:
                 raise ValueError(
                     f"Service {service} not found in available services.",
-                    f"Available services are: {cls.services()}",
+                    f"Available services are: {known_services}",
                 )
 
-        full_list = cls.get_registry().available(service=service)
+        full_list = cls.get_registry().available(
+            service=service, force_refresh=force_refresh
+        )
 
         if search_term is None:
             if name_only:
@@ -287,6 +337,9 @@ class Model(metaclass=Meta):
         """
         Returns an example Model instance.
 
+        >>> Model.example()
+        Model(...)
+
         :param randomize: If True, the temperature is set to a random decimal between 0 and 1.
         """
         temperature = 0.5 if not randomize else round(random(), 2)
@@ -299,7 +352,7 @@ if __name__ == "__main__":
 
     doctest.testmod(optionflags=doctest.ELLIPSIS)
 
-    available = Model.available()
-    m = Model("gpt-4-1106-preview")
-    results = m.execute_model_call("Hello world")
-    print(results)
+    # available = Model.available()
+    # m = Model("gpt-4-1106-preview")
+    # results = m.execute_model_call("Hello world")
+    # print(results)

edsl/questions/QuestionBase.py

@@ -85,6 +85,9 @@ class QuestionBase(
         >>> Q.example()._simulate_answer()
         {'answer': '...', 'generated_tokens': ...}
         """
+        if self.question_type == "free_text":
+            return {"answer": "Hello, how are you?", 'generated_tokens': "Hello, how are you?"}
+        
         simulated_answer = self.fake_data_factory.build().dict()
         if human_readable and hasattr(self, "question_options") and self.use_code:
             simulated_answer["answer"] = [
@@ -432,6 +435,24 @@ class QuestionBase(
 
         return Survey([self])
 
+    def humanize(
+        self,
+        project_name: str = "Project",
+        survey_description: Optional[str] = None,
+        survey_alias: Optional[str] = None,
+        survey_visibility: Optional["VisibilityType"] = "unlisted",
+    ) -> dict:
+        """
+        Turn a single question into a survey and send the survey to Coop.
+
+        Then, create a project on Coop so you can share the survey with human respondents.
+        """
+        s = self.to_survey()
+        project_details = s.humanize(
+            project_name, survey_description, survey_alias, survey_visibility
+        )
+        return project_details
+
     def by(self, *args) -> "Jobs":
         """Turn a single question into a survey and then a Job."""
         from edsl.surveys.Survey import Survey

edsl/questions/QuestionBasePromptsMixin.py

@@ -187,6 +187,73 @@ class QuestionBasePromptsMixin:
         from edsl.prompts import Prompt
 
         return Prompt(self.question_presentation) + Prompt(self.answering_instructions)
+    
+
+    def detailed_parameters_by_key(self) -> dict[str, set[tuple[str, ...]]]:
+        """
+        Return a dictionary of parameters by key.
+
+        >>> from edsl import QuestionMultipleChoice
+        >>> QuestionMultipleChoice.example().detailed_parameters_by_key()
+        {'question_name': set(), 'question_text': set()}
+
+        >>> from edsl import QuestionFreeText
+        >>> q = QuestionFreeText(question_name = "example", question_text = "What is your name, {{ nickname }}, based on {{ q0.answer }}?")
+        >>> r = q.detailed_parameters_by_key()
+        >>> r == {'question_name': set(), 'question_text': {('q0', 'answer'), ('nickname',)}}
+        True
+        """
+        params_by_key = {}
+        for key, value in self.data.items():
+            if isinstance(value, str):
+                params_by_key[key] = self.extract_parameters(value)
+        return params_by_key
+
+    @staticmethod
+    def extract_parameters(txt: str) -> set[tuple[str, ...]]:
+        """Return all parameters of the question as tuples representing their full paths.
+        
+        :param txt: The text to extract parameters from.
+        :return: A set of tuples representing the parameters.
+
+        >>> from edsl.questions import QuestionMultipleChoice
+        >>> d = QuestionMultipleChoice.example().extract_parameters("What is your name, {{ nickname }}, based on {{ q0.answer }}?")
+        >>> d =={('nickname',), ('q0', 'answer')}
+        True
+        """
+        from jinja2 import Environment, nodes
+
+        env = Environment()
+        #txt = self._all_text()
+        ast = env.parse(txt)
+        
+        variables = set()
+        processed_nodes = set()  # Keep track of nodes we've processed
+        
+        def visit_node(node, path=()):
+            if id(node) in processed_nodes:
+                return
+            processed_nodes.add(id(node))
+            
+            if isinstance(node, nodes.Name):
+                # Only add the name if we're not in the middle of building a longer path
+                if not path:
+                    variables.add((node.name,))
+                else:
+                    variables.add((node.name,) + path)
+            elif isinstance(node, nodes.Getattr):
+                # Build path from bottom up
+                new_path = (node.attr,) + path
+                visit_node(node.node, new_path)
+        
+        for node in ast.find_all((nodes.Name, nodes.Getattr)):
+            visit_node(node)
+
+        return variables
+    
+    @property
+    def detailed_parameters(self):
+        return [".".join(p) for p in self.extract_parameters(self._all_text())]
 
     @property
     def parameters(self) -> set[str]:
@@ -219,3 +286,39 @@ class QuestionBasePromptsMixin:
                 return self.new_default_instructions
             else:
                 return self.applicable_prompts(model)[0]()
+
+    @staticmethod
+    def sequence_in_dict(d: dict, path: tuple[str, ...]) -> tuple[bool, any]:
+        """Check if a sequence of nested keys exists in a dictionary and return the value.
+        
+        Args:
+            d: The dictionary to check
+            path: Tuple of keys representing the nested path
+            
+        Returns:
+            tuple[bool, any]: (True, value) if the path exists, (False, None) otherwise
+            
+        Example:
+            >>> sequence_in_dict = QuestionBasePromptsMixin.sequence_in_dict
+            >>> d = {'a': {'b': {'c': 1}}}
+            >>> sequence_in_dict(d, ('a', 'b', 'c'))
+            (True, 1)
+            >>> sequence_in_dict(d, ('a', 'b', 'd'))
+            (False, None)
+            >>> sequence_in_dict(d, ('x',))
+            (False, None)
+        """
+        try:
+            current = d
+            for key in path:
+                current = current.get(key)
+                if current is None:
+                    return (False, None)
+            return (True, current)
+        except (AttributeError, TypeError):
+            return (False, None)
+
+
+if __name__ == "__main__":
+    import doctest
+    doctest.testmod()

edsl/questions/QuestionFreeText.py

@@ -2,7 +2,7 @@ from __future__ import annotations
 from typing import Any, Optional
 from uuid import uuid4
 
-from pydantic import field_validator
+from pydantic import field_validator, model_validator
 
 from edsl.questions.QuestionBase import QuestionBase
 from edsl.questions.response_validator_abc import ResponseValidatorABC
@@ -24,6 +24,17 @@ class FreeTextResponse(BaseModel):
     answer: str
     generated_tokens: Optional[str] = None
 
+    @model_validator(mode='after')
+    def validate_tokens_match_answer(self):
+        if self.generated_tokens is not None:  # If generated_tokens exists
+            # Ensure exact string equality
+            if self.answer.strip() != self.generated_tokens.strip():  # They MUST match exactly
+                raise ValueError(
+                    f"answer '{self.answer}' must exactly match generated_tokens '{self.generated_tokens}'. "
+                    f"Type of answer: {type(self.answer)}, Type of tokens: {type(self.generated_tokens)}"
+                )
+        return self
+
 
 class FreeTextResponseValidator(ResponseValidatorABC):
     required_params = []
@@ -37,10 +48,16 @@ class FreeTextResponseValidator(ResponseValidatorABC):
     ]
 
     def fix(self, response, verbose=False):
-        return {
-            "answer": str(response.get("generated_tokens")),
-            "generated_tokens": str(response.get("generated_tokens")),
-        }
+        if response.get("generated_tokens") != response.get("answer"):
+            return {
+                "answer": str(response.get("generated_tokens")),
+                "generated_tokens": str(response.get("generated_tokens")),
+            }
+        else:
+            return {
+                "answer": str(response.get("generated_tokens")),
+                "generated_tokens": str(response.get("generated_tokens")),
+            }
 
 
 class QuestionFreeText(QuestionBase):

edsl/questions/descriptors.py

@@ -2,6 +2,7 @@
 
 from abc import ABC, abstractmethod
 import re
+import textwrap
 from typing import Any, Callable, List, Optional
 from edsl.exceptions.questions import (
     QuestionCreationValidationError,
@@ -404,6 +405,9 @@ class QuestionTextDescriptor(BaseDescriptor):
             raise Exception("Question is too short!")
         if not isinstance(value, str):
             raise Exception("Question must be a string!")
+        
+        #value = textwrap.dedent(value).strip()
+
         if contains_single_braced_substring(value):
             import warnings
 

edsl/questions/question_base_gen_mixin.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 import copy
 import itertools
-from typing import Optional, List, Callable, Type, TYPE_CHECKING
+from typing import Optional, List, Callable, Type, TYPE_CHECKING, Union
 
 if TYPE_CHECKING:
     from edsl.questions.QuestionBase import QuestionBase
@@ -9,7 +9,11 @@ if TYPE_CHECKING:
 
 
 class QuestionBaseGenMixin:
-    """Mixin for QuestionBase."""
+    """Mixin for QuestionBase.
+    
+    This mostly has functions that are used to generate new questions from existing ones.
+    
+    """
 
     def copy(self) -> QuestionBase:
         """Return a deep copy of the question.
@@ -85,48 +89,110 @@ class QuestionBaseGenMixin:
         lp = LoopProcessor(self)
         return lp.process_templates(scenario_list)
 
-    def render(self, replacement_dict: dict) -> "QuestionBase":
-        """Render the question components as jinja2 templates with the replacement dictionary.
-
-        :param replacement_dict: The dictionary of values to replace in the question components.
+    class MaxTemplateNestingExceeded(Exception):
+        """Raised when template rendering exceeds maximum allowed nesting level."""
+        pass
 
+    def render(self, replacement_dict: dict, return_dict: bool = False) -> Union["QuestionBase", dict]:
+        """Render the question components as jinja2 templates with the replacement dictionary.
+        Handles nested template variables by recursively rendering until all variables are resolved.
+        
+        Raises:
+            MaxTemplateNestingExceeded: If template nesting exceeds MAX_NESTING levels
+        
         >>> from edsl.questions.QuestionFreeText import QuestionFreeText
         >>> q = QuestionFreeText(question_name = "color", question_text = "What is your favorite {{ thing }}?")
         >>> q.render({"thing": "color"})
         Question('free_text', question_name = \"""color\""", question_text = \"""What is your favorite color?\""")
 
+        >>> from edsl.questions.QuestionMultipleChoice import QuestionMultipleChoice
+        >>> q = QuestionMultipleChoice(question_name = "color", question_text = "What is your favorite {{ thing }}?", question_options = ["red", "blue", "green"])
+        >>> from edsl.scenarios.Scenario import Scenario
+        >>> q.render(Scenario({"thing": "color"})).data
+        {'question_name': 'color', 'question_text': 'What is your favorite color?', 'question_options': ['red', 'blue', 'green']}
+
+        >>> from edsl.questions.QuestionMultipleChoice import QuestionMultipleChoice
+        >>> q = QuestionMultipleChoice(question_name = "color", question_text = "What is your favorite {{ thing }}?", question_options = ["red", "blue", "green"])
+        >>> q.render({"thing": 1}).data
+        {'question_name': 'color', 'question_text': 'What is your favorite 1?', 'question_options': ['red', 'blue', 'green']}
+
+
+        >>> from edsl.questions.QuestionMultipleChoice import QuestionMultipleChoice
+        >>> from edsl.scenarios.Scenario import Scenario
+        >>> q = QuestionMultipleChoice(question_name = "color", question_text = "What is your favorite {{ thing }}?", question_options = ["red", "blue", "green"])
+        >>> q.render(Scenario({"thing": "color of {{ object }}", "object":"water"})).data
+        {'question_name': 'color', 'question_text': 'What is your favorite color of water?', 'question_options': ['red', 'blue', 'green']}
+
+        
+        >>> from edsl.questions.QuestionFreeText import QuestionFreeText
+        >>> q = QuestionFreeText(question_name = "infinite", question_text = "This has {{ a }}")
+        >>> q.render({"a": "{{ b }}", "b": "{{ a }}"}) # doctest: +IGNORE_EXCEPTION_DETAIL
+        Traceback (most recent call last):
+            ...
+        edsl.questions.question_base_gen_mixin.QuestionBaseGenMixin.MaxTemplateNestingExceeded:...
         """
-        from jinja2 import Environment
+        from jinja2 import Environment, meta
         from edsl.scenarios.Scenario import Scenario
 
+        MAX_NESTING = 10  # Maximum allowed nesting levels
+        
         strings_only_replacement_dict = {
             k: v for k, v in replacement_dict.items() if not isinstance(v, Scenario)
         }
 
+        def _has_unrendered_variables(template_str: str, env: Environment) -> bool:
+            """Check if the template string has any unrendered variables."""
+            if not isinstance(template_str, str):
+                return False
+            ast = env.parse(template_str)
+            return bool(meta.find_undeclared_variables(ast))
+
         def render_string(value: str) -> str:
             if value is None or not isinstance(value, str):
                 return value
-            else:
-                try:
-                    return (
-                        Environment()
-                        .from_string(value)
-                        .render(strings_only_replacement_dict)
-                    )
-                except Exception as e:
-                    #breakpoint()
-                    import warnings
-
-                    warnings.warn("Failed to render string: " + value)
-                    # breakpoint()
-                    return value
-
-        return self.apply_function(render_string)
-
+            
+            try:
+                env = Environment()
+                result = value
+                nesting_count = 0
+                
+                while _has_unrendered_variables(result, env):
+                    if nesting_count >= MAX_NESTING:
+                        raise self.MaxTemplateNestingExceeded(
+                            f"Template rendering exceeded {MAX_NESTING} levels of nesting. "
+                            f"Current value: {result}"
+                        )
+                    
+                    template = env.from_string(result)
+                    new_result = template.render(strings_only_replacement_dict)
+                    if new_result == result:  # Break if no changes made
+                        break
+                    result = new_result
+                    nesting_count += 1
+                
+                return result
+            except self.MaxTemplateNestingExceeded:
+                raise
+            except Exception as e:
+                import warnings
+                warnings.warn("Failed to render string: " + value)
+                return value
+        if return_dict:
+            return self._apply_function_dict(render_string)
+        else:
+            return self.apply_function(render_string)
+      
     def apply_function(
-        self, func: Callable, exclude_components: List[str] = None
+        self, func: Callable, exclude_components: Optional[List[str]] = None
     ) -> QuestionBase:
-        """Apply a function to the question parts
+        from edsl.questions.QuestionBase import QuestionBase
+        d = self._apply_function_dict(func, exclude_components)
+        return QuestionBase.from_dict(d)
+
+    def _apply_function_dict(
+        self, func: Callable, exclude_components: Optional[List[str]] = None
+    ) -> dict:
+        """Apply a function to the question parts, excluding certain components.
 
         :param func: The function to apply to the question parts.
         :param exclude_components: The components to exclude from the function application.
@@ -141,7 +207,6 @@ class QuestionBaseGenMixin:
         Question('free_text', question_name = \"""COLOR\""", question_text = \"""WHAT IS YOUR FAVORITE COLOR?\""")
 
         """
-        from edsl.questions.QuestionBase import QuestionBase
 
         if exclude_components is None:
             exclude_components = ["question_name", "question_type"]
@@ -160,10 +225,10 @@ class QuestionBaseGenMixin:
                 d[key] = value
                 continue
             d[key] = func(value)
-        return QuestionBase.from_dict(d)
+        return d
 
 
 if __name__ == "__main__":
     import doctest
 
-    doctest.testmod()
+    doctest.testmod(optionflags=doctest.ELLIPSIS)