edsl 0.1.40.dev2__py3-none-any.whl → 0.1.42__py3-none-any.whl

edsl/questions/QuestionDict.py

@@ -0,0 +1,343 @@
+from __future__ import annotations
+from typing import Union, Optional, Dict, List, Any, Type
+from pydantic import BaseModel, Field, field_validator
+from jinja2 import Environment, FileSystemLoader, TemplateNotFound
+from pathlib import Path
+
+from edsl.questions.QuestionBase import QuestionBase
+from edsl.questions.descriptors import (
+    AnswerKeysDescriptor,
+    ValueTypesDescriptor,
+    ValueDescriptionsDescriptor,
+    QuestionTextDescriptor,
+)
+from edsl.questions.response_validator_abc import ResponseValidatorABC
+from edsl.exceptions.questions import QuestionCreationValidationError
+from edsl.questions.decorators import inject_exception
+
+
+class DictResponseValidator(ResponseValidatorABC):
+    required_params = ["answer_keys", "permissive"]
+
+    valid_examples = [
+        (
+            {
+                "answer": {
+                    "name": "Hot Chocolate",
+                    "num_ingredients": 5,
+                    "ingredients": ["milk", "cocoa", "sugar"]
+                }
+            },
+            {
+                "answer_keys": ["name", "num_ingredients", "ingredients"],
+                "value_types": ["str", "int", "list[str]"]
+            },
+        )
+    ]
+    invalid_examples = [
+        (
+            {"answer": {"name": 123}},  # Name should be a string
+            {"answer_keys": ["name"], "value_types": ["str"]},
+            "Key 'name' has value of type int, expected str",
+        ),
+        (
+            {"answer": {"ingredients": "milk"}},  # Should be a list
+            {"answer_keys": ["ingredients"], "value_types": ["list"]},
+            "Key 'ingredients' should be a list, got str",
+        )
+    ]
+
+
+class QuestionDict(QuestionBase):
+    question_type = "dict"
+    question_text: str = QuestionTextDescriptor()
+    answer_keys: List[str] = AnswerKeysDescriptor()
+    value_types: Optional[List[str]] = ValueTypesDescriptor()
+    value_descriptions: Optional[List[str]] = ValueDescriptionsDescriptor()
+
+    _response_model = None
+    response_validator_class = DictResponseValidator
+
+    def _get_default_answer(self) -> Dict[str, Any]:
+        """Get default answer based on types."""
+        answer = {}
+        if not self.value_types:
+            return {
+                "title": "Sample Recipe",
+                "ingredients": ["ingredient1", "ingredient2"],
+                "num_ingredients": 2,
+                "instructions": "Sample instructions"
+            }
+
+        for key, type_str in zip(self.answer_keys, self.value_types):
+            if type_str.startswith(('list[', 'list')):
+                if '[' in type_str:
+                    element_type = type_str[type_str.index('[') + 1:type_str.rindex(']')].lower()
+                    if element_type == 'str':
+                        answer[key] = ["sample_string"]
+                    elif element_type == 'int':
+                        answer[key] = [1]
+                    elif element_type == 'float':
+                        answer[key] = [1.0]
+                    else:
+                        answer[key] = []
+                else:
+                    answer[key] = []
+            else:
+                if type_str == 'str':
+                    answer[key] = "sample_string"
+                elif type_str == 'int':
+                    answer[key] = 1
+                elif type_str == 'float':
+                    answer[key] = 1.0
+                else:
+                    answer[key] = None
+                    
+        return answer
+
+    def create_response_model(
+        self,
+    ) -> Type[BaseModel]:
+        """Create a response model for dict questions."""
+        default_answer = self._get_default_answer()
+        
+        class DictResponse(BaseModel):
+            answer: Dict[str, Any] = Field(
+                default_factory=lambda: default_answer.copy()
+            )
+            comment: Optional[str] = None
+            
+            @field_validator("answer")
+            def validate_answer(cls, v, values, **kwargs):
+                # Ensure all keys exist
+                missing_keys = set(self.answer_keys) - set(v.keys())
+                if missing_keys:
+                    raise ValueError(f"Missing required keys: {missing_keys}")
+                    
+                # Validate value types if not permissive
+                if not self.permissive and self.value_types:
+                    for key, type_str in zip(self.answer_keys, self.value_types):
+                        if key not in v:
+                            continue
+                            
+                        value = v[key]
+                        type_str = type_str.lower()  # Normalize to lowercase
+                        
+                        # Handle list types
+                        if type_str.startswith(('list[', 'list')):
+                            if not isinstance(value, list):
+                                raise ValueError(f"Key '{key}' should be a list, got {type(value).__name__}")
+                                
+                            # If it's a parameterized list, check element types
+                            if '[' in type_str:
+                                element_type = type_str[type_str.index('[') + 1:type_str.rindex(']')]
+                                element_type = element_type.lower().strip()
+                                
+                                for i, elem in enumerate(value):
+                                    expected_type = {
+                                        'str': str,
+                                        'int': int,
+                                        'float': float,
+                                        'list': list
+                                    }.get(element_type)
+                                    
+                                    if expected_type and not isinstance(elem, expected_type):
+                                        raise ValueError(
+                                            f"List element at index {i} for key '{key}' "
+                                            f"has type {type(elem).__name__}, expected {element_type}"
+                                        )
+                        else:
+                            # Handle basic types
+                            expected_type = {
+                                'str': str,
+                                'int': int,
+                                'float': float,
+                                'list': list,
+                            }.get(type_str)
+                            
+                            if expected_type and not isinstance(value, expected_type):
+                                raise ValueError(
+                                    f"Key '{key}' has value of type {type(value).__name__}, expected {type_str}"
+                                )
+                return v
+
+            model_config = {
+                "json_schema_extra": {
+                    "examples": [{
+                        "answer": default_answer,
+                        "comment": None
+                    }]
+                }
+            }
+
+        DictResponse.__name__ = "DictResponse"
+        return DictResponse
+
+    def __init__(
+        self,
+        question_name: str,
+        question_text: str,
+        answer_keys: List[str],
+        value_types: Optional[List[Union[str, type]]] = None,
+        value_descriptions: Optional[List[str]] = None,
+        include_comment: bool = True,
+        question_presentation: Optional[str] = None,
+        answering_instructions: Optional[str] = None,
+        permissive: bool = False,
+    ):
+        self.question_name = question_name
+        self.question_text = question_text
+        self.answer_keys = answer_keys
+        self.value_types = self._normalize_value_types(value_types)
+        self.value_descriptions = value_descriptions
+        self.include_comment = include_comment
+        self.question_presentation = question_presentation or self._render_template(
+            "question_presentation.jinja"
+        )
+        self.answering_instructions = answering_instructions or self._render_template(
+            "answering_instructions.jinja"
+        )
+        self.permissive = permissive
+
+        # Validation
+        if self.value_types and len(self.value_types) != len(self.answer_keys):
+            raise QuestionCreationValidationError(
+                "Length of value_types must match length of answer_keys."
+            )
+        if self.value_descriptions and len(self.value_descriptions) != len(self.answer_keys):
+            raise QuestionCreationValidationError(
+                "Length of value_descriptions must match length of answer_keys."
+            )
+
+    @staticmethod
+    def _normalize_value_types(value_types: Optional[List[Union[str, type]]]) -> Optional[List[str]]:
+        """Convert all value_types to string representations, including type hints."""
+        if not value_types:
+            return None
+
+        def normalize_type(t) -> str:
+            # Handle string representations of List
+            t_str = str(t)
+            if t_str == 'List':
+                return 'list'
+                
+            # Handle string inputs
+            if isinstance(t, str):
+                t = t.lower()
+                # Handle list types
+                if t.startswith(('list[', 'list')):
+                    if '[' in t:
+                        # Normalize the inner type
+                        inner_type = t[t.index('[') + 1:t.rindex(']')].strip().lower()
+                        return f"list[{inner_type}]"
+                    return "list"
+                return t
+
+            # Handle List the same as list
+            if t_str == "<class 'List'>":
+                return "list"
+
+            # If it's list type
+            if t is list:
+                return "list"
+
+            # If it's a basic type
+            if hasattr(t, "__name__"):
+                return t.__name__.lower()
+            
+            # If it's a typing.List
+            if t_str.startswith(('list[', 'list')):
+                return t_str.replace('typing.', '').lower()
+
+            # Handle generic types
+            if hasattr(t, "__origin__"):
+                origin = t.__origin__.__name__.lower()
+                args = [
+                    arg.__name__.lower() if hasattr(arg, "__name__") else str(arg).lower()
+                    for arg in t.__args__
+                ]
+                return f"{origin}[{', '.join(args)}]"
+
+            raise QuestionCreationValidationError(
+                f"Invalid type in value_types: {t}. Must be a type or string."
+            )
+
+        normalized = []
+        for t in value_types:
+            try:
+                normalized.append(normalize_type(t))
+            except Exception as e:
+                raise QuestionCreationValidationError(f"Error normalizing type {t}: {str(e)}")
+        
+        return normalized
+
+    def _render_template(self, template_name: str) -> str:
+        """Render a template using Jinja."""
+        try:
+            template_dir = Path(__file__).parent / "templates" / "dict"
+            env = Environment(loader=FileSystemLoader(template_dir))
+            template = env.get_template(template_name)
+            return template.render(
+                question_name=self.question_name,
+                question_text=self.question_text,
+                answer_keys=self.answer_keys,
+                value_types=self.value_types,
+                value_descriptions=self.value_descriptions,
+                include_comment=self.include_comment,
+            )
+        except TemplateNotFound:
+            return f"Template {template_name} not found in {template_dir}."
+
+    def to_dict(self, add_edsl_version: bool = True) -> dict:
+        """Serialize to JSON-compatible dictionary."""
+        return {
+            "question_type": self.question_type,
+            "question_name": self.question_name,
+            "question_text": self.question_text,
+            "answer_keys": self.answer_keys,
+            "value_types": self.value_types or [],
+            "value_descriptions": self.value_descriptions or [],
+            "include_comment": self.include_comment,
+            "permissive": self.permissive,
+        }
+
+    @classmethod
+    def from_dict(cls, data: dict) -> 'QuestionDict':
+        """Recreate from a dictionary."""
+        return cls(
+            question_name=data["question_name"],
+            question_text=data["question_text"],
+            answer_keys=data["answer_keys"],
+            value_types=data.get("value_types"),
+            value_descriptions=data.get("value_descriptions"),
+            include_comment=data.get("include_comment", True),
+            permissive=data.get("permissive", False),
+        )
+
+    @classmethod
+    @inject_exception
+    def example(cls) -> 'QuestionDict':
+        """Return an example question."""
+        return cls(
+            question_name="example",
+            question_text="Please provide a simple recipe for hot chocolate.",
+            answer_keys=["title", "ingredients", "num_ingredients", "instructions"],
+            value_types=["str", "list[str]", "int", "str"],
+            value_descriptions=[
+                "The title of the recipe.",
+                "A list of ingredients.",
+                "The number of ingredients.",
+                "The instructions for making the recipe."
+            ],
+        )
+
+    def _simulate_answer(self) -> dict:
+            """Simulate an answer for the question."""
+            return {
+                "answer": self._get_default_answer(),
+                "comment": None
+            }
+
+if __name__ == "__main__":
+    q = QuestionDict.example()
+    print(q.to_dict())

edsl/questions/QuestionExtract.py

@@ -50,7 +50,7 @@ def extract_json(text, expected_keys, verbose=False):
 
 def dict_to_pydantic_model(input_dict: Dict[str, Any]) -> Any:
     field_definitions = {
-        key: (str, Field(default=str(value))) for key, value in input_dict.items()
+        key: (type(value), Field(default=value)) for key, value in input_dict.items()
     }
 
     DynamicModel = create_model("DynamicModel", **field_definitions)

edsl/questions/__init__.py

@@ -12,6 +12,7 @@ from edsl.questions.QuestionFreeText import QuestionFreeText
 from edsl.questions.QuestionFunctional import QuestionFunctional
 from edsl.questions.QuestionList import QuestionList
 from edsl.questions.QuestionMatrix import QuestionMatrix
+from edsl.questions.QuestionDict import QuestionDict
 from edsl.questions.QuestionMultipleChoice import QuestionMultipleChoice
 from edsl.questions.QuestionNumerical import QuestionNumerical
 from edsl.questions.QuestionBudget import QuestionBudget

edsl/questions/answer_validator_mixin.py

@@ -324,6 +324,35 @@ class AnswerValidatorMixin:
                     f"Must be one of: {valid_options}"
                 )
 
+    def _validate_answer_dict(self, answer: dict[str, Any]) -> None:
+        """Validate QuestionDict-specific answer.
+
+        Check that answer["answer"]:
+        - is a dictionary
+        - has all required answer_keys as keys
+        """
+        value = answer.get("answer")
+
+        # Check that answer is a dictionary
+        if not isinstance(value, dict):
+            raise QuestionAnswerValidationError(
+                f"Dict answer must be a dictionary mapping values to specified keys (got {value})"
+            )
+
+        # Check that all required answer keys are present
+        required_keys = set(self.answer_keys)
+        provided_keys = set(value.keys())
+
+        if missing_keys := (required_keys - provided_keys):
+            raise QuestionAnswerValidationError(
+                f"Missing required keys: {missing_keys}"
+            )
+
+        if extra_keys := (provided_keys - required_keys):
+            raise QuestionAnswerValidationError(
+                f"Unexpected keys: {extra_keys}"
+            )
+
 
 if __name__ == "__main__":
     pass

edsl/questions/derived/QuestionLinearScale.py

@@ -36,7 +36,7 @@ class QuestionLinearScale(QuestionMultipleChoice):
             question_name=question_name,
             question_text=question_text,
             question_options=question_options,
-            use_code=False,  # question linear scale will have it's own code
+            use_code=False,  # question linear scale will have its own code
             include_comment=include_comment,
         )
         self.question_options = question_options

edsl/questions/descriptors.py

@@ -302,10 +302,10 @@ class QuestionOptionsDescriptor(BaseDescriptor):
             raise QuestionCreationValidationError(
                 f"Question options must be a list (got {value})."
             )
-        if len(value) > Settings.MAX_NUM_OPTIONS:
-            raise QuestionCreationValidationError(
-                f"Too many question options (got {value})."
-            )
+        # if len(value) > Settings.MAX_NUM_OPTIONS:
+        #     raise QuestionCreationValidationError(
+        #         f"Too many question options (got {value})."
+        #     )
         if len(value) < Settings.MIN_NUM_OPTIONS:
             raise QuestionCreationValidationError(
                 f"Too few question options (got {value})."
@@ -408,7 +408,7 @@ class QuestionTextDescriptor(BaseDescriptor):
             # Automatically replace single braces with double braces
             # This is here because if the user is using an f-string, the double brace will get converted to a single brace.
             # This undoes that.
-            value = re.sub(r"\{([^\{\}]+)\}", r"{{\1}}", value)
+            # value = re.sub(r"\{([^\{\}]+)\}", r"{{\1}}", value)
             return value
 
         # iterate through all doubles braces and check if they are valid python identifiers
@@ -421,6 +421,50 @@ class QuestionTextDescriptor(BaseDescriptor):
         return None
 
 
+class ValueTypesDescriptor(BaseDescriptor):
+    def validate(self, value, instance):
+        """Validate the value is a list of strings or None."""
+        if value is None:  # Allow None as a valid value
+            return None
+        if not isinstance(value, list):
+            raise QuestionCreationValidationError(
+                f"`value_types` must be a list or None (got {value})."
+            )
+        # Convert all items in the list to strings
+        return [str(item) for item in value]
+
+
+class ValueDescriptionsDescriptor(BaseDescriptor):
+    def validate(self, value, instance):
+        """Validate the value is a list of strings or None."""
+        if value is None:  # Allow None as a valid value
+            return None
+        if not isinstance(value, list):
+            raise QuestionCreationValidationError(
+                f"`value_descriptions` must be a list or None (got {value})."
+            )
+        if not all(isinstance(x, str) for x in value):
+            raise QuestionCreationValidationError(
+                f"`value_descriptions` must be a list of strings (got {value})."
+            )
+        return value
+
+
+class AnswerKeysDescriptor(BaseDescriptor):
+    """Validate that the `answer_keys` attribute is a list of strings or integers."""
+
+    def validate(self, value, instance):
+        """Validate the value is a list of strings or integers."""
+        if not isinstance(value, list):
+            raise QuestionCreationValidationError(
+                f"`answer_keys` must be a list (got {value})."
+            )
+        if not all(isinstance(x, (str, int)) for x in value):
+            raise QuestionCreationValidationError(
+                f"`answer_keys` must be a list of strings or integers (got {value})."
+            )
+
+
 if __name__ == "__main__":
     import doctest
 

edsl/questions/question_registry.py

@@ -96,7 +96,7 @@ class Question(metaclass=Meta):
 
         >>> from edsl import Question
         >>> Question.list_question_types()
-        ['checkbox', 'extract', 'free_text', 'functional', 'likert_five', 'linear_scale', 'list', 'matrix', 'multiple_choice', 'numerical', 'rank', 'top_k', 'yes_no']
+        ['checkbox', 'dict', 'extract', 'free_text', 'functional', 'likert_five', 'linear_scale', 'list', 'matrix', 'multiple_choice', 'numerical', 'rank', 'top_k', 'yes_no']
         """
         return [
             q

edsl/questions/templates/dict/answering_instructions.jinja

@@ -0,0 +1,21 @@
+Please respond with a dictionary using the following keys: {{ answer_keys | join(', ') }}.
+
+{% if value_descriptions %}
+Here are descriptions of the values to provide:
+{% for idx in range(answer_keys | length) %}
+- "{{ answer_keys[idx] }}": "{{ value_descriptions[idx] }}"
+{% endfor %}
+{% endif %}
+
+{% if value_types %}
+The values should be formatted in the following types:
+{% for idx in range(answer_keys | length) %}
+- "{{ answer_keys[idx] }}": "{{ value_types[idx] }}"
+{% endfor %}
+{% endif %}
+
+If you do not have a value for a given key, use "null".
+
+{% if include_comment %}
+After the answer, you can put a comment explaining your response on the next line.
+{% endif %}

edsl/questions/templates/dict/question_presentation.jinja

@@ -0,0 +1 @@
+{{question_text}}

edsl/results/Result.py

@@ -56,6 +56,7 @@ class Result(Base, UserDict):
         comments_dict: Optional[dict] = None,
         cache_used_dict: Optional[dict[QuestionName, bool]] = None,
         indices: Optional[dict] = None,
+        cache_keys: Optional[dict[QuestionName, str]] = None,
     ):
         """Initialize a Result object.
 
@@ -77,7 +78,6 @@ class Result(Base, UserDict):
         self.question_to_attributes = (
             question_to_attributes or self._create_question_to_attributes(survey)
         )
-
         data = {
             "agent": agent,
             "scenario": scenario,
@@ -86,10 +86,11 @@ class Result(Base, UserDict):
             "answer": answer,
             "prompt": prompt or {},
             "raw_model_response": raw_model_response or {},
-            "question_to_attributes": question_to_attributes,
+            "question_to_attributes": self.question_to_attributes,
             "generated_tokens": generated_tokens or {},
             "comments_dict": comments_dict or {},
             "cache_used_dict": cache_used_dict or {},
+            "cache_keys": cache_keys or {},
         }
         super().__init__(**data)
         self.indices = indices
@@ -152,7 +153,9 @@ class Result(Base, UserDict):
     @staticmethod
     def _create_model_sub_dict(model) -> dict:
         return {
-            "model": model.parameters | {"model": model.model},
+            "model": model.parameters
+            | {"model": model.model}
+            | {"inference_service": model._inference_service_},
         }
 
     @staticmethod
@@ -163,6 +166,7 @@ class Result(Base, UserDict):
 
     def _construct_sub_dicts(self) -> dict[str, dict]:
         """Construct a dictionary of sub-dictionaries for the Result object."""
+
         sub_dicts_needing_new_keys = {
             "question_text": {},
             "question_options": {},
@@ -181,6 +185,8 @@ class Result(Base, UserDict):
             f"{k}_cache_used": v for k, v in self.data["cache_used_dict"].items()
         }
 
+        cache_keys = {f"{k}_cache_key": v for k, v in self.data["cache_keys"].items()}
+
         d = {
             **self._create_agent_sub_dict(self.data["agent"]),
             **self._create_model_sub_dict(self.data["model"]),
@@ -195,11 +201,13 @@ class Result(Base, UserDict):
             "question_options": sub_dicts_needing_new_keys["question_options"],
             "question_type": sub_dicts_needing_new_keys["question_type"],
             "cache_used": new_cache_dict,
+            "cache_keys": cache_keys,
         }
         if hasattr(self, "indices") and self.indices is not None:
             d["agent"].update({"agent_index": self.indices["agent"]})
             d["scenario"].update({"scenario_index": self.indices["scenario"]})
             d["model"].update({"model_index": self.indices["model"]})
+
         return d
 
     @property
@@ -358,6 +366,10 @@ class Result(Base, UserDict):
                         else prompt_obj.to_dict()
                     )
                 d[key] = new_prompt_dict
+            
+        if self.indices is not None:
+            d["indices"] = self.indices
+
         if add_edsl_version:
             from edsl import __version__
 
@@ -406,6 +418,8 @@ class Result(Base, UserDict):
             generated_tokens=json_dict.get("generated_tokens", {}),
             comments_dict=json_dict.get("comments_dict", {}),
             cache_used_dict=json_dict.get("cache_used_dict", {}),
+            cache_keys=json_dict.get("cache_keys", {}),
+            indices = json_dict.get("indices", None)
         )
         return result
 
@@ -459,6 +473,12 @@ class Result(Base, UserDict):
                 question_results[result.question_name] = result
             return question_results
 
+        def get_cache_keys(model_response_objects) -> dict[str, bool]:
+            cache_keys = {}
+            for result in model_response_objects:
+                cache_keys[result.question_name] = result.cache_key
+            return cache_keys
+
         def get_generated_tokens_dict(answer_key_names) -> dict[str, str]:
             generated_tokens_dict = {
                 k + "_generated_tokens": question_results[k].generated_tokens
@@ -523,6 +543,7 @@ class Result(Base, UserDict):
         generated_tokens_dict = get_generated_tokens_dict(answer_key_names)
         comments_dict = get_comments_dict(answer_key_names)
         answer_dict = {k: extracted_answers[k] for k in answer_key_names}
+        cache_keys = get_cache_keys(model_response_objects)
 
         question_name_to_prompts = get_question_name_to_prompts(model_response_objects)
         prompt_dictionary = get_prompt_dictionary(
@@ -546,6 +567,7 @@ class Result(Base, UserDict):
             comments_dict=comments_dict,
             cache_used_dict=cache_used_dictionary,
             indices=interview.indices,
+            cache_keys=cache_keys,
         )
         result.interview_hash = interview.initial_hash
         return result