edsl 0.1.57__py3-none-any.whl → 0.1.59__py3-none-any.whl

edsl/questions/question_list.py

@@ -299,23 +299,24 @@ class ListResponseValidator(ResponseValidatorABC):
         # This method can now be removed since validation is handled in the Pydantic model
         pass
 
-    def fix(self, response, verbose=False):
+    def fix(self, response, verbose=False) -> dict[str, Any]:
         """
         Fix common issues in list responses by splitting strings into lists.
         
         Examples:
             >>> from edsl import QuestionList
-            >>> q = QuestionList.example(min_list_items=2, max_list_items=4)
-            >>> validator = q.response_validator
+            >>> q_constrained = QuestionList.example(min_list_items=2, max_list_items=4)
+            >>> validator_constrained = q_constrained.response_validator
             
+            >>> q_permissive = QuestionList.example(permissive=True)
+            >>> validator_permissive = q_permissive.response_validator
+
             >>> # Fix a string that should be a list
             >>> bad_response = {"answer": "apple,banana,cherry"}
-            >>> try:
-            ...     validator.validate(bad_response)
-            ... except Exception:
-            ...     fixed = validator.fix(bad_response)
-            ...     validated = validator.validate(fixed)
-            ...     validated  # Show full response
+            >>> fixed = validator_constrained.fix(bad_response)
+            >>> fixed
+            {'answer': ['apple', 'banana', 'cherry']}
+            >>> validator_constrained.validate(fixed)  # Show full response after validation
             {'answer': ['apple', 'banana', 'cherry'], 'comment': None, 'generated_tokens': None}
 
             >>> # Fix using generated_tokens when answer is invalid
@@ -323,12 +324,10 @@ class ListResponseValidator(ResponseValidatorABC):
             ...     "answer": None,
             ...     "generated_tokens": "pizza, pasta, salad"
             ... }
-            >>> try:
-            ...     validator.validate(bad_response)
-            ... except Exception:
-            ...     fixed = validator.fix(bad_response)
-            ...     validated = validator.validate(fixed)
-            ...     validated
+            >>> fixed = validator_constrained.fix(bad_response)
+            >>> fixed
+            {'answer': ['pizza', ' pasta', ' salad']}
+            >>> validator_constrained.validate(fixed)
             {'answer': ['pizza', ' pasta', ' salad'], 'comment': None, 'generated_tokens': None}
 
             >>> # Preserve comments during fixing
@@ -336,17 +335,74 @@ class ListResponseValidator(ResponseValidatorABC):
             ...     "answer": "red,blue,green",
             ...     "comment": "These are colors"
             ... }
-            >>> fixed = validator.fix(bad_response)
-            >>> fixed == {
+            >>> fixed_output = validator_constrained.fix(bad_response)
+            >>> fixed_output
+            {'answer': ['red', 'blue', 'green'], 'comment': 'These are colors'}
+            >>> validated_output = validator_constrained.validate(fixed_output)
+            >>> validated_output == {
             ...     "answer": ["red", "blue", "green"],
-            ...     "comment": "These are colors"
+            ...     "comment": "These are colors",
+            ...     "generated_tokens": None
             ... }
             True
+
+            >>> # Fix an empty string answer
+            >>> bad_response = {"answer": ""}
+            >>> fixed = validator_constrained.fix(bad_response)
+            >>> fixed
+            {'answer': []}
+            >>> validator_permissive.validate(fixed)
+            {'answer': [], 'comment': None, 'generated_tokens': None}
+
+            >>> # Fix a single item string answer (no commas)
+            >>> bad_response = {"answer": "single_item"}
+            >>> fixed = validator_constrained.fix(bad_response)
+            >>> fixed
+            {'answer': ['single_item']}
+            >>> validator_permissive.validate(fixed)
+            {'answer': ['single_item'], 'comment': None, 'generated_tokens': None}
+
+            >>> # Fix when answer is None and no generated_tokens
+            >>> bad_response = {"answer": None}
+            >>> fixed = validator_constrained.fix(bad_response)
+            >>> fixed
+            {'answer': []}
+            >>> validator_permissive.validate(fixed)
+            {'answer': [], 'comment': None, 'generated_tokens': None}
+
+            >>> # Fix when answer key is missing but generated_tokens is present
+            >>> bad_response = {"generated_tokens": "token1,token2"}
+            >>> fixed = validator_constrained.fix(bad_response)
+            >>> fixed
+            {'answer': ['token1', 'token2']}
+            >>> validator_constrained.validate(fixed) # 2 items, OK for constrained validator
+            {'answer': ['token1', 'token2'], 'comment': None, 'generated_tokens': None}
+
+            >>> # Fix when answer key is missing and generated_tokens is an empty string
+            >>> bad_response = {"generated_tokens": ""}
+            >>> fixed = validator_constrained.fix(bad_response)
+            >>> fixed
+            {'answer': []}
+            >>> validator_permissive.validate(fixed)
+            {'answer': [], 'comment': None, 'generated_tokens': None}
+            
+            >>> # Fix when answer key is missing and generated_tokens is a single item
+            >>> bad_response = {"generated_tokens": "single_token"}
+            >>> fixed = validator_constrained.fix(bad_response)
+            >>> fixed
+            {'answer': ['single_token']}
+            >>> validator_permissive.validate(fixed)
+            {'answer': ['single_token'], 'comment': None, 'generated_tokens': None}
         """
         if verbose:
             print(f"Fixing list response: {response}")
         answer = str(response.get("answer") or response.get("generated_tokens", ""))
-        result = {"answer": answer.split(",")}
+        if "," in answer:
+            result = {"answer": answer.split(",")}
+        elif answer == "":
+            result = {"answer": []}
+        else:
+            result = {"answer": [answer]}
         if "comment" in response:
             result["comment"] = response["comment"]
         return result
@@ -395,7 +451,7 @@ class QuestionList(QuestionBase):
 
         self.include_comment = include_comment
         self.answering_instructions = answering_instructions
-        self.question_presentations = question_presentation
+        self.question_presentation = question_presentation
 
     def create_response_model(self):
         return create_model(self.min_list_items, self.max_list_items, self.permissive)

edsl/results/results.py

@@ -771,6 +771,10 @@ class Results(MutableSequence, ResultsOperationsMixin, Base):
     def to_dataset(self) -> "Dataset":
         return self.select()
 
+    def optimzie_scenarios(self):
+        for result in self.data:
+            result.scenario.offload(inplace=True)
+
     def to_dict(
         self,
         sort: bool = False,
@@ -778,9 +782,12 @@ class Results(MutableSequence, ResultsOperationsMixin, Base):
         include_cache: bool = True,
         include_task_history: bool = False,
         include_cache_info: bool = True,
+        offload_scenarios: bool = True,
     ) -> dict[str, Any]:
         from ..caching import Cache
 
+        if offload_scenarios:
+            self.optimzie_scenarios()
         if sort:
             data = sorted([result for result in self.data], key=lambda x: hash(x))
         else:
@@ -809,7 +816,7 @@ class Results(MutableSequence, ResultsOperationsMixin, Base):
             )
 
         if self.task_history.has_unfixed_exceptions or include_task_history:
-            d.update({"task_history": self.task_history.to_dict()})
+            d.update({"task_history": self.task_history.to_dict(offload_content=True)})
 
         if add_edsl_version:
             from .. import __version__

edsl/scenarios/file_store.py

@@ -446,9 +446,7 @@ class FileStore(Scenario):
         if suffix is None:
             suffix = self.suffix
         if self.binary:
-            file_like_object = self.base64_to_file(
-                self["base64_string"], is_binary=True
-            )
+            file_like_object = self.base64_to_file(self.base64_string, is_binary=True)
         else:
             file_like_object = self.base64_to_text_file(self.base64_string)
 
@@ -765,15 +763,13 @@ class FileStore(Scenario):
         if name.startswith("__") and name.endswith("__"):
             raise AttributeError(name)
 
-        # Only try to access suffix if it's in our __dict__
-        if hasattr(self, "_data") and "suffix" in self._data:
-            if self._data["suffix"] == "csv":
-                # Get the pandas DataFrame
-                df = self.to_pandas()
-                # Check if the requested attribute exists in the DataFrame
-                if hasattr(df, name):
-                    return getattr(df, name)
-        # If not a CSV or attribute doesn't exist in DataFrame, raise AttributeError
+        # Check for _data directly in __dict__ to avoid recursion
+        _data = self.__dict__.get("_data", None)
+        if _data and _data.get("suffix") == "csv":
+            df = self.to_pandas()
+            if hasattr(df, name):
+                return getattr(df, name)
+
         raise AttributeError(
             f"'{self.__class__.__name__}' object has no attribute '{name}'"
         )

edsl/scenarios/scenario.py

@@ -264,9 +264,49 @@ class Scenario(Base, UserDict):
         """Display a scenario as a table."""
         return self.to_dataset().table(tablefmt=tablefmt)
 
-    def to_dict(self, add_edsl_version: bool = True) -> dict:
+    def offload(self, inplace=False) -> "Scenario":
+        """
+        Offloads base64-encoded content from the scenario by replacing 'base64_string'
+        fields with 'offloaded'. This reduces memory usage.
+
+        Args:
+            inplace (bool): If True, modify the current scenario. If False, return a new one.
+
+        Returns:
+            Scenario: The modified scenario (either self or a new instance).
+        """
+        from edsl.scenarios import FileStore
+        from edsl.prompts import Prompt
+
+        target = self if inplace else Scenario()
+
+        for key, value in self.items():
+            if isinstance(value, FileStore):
+                file_store_dict = value.to_dict()
+                if "base64_string" in file_store_dict:
+                    file_store_dict["base64_string"] = "offloaded"
+                modified_value = FileStore.from_dict(file_store_dict)
+            elif isinstance(value, dict) and "base64_string" in value:
+                value_copy = value.copy()
+                value_copy["base64_string"] = "offloaded"
+                modified_value = value_copy
+            else:
+                modified_value = value
+
+            target[key] = modified_value
+
+        return target
+
+    def to_dict(
+        self, add_edsl_version: bool = True, offload_base64: bool = False
+    ) -> dict:
         """Convert a scenario to a dictionary.
 
+        Args:
+            add_edsl_version: If True, adds the EDSL version to the returned dictionary.
+            offload_base64: If True, replaces any base64_string fields with 'offloaded'
+                           to reduce memory usage.
+
         Example:
 
         >>> s = Scenario({"food": "wood chips"})
@@ -283,7 +323,15 @@ class Scenario(Base, UserDict):
         d = self.data.copy()
         for key, value in d.items():
             if isinstance(value, FileStore) or isinstance(value, Prompt):
-                d[key] = value.to_dict(add_edsl_version=add_edsl_version)
+                value_dict = value.to_dict(add_edsl_version=add_edsl_version)
+                if (
+                    offload_base64
+                    and isinstance(value_dict, dict)
+                    and "base64_string" in value_dict
+                ):
+                    value_dict["base64_string"] = "offloaded"
+                d[key] = value_dict
+
         if add_edsl_version:
             from edsl import __version__
 

edsl/scenarios/scenario_list.py

@@ -145,22 +145,18 @@ class ScenarioList(MutableSequence, Base, ScenarioListOperationsMixin):
         """Initialize a new ScenarioList with optional data and codebook."""
         self._data_class = data_class
         self.data = self._data_class([])
-        warned = False
         for item in data or []:
-            try: 
-                _ = json.dumps(item.to_dict())
-            except:
-                import warnings 
-                if not warned:
-                    warnings.warn( 
-                        f"One or more items in the data list are not JSON serializable. "
-                        "This would prevent running a job that uses this ScenarioList."
-                        "One solution is to use 'str(item)' to convert the item to a string before adding."
-                    )
-                    warned = True
             self.data.append(item)
         self.codebook = codebook or {}
 
+    def is_serializable(self):
+        for item in self.data:
+            try:
+                _ = json.dumps(item.to_dict())
+            except Exception as e:
+                return False
+        return True
+
     # Required MutableSequence abstract methods
     def __getitem__(self, index):
         """Get item at index."""
@@ -360,6 +356,32 @@ class ScenarioList(MutableSequence, Base, ScenarioListOperationsMixin):
                 new_scenarios.append(Scenario(new_scenario))
 
         return new_scenarios
+    
+    @classmethod
+    def from_search_terms(cls, search_terms: List[str]) -> ScenarioList:
+        """Create a ScenarioList from a list of search terms, using Wikipedia.
+
+        Args:
+            search_terms: A list of search terms.
+        """
+        from ..utilities.wikipedia import fetch_wikipedia_content
+        results = fetch_wikipedia_content(search_terms)
+        return cls([Scenario(result) for result in results])
+    
+    def augment_with_wikipedia(self, search_key:str, content_only: bool = True, key_name: str = "wikipedia_content") -> ScenarioList:
+        """Augment the ScenarioList with Wikipedia content."""
+        search_terms = self.select(search_key).to_list()
+        wikipedia_results = ScenarioList.from_search_terms(search_terms)
+        new_sl = ScenarioList(data = [], codebook = self.codebook)
+        for scenario, wikipedia_result in zip(self, wikipedia_results):
+            if content_only:
+                scenario[key_name] = wikipedia_result["content"]
+                new_sl.append(scenario)
+            else:
+                scenario[key_name] = wikipedia_result
+                new_sl.append(scenario)
+        return new_sl
+
 
     def pivot(
         self,

edsl/surveys/survey.py

@@ -384,6 +384,10 @@ class Survey(Base):
         if question_name not in self.question_name_to_index:
             raise SurveyError(f"Question name {question_name} not found in survey.")
         return self.questions[self.question_name_to_index[question_name]]
+    
+    def get(self, question_name: str) -> QuestionBase:
+        """Return the question object given the question name."""
+        return self._get_question_by_name(question_name)
 
     def question_names_to_questions(self) -> dict:
         """Return a dictionary mapping question names to question attributes."""

edsl/tasks/task_history.py

@@ -43,6 +43,7 @@ class TaskHistory(RepresentationMixin):
     - Generates interactive HTML reports with filtering and drill-down
     - Computes statistics across interviews (by model, question type, etc.)
     - Exports to various formats (HTML, notebook, etc.)
+    - Memory optimization via offloading of large file content
     """
 
     def __init__(
@@ -191,8 +192,22 @@ class TaskHistory(RepresentationMixin):
         """Return a string representation of the TaskHistory."""
         return f"TaskHistory(interviews={self.total_interviews})."
 
-    def to_dict(self, add_edsl_version=True):
-        """Return the TaskHistory as a dictionary."""
+    def to_dict(self, add_edsl_version=True, offload_content=False):
+        """
+        Return the TaskHistory as a dictionary.
+
+        Parameters:
+            add_edsl_version: Whether to include EDSL version in the output
+            offload_content: Whether to offload large file content like videos and images
+                            to reduce memory usage
+
+        Returns:
+            A dictionary representation of this TaskHistory instance
+        """
+        # Offload large file content if requested
+        if offload_content:
+            self.offload_files_content()
+
         # Serialize each interview object
         interview_dicts = []
         for i in self.total_interviews:
@@ -255,27 +270,60 @@ class TaskHistory(RepresentationMixin):
                             InterviewExceptionCollection,
                         )
 
+                        # Store the original data in full
+                        self._original_data = data
+
+                        # Preserve the original interview id
+                        self._interview_id = data.get("id", None)
+
+                        # Store exceptions using the original data structure
+                        # This ensures when we re-serialize, we keep original data intact
+                        self._exceptions_data = data.get("exceptions", {})
+
+                        # Create the InterviewExceptionCollection for runtime use
                         exceptions_data = data.get("exceptions", {})
                         self.exceptions = (
                             InterviewExceptionCollection.from_dict(exceptions_data)
                             if exceptions_data
                             else InterviewExceptionCollection()
                         )
+
+                        # Store other fields
                         self.task_status_logs = data.get("task_status_logs", {})
                         self.model = data.get("model", {})
                         self.survey = data.get("survey", {})
 
                     def to_dict(self, add_edsl_version=True):
-                        return {
+                        # Use the original exceptions data structure when serializing again
+                        # This preserves all exception details exactly as they were
+                        data = {
                             "type": "InterviewReference",
-                            "exceptions": self.exceptions.to_dict()
-                            if hasattr(self.exceptions, "to_dict")
-                            else self.exceptions,
+                            "exceptions": self._exceptions_data
+                            if hasattr(self, "_exceptions_data")
+                            else (
+                                self.exceptions.to_dict()
+                                if hasattr(self.exceptions, "to_dict")
+                                else self.exceptions
+                            ),
                             "task_status_logs": self.task_status_logs,
                             "model": self.model,
                             "survey": self.survey,
                         }
 
+                        # Preserve the original interview id if it exists
+                        if self._interview_id:
+                            data["id"] = self._interview_id
+
+                        # Preserve original version info
+                        if (
+                            add_edsl_version
+                            and hasattr(self, "_original_data")
+                            and "edsl_version" in self._original_data
+                        ):
+                            data["edsl_version"] = self._original_data["edsl_version"]
+
+                        return data
+
                 # Create the reference and add it directly
                 ref = DeserializedInterviewRef(interview_data)
                 instance.total_interviews.append(ref)
@@ -728,6 +776,132 @@ class TaskHistory(RepresentationMixin):
 
         return nb
 
+    def offload_files_content(self):
+        """
+        Offload large file content from scenarios in interview exceptions.
+
+        This method iterates over all the interview exceptions and calls the offload method
+        for any scenario components in the invigilator. This significantly reduces memory usage
+        by replacing base64-encoded content with a placeholder string, while preserving the
+        structure of the scenarios.
+
+        Returns:
+            self: Returns the TaskHistory instance for method chaining
+
+        This is particularly useful for TaskHistory instances containing interviews with
+        large file content, such as videos, images, or other binary data.
+        """
+        for interview in self.total_interviews:
+            if not hasattr(interview, "exceptions") or not interview.exceptions:
+                continue
+
+            for question_name, exceptions in interview.exceptions.items():
+                for exception in exceptions:
+                    # Check if exception has an invigilator with scenario
+                    if hasattr(exception, "invigilator") and exception.invigilator:
+                        if (
+                            hasattr(exception.invigilator, "scenario")
+                            and exception.invigilator.scenario
+                        ):
+                            # Call the offload method on the scenario
+                            if hasattr(exception.invigilator.scenario, "offload"):
+                                try:
+                                    # Replace the original scenario with the offloaded version
+                                    exception.invigilator.scenario = (
+                                        exception.invigilator.scenario.offload()
+                                    )
+                                except Exception as e:
+                                    # Silently continue if offloading fails for any reason
+                                    pass
+
+        return self
+
+    def deduplicate_and_clean_interviews(self):
+        """
+        Deduplicates exception entries in this task history to reduce memory usage.
+
+        This method removes duplicate error messages across interviews while preserving
+        the first occurrence of each unique error. This significantly reduces the size
+        of serialized task history data, especially for jobs with many similar errors.
+
+        Returns:
+            self: Returns the TaskHistory instance for method chaining.
+        """
+        seen = set()
+        cleaned_interviews = []
+
+        for interview in self.total_interviews:
+            # Skip if interview has no exceptions
+            if not hasattr(interview, "exceptions") or not interview.exceptions:
+                continue
+
+            keep_interview = False
+            questions_to_modify = {}
+            questions_to_remove = []
+
+            # First pass: Collect all modifications without changing the dictionary
+            if hasattr(interview.exceptions, "items"):
+                for question_name, exceptions in list(interview.exceptions.items()):
+                    filtered_exceptions = []
+
+                    for exception in exceptions:
+                        # Get the exception message (may require different access based on structure)
+                        if hasattr(exception, "exception") and hasattr(
+                            exception.exception, "args"
+                        ):
+                            message = (
+                                str(exception.exception.args[0])
+                                if exception.exception.args
+                                else ""
+                            )
+                        else:
+                            message = str(exception)
+
+                        # Create a unique key for this exception
+                        key = (question_name, message)
+
+                        # Only keep exceptions we haven't seen before
+                        if key not in seen:
+                            seen.add(key)
+                            filtered_exceptions.append(exception)
+
+                    # Track what should happen to this question's exceptions
+                    if filtered_exceptions:
+                        keep_interview = True
+                        questions_to_modify[question_name] = filtered_exceptions
+                    else:
+                        questions_to_remove.append(question_name)
+
+            # Second pass: Apply all modifications safely
+            if hasattr(interview.exceptions, "items"):
+                # Add/replace filtered exceptions
+                for question_name, filtered_exceptions in questions_to_modify.items():
+                    interview.exceptions[question_name] = filtered_exceptions
+
+                # Remove questions with all duplicate exceptions
+                for question_name in questions_to_remove:
+                    if hasattr(interview.exceptions, "pop"):
+                        interview.exceptions.pop(question_name, None)
+                    elif (
+                        hasattr(interview.exceptions, "__delitem__")
+                        and question_name in interview.exceptions
+                    ):
+                        del interview.exceptions[question_name]
+
+            # Only keep the interview if it still has exceptions after filtering
+            if keep_interview:
+                cleaned_interviews.append(interview)
+
+        # Replace the total_interviews with our cleaned list
+        self.total_interviews = cleaned_interviews
+
+        # Rebuild the _interviews dictionary
+        self._interviews = {
+            index: interview for index, interview in enumerate(self.total_interviews)
+        }
+
+        return self
+
 
 if __name__ == "__main__":
     import doctest