edsl 0.1.54__py3-none-any.whl → 0.1.56__py3-none-any.whl

edsl/results/results.py

@@ -1,7 +1,7 @@
 """The Results module provides tools for working with collections of Result objects.
 
-The Results class is the primary container for analyzing and manipulating data obtained 
-from running surveys with language models. It implements a powerful data analysis interface 
+The Results class is the primary container for analyzing and manipulating data obtained
+from running surveys with language models. It implements a powerful data analysis interface
 with methods for filtering, selecting, mutating, and visualizing your results, similar to
 data manipulation libraries like dplyr or pandas.
 
@@ -11,7 +11,7 @@ Key components:
 2. Report - A flexible reporting system for generating formatted output from Results
 3. Selectors - Tools for efficiently extracting specific data from Results
 
-The Results class is not typically instantiated directly; instead, it's returned by the 
+The Results class is not typically instantiated directly; instead, it's returned by the
 run() method of a Job object. Once you have a Results object, you can use its methods
 to analyze and extract insights from your survey data.
 
@@ -39,9 +39,10 @@ from __future__ import annotations
 import json
 import random
 import warnings
-from collections import UserList, defaultdict
+from collections import defaultdict
 from typing import Optional, Callable, Any, Union, List, TYPE_CHECKING
 from bisect import bisect_left
+from collections.abc import MutableSequence
 
 from ..base import Base
 from ..caching import Cache, CacheEntry
@@ -59,6 +60,9 @@ if TYPE_CHECKING:
 from ..utilities import remove_edsl_version, dict_hash
 from ..dataset import ResultsOperationsMixin
 
+from .result import Result
+from ..db_list.sqlite_list import SQLiteList
+
 from .exceptions import (
     ResultsError,
     ResultsBadMutationstringError,
@@ -70,6 +74,18 @@ from .exceptions import (
 )
 
 
+class ResultsSQLList(SQLiteList):
+    def serialize(self, obj):
+        return json.dumps(obj.to_dict()) if hasattr(obj, "to_dict") else json.dumps(obj)
+
+    def deserialize(self, data):
+        return (
+            Result.from_dict(json.loads(data))
+            if hasattr(Result, "from_dict")
+            else json.loads(data)
+        )
+
+
 def ensure_fetched(method):
     """A decorator that checks if remote data is loaded, and if not, attempts to fetch it.
 
@@ -188,7 +204,7 @@ class NotReadyObject:
         return self
 
 
-class Results(UserList, ResultsOperationsMixin, Base):
+class Results(MutableSequence, ResultsOperationsMixin, Base):
     """A collection of Result objects with powerful data analysis capabilities.
 
     The Results class is the primary container for working with data from EDSL surveys.
@@ -297,13 +313,11 @@ class Results(UserList, ResultsOperationsMixin, Base):
         job_uuid: Optional[str] = None,
         total_results: Optional[int] = None,
         task_history: Optional[TaskHistory] = None,
+        sort_by_iteration: bool = False,
+        data_class: Optional[type] = list,  # ResultsSQLList,
     ):
         """Instantiate a Results object with a survey and a list of Result objects.
 
-        This initializes a completed Results object with the provided data.
-        For creating a not-ready Results object from a job info dictionary,
-        use the from_job_info class method instead.
-
         Args:
             survey: A Survey object containing the questions used to generate results.
             data: A list of Result objects containing the responses.
@@ -312,29 +326,49 @@ class Results(UserList, ResultsOperationsMixin, Base):
             job_uuid: A string representing the job UUID.
             total_results: An integer representing the total number of results.
             task_history: A TaskHistory object containing information about the tasks.
-
-        Examples:
-            >>> from ..results import Result
-            >>> # Create an empty Results object
-            >>> r = Results()
-            >>> r.completed
-            True
-            >>> len(r.created_columns)
-            0
-
-            >>> # Create a Results object with data
-            >>> from unittest.mock import Mock
-            >>> mock_survey = Mock()
-            >>> mock_result = Mock(spec=Result)
-            >>> r = Results(survey=mock_survey, data=[mock_result])
-            >>> len(r)
-            1
+            sort_by_iteration: Whether to sort data by iteration before initializing.
+            data_class: The class to use for the data container (default: list).
         """
         self.completed = True
         self._fetching = False
-        super().__init__(data)
+
+        # Determine the data class to use
+        if data is not None:
+            # Use the class of the provided data if it's not a basic list
+            self._data_class = (
+                data.__class__ if not isinstance(data, list) else data_class
+            )
+        else:
+            self._data_class = data_class
+
+        # Sort data appropriately before initialization if needed
+        if data and sort_by_iteration:
+            # First try to sort by order attribute if present on any result
+            has_order = any(hasattr(item, "order") for item in data)
+            if has_order:
+
+                def get_order(item):
+                    if hasattr(item, "order"):
+                        return item.order
+                    return item.data.get("iteration", 0) * 1000
+
+                data = sorted(data, key=get_order)
+            else:
+                data = sorted(data, key=lambda x: x.data.get("iteration", 0))
+
+        # Initialize data with the appropriate class
+        self.data = self._data_class(data or [])
+
         from ..caching import Cache
         from ..tasks import TaskHistory
+        import tempfile
+        import os
+
+        # Create a unique shelve path in the system temp directory
+        self._shelve_path = os.path.join(
+            tempfile.gettempdir(), f"edsl_results_{os.getpid()}"
+        )
+        self._shelf_keys = set()  # Track shelved result keys
 
         self.survey = survey
         self.created_columns = created_columns or []
@@ -347,6 +381,9 @@ class Results(UserList, ResultsOperationsMixin, Base):
         if hasattr(self, "_add_output_functions"):
             self._add_output_functions()
 
+    def add_task_history_entry(self, interview: "Interview") -> None:
+        self.task_history.add_interview(interview)
+
     def _fetch_list(self, data_type: str, key: str) -> list:
         """Return a list of values from the data for a given data type and key.
 
@@ -395,6 +432,32 @@ class Results(UserList, ResultsOperationsMixin, Base):
         return self._fetch_list("answer", question_name)
 
     def _summary(self) -> dict:
+        """Return a dictionary containing summary statistics about the Results object.
+
+        The summary includes:
+        - Number of observations (results)
+        - Number of unique agents
+        - Number of unique models
+        - Number of unique scenarios
+        - Number of questions in the survey
+        - Survey question names (truncated for readability)
+
+        Returns:
+            dict: A dictionary containing the summary statistics
+
+        Examples:
+            >>> from edsl.results import Results
+            >>> r = Results.example()
+            >>> summary = r._summary()
+            >>> isinstance(summary, dict)
+            True
+            >>> all(key in summary for key in ['observations', 'agents', 'models', 'scenarios', 'questions', 'Survey question names'])
+            True
+            >>> summary['observations'] > 0
+            True
+            >>> summary['questions'] > 0
+            True
+        """
         import reprlib
 
         d = {
@@ -407,7 +470,22 @@ class Results(UserList, ResultsOperationsMixin, Base):
         }
         return d
 
-    def _cache_keys(self):
+    def _cache_keys(self) -> List[str]:  # -> list:
+        """Return a list of all cache keys from the results.
+
+        This method collects all cache keys by iterating through each result in the data
+        and extracting the values from the 'cache_keys' dictionary. These keys can be used
+        to identify cached responses and manage the cache effectively.
+
+        Returns:
+            List[str]: A list of cache keys from all results.
+
+        Examples:
+            >>> from edsl.results import Results
+            >>> r = Results.example()
+            >>> all([type(s) == str for s in r._cache_keys()])
+            True
+        """
         cache_keys = []
         for result in self:
             cache_keys.extend(list(result["cache_keys"].values()))
@@ -417,31 +495,57 @@ class Results(UserList, ResultsOperationsMixin, Base):
         cache_keys = self._cache_keys()
         return cache.subset(cache_keys)
 
-    def insert(self, item):
-        item_order = getattr(item, "order", None)
-        if item_order is not None:
-            # Get list of orders, putting None at the end
-            orders = [getattr(x, "order", None) for x in self]
-            # Filter to just the non-None orders for bisect
-            sorted_orders = [x for x in orders if x is not None]
-            if sorted_orders:
-                index = bisect_left(sorted_orders, item_order)
-                # Account for any None values before this position
-                index += orders[:index].count(None)
-            else:
-                # If no sorted items yet, insert before any unordered items
-                index = 0
-            self.data.insert(index, item)
-        else:
-            # No order - append to end
-            self.data.append(item)
+    # def insert(self, item):
+    #     """Insert a Result object into the Results list in the correct order.
+
+    #     If the Result has an 'order' attribute, it uses that for ordering.
+    #     Otherwise, it falls back to ordering by the 'iteration' attribute.
+
+    #     >>> from edsl.results import Result
+    #     >>> rnew = Result.example()
+    #     >>> results = Results.example()
+    #     >>> results.insert(rnew)
+    #     >>> results[0] == rnew
+    #     True
+    #     >>> results = Results.example()
+    #     >>> rnew.order = 100
+    #     >>> results.insert(rnew)
+    #     >>> results[-1] == rnew  # The new result is at the end
+    #     True
+    #     """
+
+    #     def get_sort_key(result):
+    #         if hasattr(result, "order"):
+    #             return result.order
+    #         return result.data["iteration"]
+
+    #     # Find insertion point using bisect with custom key function
+    #     index = bisect_left([get_sort_key(x) for x in self.data], get_sort_key(item))
+
+    #     # Call the parent class's insert directly
+    #     MutableSequence.insert(self, index, item)
+
+    def extend_sorted(self, other):
+        """Extend the Results list with items from another iterable.
+
+        This method preserves ordering based on 'order' attribute if present,
+        otherwise falls back to 'iteration' attribute.
+        """
+        # Collect all items (existing and new)
+        all_items = list(self.data)
+        all_items.extend(other)
 
-    def append(self, item):
-        self.insert(item)
+        # Sort combined list by order attribute if available, otherwise by iteration
+        def get_sort_key(item):
+            if hasattr(item, "order"):
+                return (0, item.order)  # Order attribute takes precedence
+            return (1, item.data["iteration"])  # Iteration is secondary
 
-    def extend(self, other):
-        for item in other:
-            self.insert(item)
+        all_items.sort(key=get_sort_key)
+
+        # Clear and refill with sorted items
+        self.data.clear()
+        self.data.extend(all_items)
 
     def compute_job_cost(self, include_cached_responses_in_cost: bool = False) -> float:
         """Compute the cost of a completed job in USD.
@@ -468,8 +572,16 @@ class Results(UserList, ResultsOperationsMixin, Base):
                 if key.endswith("_cost"):
                     result_cost = result["raw_model_response"][key]
 
+                    # Extract the question name from the key
                     question_name = key.removesuffix("_cost")
-                    cache_used = result["cache_used_dict"][question_name]
+
+                    # Get cache status safely - default to False if not found
+                    cache_used = False
+                    if (
+                        "cache_used_dict" in result
+                        and question_name in result["cache_used_dict"]
+                    ):
+                        cache_used = result["cache_used_dict"][question_name]
 
                     if isinstance(result_cost, (int, float)):
                         if include_cached_responses_in_cost:
@@ -496,48 +608,59 @@ class Results(UserList, ResultsOperationsMixin, Base):
         """
         raise ResultsError("The code() method is not implemented for Results objects")
 
+    @ensure_ready
     def __getitem__(self, i):
-        """Get an item from the Results object by index, slice, or key.
+        if isinstance(i, int):
+            return self.data[i]
+        if isinstance(i, slice):
+            return self.__class__(survey=self.survey, data=self.data[i])
+        if isinstance(i, str):
+            return self.to_dict()[i]
+        raise ResultsError("Invalid argument type for indexing Results object")
 
-        Args:
-            i: An integer index, a slice, or a string key.
+    @ensure_ready
+    def __setitem__(self, i, item):
+        self.data[i] = item
 
-        Returns:
-            The requested item, slice of results, or dictionary value.
+    @ensure_ready
+    def __delitem__(self, i):
+        del self.data[i]
 
-        Raises:
-            ResultsError: If the argument type is invalid for indexing.
+    @ensure_ready
+    def __len__(self):
+        return len(self.data)
 
-        Examples:
-            >>> from edsl.results import Results
-            >>> r = Results.example()
-            >>> # Get by integer index
-            >>> result = r[0]
-            >>> # Get by slice
-            >>> subset = r[0:2]
-            >>> len(subset) == 2
-            True
-            >>> # Get by string key
-            >>> data = r["data"]
-            >>> isinstance(data, list)
-            True
-            >>> # Invalid index type
-            >>> try:
-            ...     r[1.5]
-            ... except ResultsError:
-            ...     True
-            True
+    @ensure_ready
+    def insert(self, index, item):
+        self.data.insert(index, item)
+
+    @ensure_ready
+    def extend(self, other):
+        """Extend the Results list with items from another iterable."""
+        self.data.extend(other)
+
+    @ensure_ready
+    def extend_sorted(self, other):
+        """Extend the Results list with items from another iterable, maintaining sort order.
+
+        This method preserves ordering based on 'order' attribute if present,
+        otherwise falls back to 'iteration' attribute.
         """
-        if isinstance(i, int):
-            return self.data[i]
+        # Collect all items (existing and new)
+        all_items = list(self.data)
+        all_items.extend(other)
 
-        if isinstance(i, slice):
-            return self.__class__(survey=self.survey, data=self.data[i])
+        # Sort combined list by order attribute if available, otherwise by iteration
+        def get_sort_key(item):
+            if hasattr(item, "order"):
+                return (0, item.order)  # Order attribute takes precedence
+            return (1, item.data["iteration"])  # Iteration is secondary
 
-        if isinstance(i, str):
-            return self.to_dict()[i]
+        all_items.sort(key=get_sort_key)
 
-        raise ResultsError("Invalid argument type for indexing Results object")
+        # Clear and refill with sorted items
+        self.data.clear()
+        self.data.extend(all_items)
 
     def __add__(self, other: Results) -> Results:
         """Add two Results objects together.
@@ -581,9 +704,15 @@ class Results(UserList, ResultsOperationsMixin, Base):
                 "The created columns are not the same so they cannot be added together."
             )
 
+        # Create a new ResultsSQLList with the combined data
+        # combined_data = ResultsSQLList()
+        combined_data = self._data_class()
+        combined_data.extend(self.data)
+        combined_data.extend(other.data)
+
         return Results(
             survey=self.survey,
-            data=self.data + other.data,
+            data=combined_data,
             created_columns=self.created_columns,
         )
 
@@ -743,7 +872,12 @@ class Results(UserList, ResultsOperationsMixin, Base):
 
     def __hash__(self) -> int:
         return dict_hash(
-            self.to_dict(sort=True, add_edsl_version=False, include_cache_info=False)
+            self.to_dict(
+                sort=True,
+                add_edsl_version=False,
+                include_cache=False,
+                include_cache_info=False,
+            )
         )
 
     @property
@@ -792,10 +926,11 @@ class Results(UserList, ResultsOperationsMixin, Base):
         """
         from ..surveys import Survey
         from ..caching import Cache
-        from ..results import Result
+        from .result import Result
         from ..tasks import TaskHistory
 
         survey = Survey.from_dict(data["survey"])
+        # Convert dictionaries to Result objects
         results_data = [Result.from_dict(r) for r in data["data"]]
         created_columns = data.get("created_columns", None)
         cache = Cache.from_dict(data.get("cache")) if "cache" in data else Cache()
@@ -804,9 +939,12 @@ class Results(UserList, ResultsOperationsMixin, Base):
             if "task_history" in data
             else TaskHistory(interviews=[])
         )
+
+        # Create a Results object with original order preserved
+        # using the empty data list initially
         params = {
             "survey": survey,
-            "data": results_data,
+            "data": [],  # Start with empty data
             "created_columns": created_columns,
             "cache": cache,
             "task_history": task_history,
@@ -814,6 +952,9 @@ class Results(UserList, ResultsOperationsMixin, Base):
 
         try:
             results = cls(**params)
+            # Add each result individually to respect order attributes
+            for result in results_data:
+                results.append(result)
         except Exception as e:
             raise ResultsDeserializationError(f"Error in Results.from_dict: {e}")
         return results
@@ -1081,19 +1222,26 @@ class Results(UserList, ResultsOperationsMixin, Base):
         >>> r.add_column('a', [1,2,3, 4]).select('a')
         Dataset([{'answer.a': [1, 2, 3, 4]}])
         """
-
         assert len(values) == len(
             self.data
         ), "The number of values must match the number of results."
-        new_results = self.data.copy()
-        for i, result in enumerate(new_results):
-            result["answer"][column_name] = values[i]
-        return Results(
+
+        # Create new Results object with same properties but empty data
+        new_results = Results(
             survey=self.survey,
-            data=new_results,
+            data=[],
             created_columns=self.created_columns + [column_name],
+            data_class=self._data_class,
         )
 
+        # Process one result at a time
+        for i, result in enumerate(self.data):
+            new_result = result.copy()
+            new_result["answer"][column_name] = values[i]
+            new_results.append(new_result)
+
+        return new_results
+
     @ensure_ready
     def add_columns_from_dict(self, columns: List[dict]) -> Results:
         """Adds columns to Results from a list of dictionaries.
@@ -1234,33 +1382,63 @@ class Results(UserList, ResultsOperationsMixin, Base):
         >>> s = Results.example()
         >>> s.rename('how_feeling', 'how_feeling_new').select('how_feeling_new')
         Dataset([{'answer.how_feeling_new': ['OK', 'Great', 'Terrible', 'OK']}])
-
-        # TODO: Should we allow renaming of scenario fields as well? Probably.
-
         """
+        # Create new Results object with same properties but empty data
+        new_results = Results(
+            survey=self.survey,
+            data=[],
+            created_columns=self.created_columns,
+            data_class=self._data_class,
+        )
+
+        # Update created_columns if old_name was in there
+        if old_name in new_results.created_columns:
+            new_results.created_columns.remove(old_name)
+            new_results.created_columns.append(new_name)
 
+        # Process one result at a time
         for obs in self.data:
-            obs["answer"][new_name] = obs["answer"][old_name]
-            del obs["answer"][old_name]
+            new_result = obs.copy()
+            new_result["answer"][new_name] = new_result["answer"][old_name]
+            del new_result["answer"][old_name]
+            new_results.append(new_result)
 
-        return self
+        return new_results
 
     @ensure_ready
     def shuffle(self, seed: Optional[str] = "edsl") -> Results:
-        """Shuffle the results.
+        """Return a shuffled copy of the results using Fisher-Yates algorithm.
 
-        Example:
+        Args:
+            seed: Random seed for reproducibility.
 
-        >>> r = Results.example()
-        >>> r.shuffle(seed = 1)[0]
-        Result(...)
+        Returns:
+            Results: A new Results object with shuffled data.
         """
         if seed != "edsl":
-            seed = random.seed(seed)
+            random.seed(seed)
 
-        new_data = self.data.copy()
-        random.shuffle(new_data)
-        return Results(survey=self.survey, data=new_data, created_columns=None)
+        # Create new Results object with same properties but empty data
+        shuffled_results = Results(
+            survey=self.survey,
+            data=[],
+            created_columns=self.created_columns,
+            data_class=self._data_class,
+        )
+
+        # First pass: copy data while tracking indices
+        indices = list(range(len(self.data)))
+
+        # Second pass: Fisher-Yates shuffle on indices
+        for i in range(len(indices) - 1, 0, -1):
+            j = random.randrange(i + 1)
+            indices[i], indices[j] = indices[j], indices[i]
+
+        # Final pass: append items in shuffled order
+        for idx in indices:
+            shuffled_results.append(self.data[idx])
+
+        return shuffled_results
 
     @ensure_ready
     def sample(
@@ -1270,41 +1448,61 @@ class Results(UserList, ResultsOperationsMixin, Base):
         with_replacement: bool = True,
         seed: Optional[str] = None,
     ) -> Results:
-        """Sample the results.
-
-        :param n: An integer representing the number of samples to take.
-        :param frac: A float representing the fraction of samples to take.
-        :param with_replacement: A boolean representing whether to sample with replacement.
-        :param seed: An integer representing the seed for the random number generator.
+        """Return a random sample of the results.
 
-        Example:
+        Args:
+            n: The number of samples to take.
+            frac: The fraction of samples to take (alternative to n).
+            with_replacement: Whether to sample with replacement.
+            seed: Random seed for reproducibility.
 
-        >>> r = Results.example()
-        >>> len(r.sample(2))
-        2
+        Returns:
+            Results: A new Results object containing the sampled data.
         """
         if seed:
             random.seed(seed)
 
         if n is None and frac is None:
-            from .exceptions import ResultsError
-
             raise ResultsError("You must specify either n or frac.")
 
         if n is not None and frac is not None:
-            from .exceptions import ResultsError
-
             raise ResultsError("You cannot specify both n and frac.")
 
-        if frac is not None and n is None:
+        if frac is not None:
             n = int(frac * len(self.data))
 
+        # Create new Results object with same properties but empty data
+        sampled_results = Results(
+            survey=self.survey,
+            data=[],
+            created_columns=self.created_columns,
+            data_class=self._data_class,
+        )
+
         if with_replacement:
-            new_data = random.choices(self.data, k=n)
+            # For sampling with replacement, we can generate indices and sample one at a time
+            indices = (random.randrange(len(self.data)) for _ in range(n))
+            for i in indices:
+                sampled_results.append(self.data[i])
         else:
-            new_data = random.sample(self.data, n)
+            # For sampling without replacement, use reservoir sampling
+            if n > len(self.data):
+                raise ResultsError(
+                    f"Cannot sample {n} items from a list of length {len(self.data)}."
+                )
+
+            # Reservoir sampling algorithm
+            for i, item in enumerate(self.data):
+                if i < n:
+                    # Fill the reservoir initially
+                    sampled_results.append(item)
+                else:
+                    # Randomly replace items with decreasing probability
+                    j = random.randrange(i + 1)
+                    if j < n:
+                        sampled_results.data[j] = item
 
-        return Results(survey=self.survey, data=new_data, created_columns=None)
+        return sampled_results
 
     @ensure_ready
     def select(self, *columns: Union[str, list[str]]) -> "Dataset":
@@ -1391,20 +1589,12 @@ class Results(UserList, ResultsOperationsMixin, Base):
     def order_by(self, *columns: str, reverse: bool = False) -> Results:
         """Sort the results by one or more columns.
 
-        :param columns: One or more column names as strings.
-        :param reverse: A boolean that determines whether to sort in reverse order.
-
-        Each column name can be a single key, e.g. "how_feeling", or a dot-separated string, e.g. "answer.how_feeling".
-
-        Example:
-
-        >>> r = Results.example()
-        >>> r.sort_by('how_feeling', reverse=False).select('how_feeling')
-        Dataset([{'answer.how_feeling': ['Great', 'OK', 'OK', 'Terrible']}])
-
-        >>> r.sort_by('how_feeling', reverse=True).select('how_feeling')
-        Dataset([{'answer.how_feeling': ['Terrible', 'OK', 'OK', 'Great']}])
+        Args:
+            columns: One or more column names as strings.
+            reverse: A boolean that determines whether to sort in reverse order.
 
+        Returns:
+            Results: A new Results object with sorted data.
         """
 
         def to_numeric_if_possible(v):
@@ -1418,11 +1608,52 @@ class Results(UserList, ResultsOperationsMixin, Base):
             for col in columns:
                 data_type, key = self._parse_column(col)
                 value = item.get_value(data_type, key)
-                key_components.append(to_numeric_if_possible(value))
+                if isinstance(value, (str, bytes)):
+                    key_components.append(str(value))
+                else:
+                    key_components.append(to_numeric_if_possible(value))
             return tuple(key_components)
 
-        new_data = sorted(self.data, key=sort_key, reverse=reverse)
-        return Results(survey=self.survey, data=new_data, created_columns=None)
+        # Create a new sorted view of the data without materializing it
+        sorted_data = sorted(self.data, key=sort_key, reverse=reverse)
+
+        # Create new Results object that uses the sorted iterator
+        return Results(
+            survey=self.survey,
+            data=sorted_data,  # This will be an iterator, not a materialized list
+            created_columns=self.created_columns,
+            data_class=self._data_class,
+            sort_by_iteration=False,
+        )
+
+    @staticmethod
+    def has_single_equals(expression: str) -> bool:
+        """Check if an expression contains a single equals sign not part of ==, >=, or <=.
+
+        Args:
+            expression: String expression to check
+
+        Returns:
+            bool: True if there is a standalone = sign
+
+        Examples:
+            >>> Results.has_single_equals("x = 1")
+            True
+            >>> Results.has_single_equals("x == 1")
+            False
+            >>> Results.has_single_equals("x >= 1")
+            False
+            >>> Results.has_single_equals("x <= 1")
+            False
+        """
+        # First remove valid operators that contain =
+        cleaned = (
+            expression.replace("==", "")
+            .replace(">=", "")
+            .replace("<=", "")
+            .replace("!=", "")
+        )
+        return "=" in cleaned
 
     @ensure_ready
     def filter(self, expression: str) -> Results:
@@ -1436,6 +1667,8 @@ class Results(UserList, ResultsOperationsMixin, Base):
         Args:
             expression: A string containing a Python expression that evaluates to a boolean.
                        The expression is applied to each Result object individually.
+                       Can be a multi-line string for better readability.
+                       Supports template-style syntax with {{ field }} notation.
 
         Returns:
             A new Results object containing only the Result objects that satisfy the expression.
@@ -1452,6 +1685,8 @@ class Results(UserList, ResultsOperationsMixin, Base):
             - You can use comparison operators like '==', '!=', '>', '<', '>=', '<='
             - You can use membership tests with 'in'
             - You can use string methods like '.startswith()', '.contains()', etc.
+            - The expression can be a multi-line string for improved readability
+            - You can use template-style syntax with double curly braces: {{ field }}
 
         Examples:
             >>> r = Results.example()
@@ -1468,6 +1703,17 @@ class Results(UserList, ResultsOperationsMixin, Base):
             >>> r.filter("agent.status == 'Joyful'").select('agent.status')
             Dataset([{'agent.status': ['Joyful', 'Joyful']}])
 
+            >>> # Using multi-line string for complex conditions
+            >>> r.filter('''
+            ...     how_feeling == 'Great'
+            ...     or how_feeling == 'Terrible'
+            ... ''').select('how_feeling')
+            Dataset([{'answer.how_feeling': ['Great', 'Terrible']}])
+
+            >>> # Using template-style syntax with {{}}
+            >>> r.filter("{{ answer.how_feeling }} == 'Great'").select('how_feeling')
+            Dataset([{'answer.how_feeling': ['Great']}])
+
             >>> # Common error: using = instead of ==
             >>> try:
             ...     r.filter("how_feeling = 'Great'")
@@ -1475,28 +1721,43 @@ class Results(UserList, ResultsOperationsMixin, Base):
             ...     print("ResultsFilterError: You must use '==' instead of '=' in the filter expression.")
             ResultsFilterError: You must use '==' instead of '=' in the filter expression.
         """
+        # Normalize expression by removing extra whitespace and newlines
+        normalized_expression = " ".join(expression.strip().split())
 
-        def has_single_equals(string):
-            if "!=" in string:
-                return False
-            if "=" in string and not (
-                "==" in string or "<=" in string or ">=" in string
-            ):
-                return True
+        # Remove template-style syntax (double curly braces)
+        normalized_expression = normalized_expression.replace("{{", "").replace(
+            "}}", ""
+        )
 
-        if has_single_equals(expression):
+        if self.has_single_equals(normalized_expression):
             raise ResultsFilterError(
                 "You must use '==' instead of '=' in the filter expression."
             )
 
         try:
-            # iterates through all the results and evaluates the expression
-            new_data = []
+            # Create new Results object with same class as original but empty data
+            filtered_results = Results(
+                survey=self.survey,
+                data=[],  # Empty data list
+                created_columns=self.created_columns,
+                data_class=self._data_class,  # Preserve the original data class
+            )
+
+            # Process one result at a time
             for result in self.data:
                 evaluator = self._create_evaluator(result)
-                result.check_expression(expression)  # check expression
-                if evaluator.eval(expression):
-                    new_data.append(result)
+                result.check_expression(normalized_expression)  # check expression
+                if evaluator.eval(normalized_expression):
+                    filtered_results.append(
+                        result
+                    )  # Use append method to add matching results
+
+            if len(filtered_results) == 0:
+                import warnings
+
+                warnings.warn("No results remain after applying the filter.")
+
+            return filtered_results
 
         except ValueError as e:
             raise ResultsFilterError(
@@ -1506,21 +1767,14 @@ class Results(UserList, ResultsOperationsMixin, Base):
             )
         except Exception as e:
             raise ResultsFilterError(
-                f"""Error in filter. Exception:{e}.""",
-                f"""The expression you provided was: {expression}.""",
-                """Please make sure that the expression is a valid Python expression that evaluates to a boolean.""",
-                """For example, 'how_feeling == "Great"' is a valid expression, as is 'how_feeling in ["Great", "Terrible"]'., """,
-                """However, 'how_feeling = "Great"' is not a valid expression.""",
-                """See https://docs.expectedparrot.com/en/latest/results.html#filtering-results for more details.""",
+                f"Error in filter. Exception:{e}.",
+                f"The expression you provided was: {expression}.",
+                "Please make sure that the expression is a valid Python expression that evaluates to a boolean.",
+                'For example, \'how_feeling == "Great"\' is a valid expression, as is \'how_feeling in ["Great", "Terrible"]\'.',
+                "However, 'how_feeling = \"Great\"' is not a valid expression.",
+                "See https://docs.expectedparrot.com/en/latest/results.html#filtering-results for more details.",
             )
 
-        if len(new_data) == 0:
-            import warnings
-
-            warnings.warn("No results remain after applying the filter.")
-
-        return Results(survey=self.survey, data=new_data, created_columns=None)
-
     @classmethod
     def example(cls, randomize: bool = False) -> Results:
         """Return an example `Results` object.
@@ -1529,7 +1783,7 @@ class Results(UserList, ResultsOperationsMixin, Base):
 
         >>> r = Results.example()
 
-        :param debug: if False, uses actual API calls
+        :param randomize: if True, randomizes agent and scenario combinations
         """
         from ..jobs import Jobs
         from ..caching import Cache
@@ -1544,6 +1798,7 @@ class Results(UserList, ResultsOperationsMixin, Base):
             disable_remote_cache=True,
             disable_remote_inference=True,
         )
+
         return results
 
     def rich_print(self):
@@ -1761,6 +2016,282 @@ class Results(UserList, ResultsOperationsMixin, Base):
 
         return results
 
+    def shelve_result(self, result: "Result") -> str:
+        """Store a Result object in persistent storage using its hash as the key.
+
+        Args:
+            result: A Result object to store
+
+        Returns:
+            str: The hash key for retrieving the result later
+
+        Raises:
+            ResultsError: If there's an error storing the Result
+        """
+        import shelve
+
+        key = str(hash(result))
+        try:
+            with shelve.open(self._shelve_path) as shelf:
+                shelf[key] = result.to_dict()
+                self._shelf_keys.add(key)
+            return key
+        except Exception as e:
+            raise ResultsError(f"Error storing Result in shelve database: {str(e)}")
+
+    def get_shelved_result(self, key: str) -> "Result":
+        """Retrieve a Result object from persistent storage.
+
+        Args:
+            key: The hash key of the Result to retrieve
+
+        Returns:
+            Result: The stored Result object
+
+        Raises:
+            ResultsError: If the key doesn't exist or if there's an error retrieving the Result
+        """
+        import shelve
+        from .result import Result
+
+        if key not in self._shelf_keys:
+            raise ResultsError(f"No result found with key: {key}")
+
+        try:
+            with shelve.open(self._shelve_path) as shelf:
+                return Result.from_dict(shelf[key])
+        except Exception as e:
+            raise ResultsError(
+                f"Error retrieving Result from shelve database: {str(e)}"
+            )
+
+    @property
+    def shelf_keys(self) -> set:
+        """Return a copy of the set of shelved result keys."""
+        return self._shelf_keys.copy()
+
+    @ensure_ready
+    def insert_sorted(self, item: "Result") -> None:
+        """Insert a Result object into the Results list while maintaining sort order.
+
+        Uses the 'order' attribute if present, otherwise falls back to 'iteration' attribute.
+        Utilizes bisect for efficient insertion point finding.
+
+        Args:
+            item: A Result object to insert
+
+        Examples:
+            >>> r = Results.example()
+            >>> new_result = r[0].copy()
+            >>> new_result.order = 1.5  # Insert between items
+            >>> r.insert_sorted(new_result)
+        """
+        from bisect import bisect_left
+
+        def get_sort_key(result):
+            if hasattr(result, "order"):
+                return (0, result.order)  # Order attribute takes precedence
+            return (1, result.data["iteration"])  # Iteration is secondary
+
+        # Get the sort key for the new item
+        item_key = get_sort_key(item)
+
+        # Get list of sort keys for existing items
+        keys = [get_sort_key(x) for x in self.data]
+
+        # Find insertion point
+        index = bisect_left(keys, item_key)
+
+        # Insert at the found position
+        self.data.insert(index, item)
+
+    def insert_from_shelf(self) -> None:
+        """Move all shelved results into memory using insert_sorted method.
+        Clears the shelf after successful insertion.
+
+        This method preserves the original order of results by using their 'order'
+        attribute if available, which ensures consistent ordering even after
+        serialization/deserialization.
+
+        Raises:
+            ResultsError: If there's an error accessing or clearing the shelf
+        """
+        import shelve
+        from .result import Result
+
+        if not self._shelf_keys:
+            return
+
+        try:
+            # First collect all results from shelf
+            with shelve.open(self._shelve_path) as shelf:
+                # Get and insert all results first
+                for key in self._shelf_keys:
+                    result_dict = shelf[key]
+                    result = Result.from_dict(result_dict)
+                    self.insert_sorted(result)
+
+                # Now clear the shelf
+                for key in self._shelf_keys:
+                    del shelf[key]
+
+            # Clear the tracking set
+            self._shelf_keys.clear()
+
+        except Exception as e:
+            raise ResultsError(f"Error moving results from shelf to memory: {str(e)}")
+
+    def to_disk(self, filepath: str) -> None:
+        """Serialize the Results object to a zip file, preserving the SQLite database.
+
+        This method creates a zip file containing:
+        1. The SQLite database file from the data container
+        2. A metadata.json file with the survey, created_columns, and other non-data info
+        3. The cache data if present
+
+        Args:
+            filepath: Path where the zip file should be saved
+
+        Raises:
+            ResultsError: If there's an error during serialization
+        """
+        import zipfile
+        import json
+        import os
+        import tempfile
+        from pathlib import Path
+        import sqlite3
+        import shutil
+
+        data_class = ResultsSQLList
+
+        try:
+            # Create a temporary directory to store files before zipping
+            with tempfile.TemporaryDirectory() as temp_dir:
+                temp_path = Path(temp_dir)
+
+                # 1. Handle the SQLite database
+                db_path = temp_path / "results.db"
+
+                if isinstance(self.data, list):
+                    # If data is a list, create a new SQLiteList
+                    # from .sqlite_list import SQLiteList
+                    new_db = data_class()
+                    new_db.extend(self.data)
+                    shutil.copy2(new_db.db_path, db_path)
+                elif hasattr(self.data, "db_path") and os.path.exists(
+                    self.data.db_path
+                ):
+                    # If data is already a SQLiteList, copy its database
+                    shutil.copy2(self.data.db_path, db_path)
+                else:
+                    # If no database exists, create a new one
+                    # from .sqlite_list import SQLiteList
+                    # new_db = SQLiteList()
+                    new_db = data_class()
+                    new_db.extend(self.data)
+                    shutil.copy2(new_db.db_path, db_path)
+
+                # 2. Create metadata.json
+                metadata = {
+                    "survey": self.survey.to_dict() if self.survey else None,
+                    "created_columns": self.created_columns,
+                    "cache": self.cache.to_dict() if hasattr(self, "cache") else None,
+                    "task_history": self.task_history.to_dict()
+                    if hasattr(self, "task_history")
+                    else None,
+                    "completed": self.completed,
+                    "job_uuid": self._job_uuid if hasattr(self, "_job_uuid") else None,
+                    "total_results": self._total_results
+                    if hasattr(self, "_total_results")
+                    else None,
+                }
+
+                metadata_path = temp_path / "metadata.json"
+                metadata_path.write_text(json.dumps(metadata, indent=4))
+
+                # 3. Create the zip file
+                with zipfile.ZipFile(filepath, "w", zipfile.ZIP_DEFLATED) as zipf:
+                    # Add all files from temp directory to zip
+                    for file in temp_path.glob("*"):
+                        zipf.write(file, file.name)
+
+        except Exception as e:
+            raise ResultsError(f"Error saving Results to disk: {str(e)}")
+
+    @classmethod
+    def from_disk(cls, filepath: str) -> "Results":
+        """Load a Results object from a zip file.
+
+        This method:
+        1. Extracts the SQLite database file
+        2. Loads the metadata
+        3. Creates a new Results instance with the restored data
+
+        Args:
+            filepath: Path to the zip file containing the serialized Results
+
+        Returns:
+            Results: A new Results instance with the restored data
+
+        Raises:
+            ResultsError: If there's an error during deserialization
+        """
+        import zipfile
+        import json
+        import tempfile
+        from pathlib import Path
+        from ..surveys import Survey
+        from ..caching import Cache
+        from ..tasks import TaskHistory
+
+        data_class = ResultsSQLList
+
+        try:
+            # Create a temporary directory to extract files
+            with tempfile.TemporaryDirectory() as temp_dir:
+                temp_path = Path(temp_dir)
+
+                # Extract the zip file
+                with zipfile.ZipFile(filepath, "r") as zipf:
+                    zipf.extractall(temp_path)
+
+                # 1. Load metadata
+                metadata_path = temp_path / "metadata.json"
+                metadata = json.loads(metadata_path.read_text())
+
+                # 2. Create a new Results instance
+                results = cls(
+                    survey=Survey.from_dict(metadata["survey"])
+                    if metadata["survey"]
+                    else None,
+                    created_columns=metadata["created_columns"],
+                    cache=Cache.from_dict(metadata["cache"])
+                    if metadata["cache"]
+                    else None,
+                    task_history=TaskHistory.from_dict(metadata["task_history"])
+                    if metadata["task_history"]
+                    else None,
+                    job_uuid=metadata["job_uuid"],
+                    total_results=metadata["total_results"],
+                )
+
+                # 3. Set the SQLite database path if it exists
+                db_path = temp_path / "results.db"
+                if db_path.exists():
+                    # Create a new ResultsSQLList instance
+                    new_db = data_class()
+                    # Copy data from the source database - convert Path to string
+                    new_db.copy_from(str(db_path))
+                    # Set the new database as the results data
+                    results.data = new_db
+
+                results.completed = metadata["completed"]
+                return results
+
+        except Exception as e:
+            raise ResultsError(f"Error loading Results from disk: {str(e)}")
+
 
 def main():  # pragma: no cover
     """Run example operations on a Results object.