edsl 0.1.15__py3-none-any.whl → 0.1.40__py3-none-any.whl

edsl/results/Results.py

@@ -1,179 +1,542 @@
+"""
+The Results object is the result of running a survey. 
+It is not typically instantiated directly, but is returned by the run method of a `Job` object.
+"""
+
 from __future__ import annotations
 import json
-import io
+import random
 from collections import UserList, defaultdict
+from typing import Optional, Callable, Any, Type, Union, List, TYPE_CHECKING
 
-from rich.console import Console
+from bisect import bisect_left
 
-from simpleeval import EvalWithCompoundTypes
-from typing import Any, Type, Union
-from edsl.exceptions import (
+from edsl.Base import Base
+from edsl.exceptions.results import (
+    ResultsError,
     ResultsBadMutationstringError,
     ResultsColumnNotFoundError,
     ResultsInvalidNameError,
     ResultsMutateError,
+    ResultsFilterError,
+    ResultsDeserializationError,
 )
-from edsl.agents import Agent
-from edsl.data import CRUD
-from edsl.language_models import LanguageModel
-from edsl.results.Dataset import Dataset
-from edsl.results.Result import Result
-from edsl.results.ResultsExportMixin import ResultsExportMixin
-from edsl.scenarios import Scenario
-from edsl.surveys import Survey
-from edsl.utilities import (
-    is_gzipped,
-    is_valid_variable_name,
-    print_public_methods_with_doc,
-    shorten_string,
-    is_notebook,
-)
-
-from edsl.results.ResultsDBMixin import ResultsDBMixin
-
-
-class Mixins(ResultsExportMixin, ResultsDBMixin):
-    pass
-
-
-# These are only made available if the user has installed
-# our package from pip with the [report] option
-try:
-    from edsl.report.RegressionMixin import RegressionMixin
-
-    Mixins = type("Mixins", (RegressionMixin, Mixins), {})
-except (ImportError, ModuleNotFoundError):
-    pass
 
-try:
-    from edsl.report.ResultsFetchMixin import ResultsFetchMixin
+if TYPE_CHECKING:
+    from edsl.surveys.Survey import Survey
+    from edsl.data.Cache import Cache
+    from edsl.agents.AgentList import AgentList
+    from edsl.language_models.model import Model
+    from edsl.scenarios.ScenarioList import ScenarioList
+    from edsl.results.Result import Result
+    from edsl.jobs.tasks.TaskHistory import TaskHistory
+    from edsl.language_models.ModelList import ModelList
+    from simpleeval import EvalWithCompoundTypes
 
-    Mixins = type("Mixins", (ResultsFetchMixin, Mixins), {})
-except (ImportError, ModuleNotFoundError):
-    pass
-
-try:
-    from edsl.report.ResultsOutputMixin import ResultsOutputMixin
-
-    Mixins = type("Mixins", (ResultsOutputMixin, Mixins), {})
-except (ImportError, ModuleNotFoundError):
-    pass
+from edsl.results.ResultsExportMixin import ResultsExportMixin
+from edsl.results.ResultsGGMixin import ResultsGGMixin
+from edsl.results.results_fetch_mixin import ResultsFetchMixin
+from edsl.utilities.remove_edsl_version import remove_edsl_version
+
+
+class Mixins(
+    ResultsExportMixin,
+    ResultsFetchMixin,
+    ResultsGGMixin,
+):
+    def long(self):
+        return self.table().long()
+
+    def print_long(self, max_rows: int = None) -> None:
+        """Print the results in long format.
+
+        >>> from edsl.results import Results
+        >>> r = Results.example()
+        >>> r.select('how_feeling').print_long(max_rows = 2)
+        ┏━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━┓
+        ┃ Result index ┃ Key         ┃ Value ┃
+        ┡━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━┩
+        │ 0            │ how_feeling │ OK    │
+        │ 1            │ how_feeling │ Great │
+        └──────────────┴─────────────┴───────┘
+        """
+        from edsl.utilities.interface import print_results_long
 
-from edsl.Base import Base
+        print_results_long(self, max_rows=max_rows)
 
 
 class Results(UserList, Mixins, Base):
     """
     This class is a UserList of Result objects.
-    - It is instantiated with a Survey and a list of Result objects (observations).
-    - It can be manipulated in various ways with select, filter, mutate, etc.
-    - It also has a list of created_columns, which is a list of columns that have been created with `mutate`
+
+    It is instantiated with a `Survey` and a list of `Result` objects.
+    It can be manipulated in various ways with select, filter, mutate, etc.
+    It also has a list of created_columns, which are columns that have been created with `mutate` and are not part of the original data.
     """
 
+    __documentation__ = "https://docs.expectedparrot.com/en/latest/results.html"
+
+    known_data_types = [
+        "answer",
+        "scenario",
+        "agent",
+        "model",
+        "prompt",
+        "raw_model_response",
+        "iteration",
+        "question_text",
+        "question_options",
+        "question_type",
+        "comment",
+        "generated_tokens",
+        "cache_used",
+        "cache_keys",
+    ]
+
     def __init__(
         self,
-        survey: Survey = None,
-        data: list[Result] = None,
-        created_columns: list = None,
-        job_uuid: str = None,
-        total_results: int = None,
+        survey: Optional[Survey] = None,
+        data: Optional[list[Result]] = None,
+        created_columns: Optional[list[str]] = None,
+        cache: Optional[Cache] = None,
+        job_uuid: Optional[str] = None,
+        total_results: Optional[int] = None,
+        task_history: Optional[TaskHistory] = None,
     ):
+        """Instantiate a `Results` object with a survey and a list of `Result` objects.
+
+        :param survey: A Survey object.
+        :param data: A list of Result objects.
+        :param created_columns: A list of strings that are created columns.
+        :param job_uuid: A string representing the job UUID.
+        :param total_results: An integer representing the total number of results.
+        :cache: A Cache object.
+        """
         super().__init__(data)
+        from edsl.data.Cache import Cache
+        from edsl.jobs.tasks.TaskHistory import TaskHistory
+
         self.survey = survey
         self.created_columns = created_columns or []
         self._job_uuid = job_uuid
         self._total_results = total_results
+        self.cache = cache or Cache()
 
-        if hasattr(self, "add_output_functions"):
-            self.add_output_functions()
+        self.task_history = task_history or TaskHistory(interviews=[])
 
-    ######################
-    # Streaming methods
-    ######################
+        if hasattr(self, "_add_output_functions"):
+            self._add_output_functions()
+
+    def _summary(self) -> dict:
+        import reprlib
+
+        d = {
+            "observations": len(self),
+            "agents": len(set(self.agents)),
+            "models": len(set(self.models)),
+            "scenarios": len(set(self.scenarios)),
+            "questions": len(self.survey),
+            "Survey question names": reprlib.repr(self.survey.question_names),
+        }
+        return d
+
+    def _cache_keys(self):
+        cache_keys = []
+        for result in self:
+            cache_keys.extend(list(result["cache_keys"].values()))
+        return cache_keys
+
+    def relevant_cache(self, cache: Cache) -> Cache:
+        cache_keys = self._cache_keys()
+        return cache.subset(cache_keys)
+
+    def insert(self, item):
+        item_order = getattr(item, "order", None)
+        if item_order is not None:
+            # Get list of orders, putting None at the end
+            orders = [getattr(x, "order", None) for x in self]
+            # Filter to just the non-None orders for bisect
+            sorted_orders = [x for x in orders if x is not None]
+            if sorted_orders:
+                index = bisect_left(sorted_orders, item_order)
+                # Account for any None values before this position
+                index += orders[:index].count(None)
+            else:
+                # If no sorted items yet, insert before any unordered items
+                index = 0
+            self.data.insert(index, item)
+        else:
+            # No order - append to end
+            self.data.append(item)
+
+    def append(self, item):
+        self.insert(item)
+
+    def extend(self, other):
+        for item in other:
+            self.insert(item)
+
+    def compute_job_cost(self, include_cached_responses_in_cost: bool = False) -> float:
+        """
+        Computes the cost of a completed job in USD.
+        """
+        total_cost = 0
+        for result in self:
+            for key in result["raw_model_response"]:
+                if key.endswith("_cost"):
+                    result_cost = result["raw_model_response"][key]
+
+                    question_name = key.removesuffix("_cost")
+                    cache_used = result["cache_used_dict"][question_name]
+
+                    if isinstance(result_cost, (int, float)):
+                        if include_cached_responses_in_cost:
+                            total_cost += result_cost
+                        elif not include_cached_responses_in_cost and not cache_used:
+                            total_cost += result_cost
+
+        return total_cost
+
+    def leaves(self):
+        leaves = []
+        for result in self:
+            leaves.extend(result.leaves())
+        return leaves
+
+    def tree(self, node_list: Optional[List[str]] = None):
+        return self.to_scenario_list().tree(node_list)
+
+    def interactive_tree(
+        self,
+        fold_attributes: Optional[List[str]] = None,
+        drop: Optional[List[str]] = None,
+        open_file=True,
+    ) -> dict:
+        """Return the results as a tree."""
+        from edsl.results.tree_explore import FoldableHTMLTableGenerator
+
+        if drop is None:
+            drop = []
+
+        valid_attributes = [
+            "model",
+            "scenario",
+            "agent",
+            "answer",
+            "question",
+            "iteration",
+        ]
+        if fold_attributes is None:
+            fold_attributes = []
+
+        for attribute in fold_attributes:
+            if attribute not in valid_attributes:
+                raise ValueError(
+                    f"Invalid fold attribute: {attribute}; must be in {valid_attributes}"
+                )
+        data = self.leaves()
+        generator = FoldableHTMLTableGenerator(data)
+        tree = generator.tree(fold_attributes=fold_attributes, drop=drop)
+        html_content = generator.generate_html(tree, fold_attributes)
+        import tempfile
+        from edsl.utilities.utilities import is_notebook
+
+        from IPython.display import display, HTML
+
+        if is_notebook():
+            import html
+            from IPython.display import display, HTML
+
+            height = 1000
+            width = 1000
+            escaped_output = html.escape(html_content)
+            # escaped_output = rendered_html
+            iframe = f""""
+            <iframe srcdoc="{ escaped_output }" style="width: {width}px; height: {height}px;"></iframe>
+            """
+            display(HTML(iframe))
+            return None
+
+        with tempfile.NamedTemporaryFile(suffix=".html", delete=False) as f:
+            f.write(html_content.encode())
+            print(f"HTML file has been generated: {f.name}")
+
+            if open_file:
+                import webbrowser
+                import time
+
+                time.sleep(1)  # Wait for 1 second
+                # webbrowser.open(f.name)
+                import os
+
+                filename = f.name
+                webbrowser.open(f"file://{os.path.abspath(filename)}")
+
+            else:
+                return html_content
 
     def code(self):
         raise NotImplementedError
 
     def __getitem__(self, i):
+        if isinstance(i, int):
+            return self.data[i]
+
         if isinstance(i, slice):
-            # Return a sliced view of the list
             return self.__class__(survey=self.survey, data=self.data[i])
-        else:
-            # Return a single item
-            return self.data[i]
 
-    def _update_results(self) -> None:
-        if self._job_uuid and len(self.data) < self._total_results:
-            results = [
-                Result(
-                    agent=Agent.from_dict(json.loads(r.agent)),
-                    scenario=Scenario.from_dict(json.loads(r.scenario)),
-                    model=LanguageModel.from_dict(json.loads(r.model)),
-                    iteration=1,
-                    answer=json.loads(r.answer),
-                )
-                for r in CRUD.read_results(self._job_uuid)
-            ]
-            self.data = results
+        if isinstance(i, str):
+            return self.to_dict()[i]
+
+        raise TypeError("Invalid argument type")
+
+    def __add__(self, other: Results) -> Results:
+        """Add two Results objects together.
+        They must have the same survey and created columns.
+        :param other: A Results object.
+
+        Example:
+
+        >>> r = Results.example()
+        >>> r2 = Results.example()
+        >>> r3 = r + r2
+        """
+        if self.survey != other.survey:
+            raise ResultsError(
+                "The surveys are not the same so the the results cannot be added together."
+            )
+        if self.created_columns != other.created_columns:
+            raise ResultsError(
+                "The created columns are not the same so they cannot be added together."
+            )
+
+        return Results(
+            survey=self.survey,
+            data=self.data + other.data,
+            created_columns=self.created_columns,
+        )
 
     def __repr__(self) -> str:
-        self._update_results()
-        if self._job_uuid and len(self.data) < self._total_results:
-            print(f"Completeness: {len(self.data)}/{self._total_results}")
-        data = [repr(result) for result in self.data]
-        if is_notebook():
-            return self.rich_print()
+        return f"Results(data = {self.data}, survey = {repr(self.survey)}, created_columns = {self.created_columns})"
+
+    def table(
+        self,
+        *fields,
+        tablefmt: Optional[str] = None,
+        pretty_labels: Optional[dict] = None,
+        print_parameters: Optional[dict] = None,
+    ):
+        new_fields = []
+        for field in fields:
+            if "." in field:
+                data_type, key = field.split(".")
+                if data_type not in self.known_data_types:
+                    raise ResultsInvalidNameError(
+                        f"{data_type} is not a valid data type. Must be in {self.known_data_types}"
+                    )
+                if key == "*":
+                    for k in self._data_type_to_keys[data_type]:
+                        new_fields.append(k)
+                else:
+                    if key not in self._key_to_data_type:
+                        raise ResultsColumnNotFoundError(
+                            f"{key} is not a valid key. Must be in {self._key_to_data_type}"
+                        )
+                    new_fields.append(key)
+            else:
+                new_fields.append(field)
+
+        return (
+            self.to_scenario_list()
+            .to_dataset()
+            .table(
+                *new_fields,
+                tablefmt=tablefmt,
+                pretty_labels=pretty_labels,
+                print_parameters=print_parameters,
+            )
+        )
+
+    def to_dict(
+        self,
+        sort: bool = False,
+        add_edsl_version: bool = False,
+        include_cache: bool = True,
+        include_task_history: bool = False,
+        include_cache_info: bool = True,
+    ) -> dict[str, Any]:
+        from edsl.data.Cache import Cache
+
+        if sort:
+            data = sorted([result for result in self.data], key=lambda x: hash(x))
         else:
-            return f"Results(data = {data}, survey = {repr(self.survey)}, created_columns = {self.created_columns})"
+            data = [result for result in self.data]
 
-    def to_dict(self) -> dict[str, Any]:
-        """Converts the Results object to a dictionary"""
-        return {
-            "data": [result.to_dict() for result in self.data],
-            "survey": self.survey.to_dict(),
+        d = {
+            "data": [
+                result.to_dict(
+                    add_edsl_version=add_edsl_version,
+                    include_cache_info=include_cache_info,
+                )
+                for result in data
+            ],
+            "survey": self.survey.to_dict(add_edsl_version=add_edsl_version),
             "created_columns": self.created_columns,
         }
+        if include_cache:
+            d.update(
+                {
+                    "cache": (
+                        Cache()
+                        if not hasattr(self, "cache")
+                        else self.cache.to_dict(add_edsl_version=add_edsl_version)
+                    )
+                }
+            )
+
+        if self.task_history.has_unfixed_exceptions or include_task_history:
+            d.update({"task_history": self.task_history.to_dict()})
+
+        if add_edsl_version:
+            from edsl import __version__
+
+            d["edsl_version"] = __version__
+            d["edsl_class_name"] = "Results"
+
+        return d
+
+    def compare(self, other_results: Results) -> dict:
+        """
+        Compare two Results objects and return the differences.
+        """
+        hashes_0 = [hash(result) for result in self]
+        hashes_1 = [hash(result) for result in other_results]
+
+        in_self_but_not_other = set(hashes_0).difference(set(hashes_1))
+        in_other_but_not_self = set(hashes_1).difference(set(hashes_0))
+
+        indicies_self = [hashes_0.index(h) for h in in_self_but_not_other]
+        indices_other = [hashes_1.index(h) for h in in_other_but_not_self]
+        return {
+            "a_not_b": [self[i] for i in indicies_self],
+            "b_not_a": [other_results[i] for i in indices_other],
+        }
+
+    @property
+    def has_unfixed_exceptions(self) -> bool:
+        return self.task_history.has_unfixed_exceptions
+
+    def __hash__(self) -> int:
+        from edsl.utilities.utilities import dict_hash
+
+        return dict_hash(
+            self.to_dict(sort=True, add_edsl_version=False, include_cache_info=False)
+        )
+
+    @property
+    def hashes(self) -> set:
+        return set(hash(result) for result in self.data)
+
+    def sample(self, n: int) -> Results:
+        """Return a random sample of the results.
+
+        :param n: The number of samples to return.
+
+        >>> from edsl.results import Results
+        >>> r = Results.example()
+        >>> len(r.sample(2))
+        2
+        """
+        indices = None
+
+        for entry in self:
+            key, values = list(entry.items())[0]
+            if indices is None:  # gets the indices for the first time
+                indices = list(range(len(values)))
+                sampled_indices = random.sample(indices, n)
+                if n > len(indices):
+                    raise ResultsError(
+                        f"Cannot sample {n} items from a list of length {len(indices)}."
+                    )
+            entry[key] = [values[i] for i in sampled_indices]
+
+        return self
 
     @classmethod
+    @remove_edsl_version
     def from_dict(cls, data: dict[str, Any]) -> Results:
-        """Converts a dictionary to a Results object"""
-        results = cls(
-            survey=Survey.from_dict(data["survey"]),
-            data=[Result.from_dict(r) for r in data["data"]],
-            created_columns=data.get("created_columns", None),
+        """Convert a dictionary to a Results object.
+
+        :param data: A dictionary representation of a Results object.
+
+        Example:
+
+        >>> r = Results.example()
+        >>> d = r.to_dict()
+        >>> r2 = Results.from_dict(d)
+        >>> r == r2
+        True
+        """
+        from edsl.surveys.Survey import Survey
+        from edsl.data.Cache import Cache
+        from edsl.results.Result import Result
+        from edsl.jobs.tasks.TaskHistory import TaskHistory
+        from edsl.agents.Agent import Agent
+
+        survey = Survey.from_dict(data["survey"])
+        results_data = [Result.from_dict(r) for r in data["data"]]
+        created_columns = data.get("created_columns", None)
+        cache = Cache.from_dict(data.get("cache")) if "cache" in data else Cache()
+        task_history = (
+            TaskHistory.from_dict(data.get("task_history"))
+            if "task_history" in data
+            else TaskHistory(interviews=[])
         )
-        return results
+        params = {
+            "survey": survey,
+            "data": results_data,
+            "created_columns": created_columns,
+            "cache": cache,
+            "task_history": task_history,
+        }
 
-    def show_methods(self, show_docstrings: bool = True):
-        """Prints public methods of the Results class"""
-        print_public_methods_with_doc(self)
+        try:
+            results = cls(**params)
+        except Exception as e:
+            raise ResultsDeserializationError(f"Error in Results.from_dict: {e}")
+        return results
 
-    ######################
-    ## Convenience methods
-    ## & Report methods
-    ######################
     @property
     def _key_to_data_type(self) -> dict[str, str]:
         """
-        Returns a mapping of keys (how_feeling, status, etc.) to strings representing data types (objects such as Agent, Answer, Model, Scenario, etc.)
+        Return a mapping of keys (how_feeling, status, etc.) to strings representing data types.
+
+        Objects such as Agent, Answer, Model, Scenario, etc.
         - Uses the key_to_data_type property of the Result class.
         - Includes any columns that the user has created with `mutate`
         """
-        d = {}
+        d: dict = {}
         for result in self.data:
             d.update(result.key_to_data_type)
         for column in self.created_columns:
             d[column] = "answer"
+
         return d
 
     @property
     def _data_type_to_keys(self) -> dict[str, str]:
         """
-        Returns a mapping of strings representing data types (objects such as Agent, Answer, Model, Scenario, etc.) to keys (how_feeling, status, etc.)
+        Return a mapping of strings representing data types (objects such as Agent, Answer, Model, Scenario, etc.) to keys (how_feeling, status, etc.)
         - Uses the key_to_data_type property of the Result class.
         - Includes any columns that the user has created with `mutate`
+
+        Example:
+
+        >>> r = Results.example()
+        >>> r._data_type_to_keys
+        defaultdict(...
         """
-        d = defaultdict(set)
+        d: dict = defaultdict(set)
         for result in self.data:
             for key, value in result.key_to_data_type.items():
                 d[value] = d[value].union(set({key}))
@@ -183,102 +546,305 @@ class Results(UserList, Mixins, Base):
 
     @property
     def columns(self) -> list[str]:
-        """Returns a list of all of the columns that are in the Results"""
-        # return list(self.all_keys)
+        """Return a list of all of the columns that are in the Results.
+
+        Example:
+
+        >>> r = Results.example()
+        >>> r.columns
+        ['agent.agent_index', ...]
+        """
         column_names = [f"{v}.{k}" for k, v in self._key_to_data_type.items()]
-        return sorted(column_names)
+        from edsl.utilities.PrettyList import PrettyList
+
+        return PrettyList(sorted(column_names))
 
     @property
     def answer_keys(self) -> dict[str, str]:
-        """Returns a mapping of answer keys to question text"""
+        """Return a mapping of answer keys to question text.
+
+        Example:
+
+        >>> r = Results.example()
+        >>> r.answer_keys
+        {'how_feeling': 'How are you this {{ period }}?', 'how_feeling_yesterday': 'How were you feeling yesterday {{ period }}?'}
+        """
+        from edsl.utilities.utilities import shorten_string
+
+        if not self.survey:
+            raise ResultsError("Survey is not defined so no answer keys are available.")
+
         answer_keys = self._data_type_to_keys["answer"]
         answer_keys = {k for k in answer_keys if "_comment" not in k}
         questions_text = [
-            self.survey.get_question(k).question_text for k in answer_keys
+            self.survey._get_question_by_name(k).question_text for k in answer_keys
         ]
         short_question_text = [shorten_string(q, 80) for q in questions_text]
-        return dict(zip(answer_keys, short_question_text))
+        initial_dict = dict(zip(answer_keys, short_question_text))
+        sorted_dict = {key: initial_dict[key] for key in sorted(initial_dict)}
+        return sorted_dict
 
     @property
-    def agents(self) -> list[Agent]:
-        return [r.agent for r in self.data]
+    def agents(self) -> AgentList:
+        """Return a list of all of the agents in the Results.
+
+        Example:
+
+        >>> r = Results.example()
+        >>> r.agents
+        AgentList([Agent(traits = {'status': 'Joyful'}), Agent(traits = {'status': 'Joyful'}), Agent(traits = {'status': 'Sad'}), Agent(traits = {'status': 'Sad'})])
+        """
+        from edsl.agents.AgentList import AgentList
+
+        return AgentList([r.agent for r in self.data])
 
     @property
-    def models(self) -> list[Type[LanguageModel]]:
-        return [r.model for r in self.data]
+    def models(self) -> ModelList:
+        """Return a list of all of the models in the Results.
+
+        Example:
+
+        >>> r = Results.example()
+        >>> r.models[0]
+        Model(model_name = ...)
+        """
+        from edsl.language_models.ModelList import ModelList
+
+        return ModelList([r.model for r in self.data])
+
+    def __eq__(self, other):
+        return hash(self) == hash(other)
 
     @property
-    def scenarios(self) -> list[Scenario]:
-        return [r.scenario for r in self.data]
+    def scenarios(self) -> ScenarioList:
+        """Return a list of all of the scenarios in the Results.
+
+        Example:
+
+        >>> r = Results.example()
+        >>> r.scenarios
+        ScenarioList([Scenario({'period': 'morning', 'scenario_index': 0}), Scenario({'period': 'afternoon', 'scenario_index': 1}), Scenario({'period': 'morning', 'scenario_index': 0}), Scenario({'period': 'afternoon', 'scenario_index': 1})])
+        """
+        from edsl.scenarios.ScenarioList import ScenarioList
+
+        return ScenarioList([r.scenario for r in self.data])
 
     @property
-    def agent_keys(self) -> set[str]:
-        """Returns a set of all of the keys that are in the Agent data"""
-        return self._data_type_to_keys["agent"]
+    def agent_keys(self) -> list[str]:
+        """Return a set of all of the keys that are in the Agent data.
+
+        Example:
+
+        >>> r = Results.example()
+        >>> r.agent_keys
+        ['agent_index', 'agent_instruction', 'agent_name', 'status']
+        """
+        return sorted(self._data_type_to_keys["agent"])
 
     @property
-    def model_keys(self) -> set[str]:
-        """Returns a set of all of the keys that are in the LanguageModel data"""
-        return self._data_type_to_keys["model"]
+    def model_keys(self) -> list[str]:
+        """Return a set of all of the keys that are in the LanguageModel data.
+
+        >>> r = Results.example()
+        >>> r.model_keys
+        ['frequency_penalty', 'logprobs', 'max_tokens', 'model', 'model_index', 'presence_penalty', 'temperature', 'top_logprobs', 'top_p']
+        """
+        return sorted(self._data_type_to_keys["model"])
 
     @property
-    def scenario_keys(self) -> set[str]:
-        """Returns a set of all of the keys that are in the Scenario data"""
-        return self._data_type_to_keys["scenario"]
+    def scenario_keys(self) -> list[str]:
+        """Return a set of all of the keys that are in the Scenario data.
+
+        >>> r = Results.example()
+        >>> r.scenario_keys
+        ['period', 'scenario_index']
+        """
+        return sorted(self._data_type_to_keys["scenario"])
 
     @property
     def question_names(self) -> list[str]:
-        """Returns a list of all of the question names"""
+        """Return a list of all of the question names.
+
+        Example:
+
+        >>> r = Results.example()
+        >>> r.question_names
+        ['how_feeling', 'how_feeling_yesterday']
+        """
         if self.survey is None:
             return []
-        return list(self.survey.question_names)
+        return sorted(list(self.survey.question_names))
 
     @property
-    def all_keys(self) -> set[str]:
-        """Returns a set of all of the keys that are in the Results"""
+    def all_keys(self) -> list[str]:
+        """Return a set of all of the keys that are in the Results.
+
+        Example:
+
+        >>> r = Results.example()
+        >>> r.all_keys
+        ['agent_index', ...]
+        """
         answer_keys = set(self.answer_keys)
-        return (
+        all_keys = (
             answer_keys.union(self.agent_keys)
             .union(self.scenario_keys)
             .union(self.model_keys)
         )
+        return sorted(list(all_keys))
+
+    def first(self) -> Result:
+        """Return the first observation in the results.
+
+        Example:
+
+        >>> r = Results.example()
+        >>> r.first()
+        Result(agent...
+        """
+        return self.data[0]
+
+    def answer_truncate(
+        self, column: str, top_n: int = 5, new_var_name: str = None
+    ) -> Results:
+        """Create a new variable that truncates the answers to the top_n.
+
+        :param column: The column to truncate.
+        :param top_n: The number of top answers to keep.
+        :param new_var_name: The name of the new variable. If None, it is the original name + '_truncated'.
 
-    def relevant_columns(self) -> set[str]:
-        """Returns all of the columns that are in the results."""
-        return set().union(
-            *(observation.combined_dict.keys() for observation in self.data)
+        Example:
+        >>> r = Results.example()
+        >>> r.answer_truncate('how_feeling', top_n = 2).select('how_feeling', 'how_feeling_truncated')
+        Dataset([{'answer.how_feeling': ['OK', 'Great', 'Terrible', 'OK']}, {'answer.how_feeling_truncated': ['Other', 'Other', 'Other', 'Other']}])
+
+
+        """
+        if new_var_name is None:
+            new_var_name = column + "_truncated"
+        answers = list(self.select(column).tally().keys())
+
+        def f(x):
+            if x in answers[:top_n]:
+                return x
+            else:
+                return "Other"
+
+        return self.recode(column, recode_function=f, new_var_name=new_var_name)
+
+    def recode(
+        self, column: str, recode_function: Optional[Callable], new_var_name=None
+    ) -> Results:
+        """
+        Recode a column in the Results object.
+
+        >>> r = Results.example()
+        >>> r.recode('how_feeling', recode_function = lambda x: 1 if x == 'Great' else 0).select('how_feeling', 'how_feeling_recoded')
+        Dataset([{'answer.how_feeling': ['OK', 'Great', 'Terrible', 'OK']}, {'answer.how_feeling_recoded': [0, 1, 0, 0]}])
+        """
+
+        if new_var_name is None:
+            new_var_name = column + "_recoded"
+        new_data = []
+        for result in self.data:
+            new_result = result.copy()
+            value = new_result.get_value("answer", column)
+            # breakpoint()
+            new_result["answer"][new_var_name] = recode_function(value)
+            new_data.append(new_result)
+
+        # print("Created new variable", new_var_name)
+        return Results(
+            survey=self.survey,
+            data=new_data,
+            created_columns=self.created_columns + [new_var_name],
         )
 
-    def _parse_column(self, column: str) -> tuple[str, str]:
+    def add_column(self, column_name: str, values: list) -> Results:
+        """Adds columns to Results
+
+        >>> r = Results.example()
+        >>> r.add_column('a', [1,2,3, 4]).select('a')
+        Dataset([{'answer.a': [1, 2, 3, 4]}])
         """
-        Parses a column name into a tuple containing a data type and a key.
-        - Uses the key_to_data_type property of the Results class.
-        - The standard way a column is specified is with a dot-separated string, e.g. _parse_column("agent.status")
-        - But you can also specify a single key, e.g. "status", in which case it will look up the data type.
+
+        assert len(values) == len(
+            self.data
+        ), "The number of values must match the number of results."
+        new_results = self.data.copy()
+        for i, result in enumerate(new_results):
+            result["answer"][column_name] = values[i]
+        return Results(
+            survey=self.survey,
+            data=new_results,
+            created_columns=self.created_columns + [column_name],
+        )
+
+    def add_columns_from_dict(self, columns: List[dict]) -> Results:
+        """Adds columns to Results from a list of dictionaries.
+
+        >>> r = Results.example()
+        >>> r.add_columns_from_dict([{'a': 1, 'b': 2}, {'a': 3, 'b': 4}, {'a':3, 'b':2}, {'a':3, 'b':2}]).select('a', 'b')
+        Dataset([{'answer.a': [1, 3, 3, 3]}, {'answer.b': [2, 4, 2, 2]}])
         """
-        if "." in column:
-            data_type, key = column.split(".")
-        else:
-            try:
-                data_type, key = self._key_to_data_type[column], column
-            except KeyError:
-                raise ResultsColumnNotFoundError(f"Column {column} not found in data")
-        return data_type, key
+        keys = list(columns[0].keys())
+        for key in keys:
+            values = [d[key] for d in columns]
+            self = self.add_column(key, values)
+        return self
+
+    @staticmethod
+    def _create_evaluator(
+        result: Result, functions_dict: Optional[dict] = None
+    ) -> "EvalWithCompoundTypes":
+        """Create an evaluator for the expression.
+
+        >>> from unittest.mock import Mock
+        >>> result = Mock()
+        >>> result.combined_dict = {'how_feeling': 'OK'}
+
+        >>> evaluator = Results._create_evaluator(result = result, functions_dict = {})
+        >>> evaluator.eval("how_feeling == 'OK'")
+        True
+
+        >>> result.combined_dict = {'answer': {'how_feeling': 'OK'}}
+        >>> evaluator = Results._create_evaluator(result = result, functions_dict = {})
+        >>> evaluator.eval("answer.how_feeling== 'OK'")
+        True
+
+        Note that you need to refer to the answer dictionary in the expression.
+
+        >>> evaluator.eval("how_feeling== 'OK'")
+        Traceback (most recent call last):
+        ...
+        simpleeval.NameNotDefined: 'how_feeling' is not defined for expression 'how_feeling== 'OK''
+        """
+        from simpleeval import EvalWithCompoundTypes
 
-    def first(self) -> Result:
-        """Returns the first observation in the results."""
-        return self.data[0]
+        if functions_dict is None:
+            functions_dict = {}
+        evaluator = EvalWithCompoundTypes(
+            names=result.combined_dict, functions=functions_dict
+        )
+        evaluator.functions.update(int=int, float=float)
+        return evaluator
 
-    def mutate(self, new_var_string: str, functions_dict: dict = None) -> Results:
+    def mutate(
+        self, new_var_string: str, functions_dict: Optional[dict] = None
+    ) -> Results:
         """
         Creates a value in the Results object as if has been asked as part of the survey.
-        - It splits the new_var_string at the "=" and uses simple_eval
-        - The functions dict is...
+
+        :param new_var_string: A string that is a valid Python expression.
+        :param functions_dict: A dictionary of functions that can be used in the expression. The keys are the function names and the values are the functions themselves.
+
+        It splits the new_var_string at the "=" and uses simple_eval
 
         Example:
-        >>> r = Results.create_example()
+
+        >>> r = Results.example()
         >>> r.mutate('how_feeling_x = how_feeling + "x"').select('how_feeling_x')
-        [{'answer.how_feeling_x': ['Badx', 'Badx', 'Greatx', 'Greatx']}]
+        Dataset([{'answer.how_feeling_x': ...
         """
         # extract the variable name and the expression
         if "=" not in new_var_string:
@@ -287,19 +853,16 @@ class Results(UserList, Mixins, Base):
             )
         raw_var_name, expression = new_var_string.split("=", 1)
         var_name = raw_var_name.strip()
+        from edsl.utilities.utilities import is_valid_variable_name
+
         if not is_valid_variable_name(var_name):
             raise ResultsInvalidNameError(f"{var_name} is not a valid variable name.")
 
         # create the evaluator
         functions_dict = functions_dict or {}
 
-        def create_evaluator(result: Result) -> EvalWithCompoundTypes:
-            return EvalWithCompoundTypes(
-                names=result.combined_dict, functions=functions_dict
-            )
-
-        def new_result(old_result: Result, var_name: str) -> Result:
-            evaluator = create_evaluator(old_result)
+        def new_result(old_result: "Result", var_name: str) -> "Result":
+            evaluator = self._create_evaluator(old_result, functions_dict)
             value = evaluator.eval(expression)
             new_result = old_result.copy()
             new_result["answer"][var_name] = value
@@ -316,83 +879,164 @@ class Results(UserList, Mixins, Base):
             created_columns=self.created_columns + [var_name],
         )
 
-    def select(self, *columns: Union[str, list[str]]) -> Dataset:
+    def add_column(self, column_name: str, values: list) -> Results:
+        """Adds columns to Results
+
+        >>> r = Results.example()
+        >>> r.add_column('a', [1,2,3, 4]).select('a')
+        Dataset([{'answer.a': [1, 2, 3, 4]}])
         """
-        This selects data from the results and turns it into a format like so:
-        >>> results = Results.create_example()
-        >>> results.select('how_feeling')
-        [{'answer.how_feeling': ['Bad', 'Bad', 'Great', 'Great']}]
+
+        assert len(values) == len(
+            self.data
+        ), "The number of values must match the number of results."
+        new_results = self.data.copy()
+        for i, result in enumerate(new_results):
+            result["answer"][column_name] = values[i]
+        return Results(
+            survey=self.survey,
+            data=new_results,
+            created_columns=self.created_columns + [column_name],
+        )
+
+    def rename(self, old_name: str, new_name: str) -> Results:
+        """Rename an answer column in a Results object.
+
+        >>> s = Results.example()
+        >>> s.rename('how_feeling', 'how_feeling_new').select('how_feeling_new')
+        Dataset([{'answer.how_feeling_new': ['OK', 'Great', 'Terrible', 'OK']}])
+
+        # TODO: Should we allow renaming of scenario fields as well? Probably.
+
         """
 
-        if not columns or columns == ("*",) or columns == (None,):
-            columns = ("*.*",)
+        for obs in self.data:
+            obs["answer"][new_name] = obs["answer"][old_name]
+            del obs["answer"][old_name]
 
-        if isinstance(columns[0], list):
-            columns = tuple(columns[0])
+        return self
 
-        known_data_types = [
-            "answer",
-            "scenario",
-            "agent",
-            "model",
-            "prompt",
-            "raw_model_response",
-        ]
+    def shuffle(self, seed: Optional[str] = "edsl") -> Results:
+        """Shuffle the results.
 
-        def get_data_types_to_return(parsed_data_type):
-            if parsed_data_type == "*":  # they want all of the columns
-                return known_data_types
-            else:
-                if parsed_data_type not in known_data_types:
-                    raise Exception(
-                        f"Data type {parsed_data_type} not found in data. Did you mean one of {known_data_types}"
-                    )
-                return [parsed_data_type]
+        Example:
 
-        # we're doing to populate this with the data we want to fetch
-        to_fetch = defaultdict(list)
+        >>> r = Results.example()
+        >>> r.shuffle(seed = 1)[0]
+        Result(...)
+        """
+        if seed != "edsl":
+            seed = random.seed(seed)
 
-        new_data = []
-        items_in_order = []
-        # iterate through the passed columns
-        for column in columns:
-            # a user could pass 'result.how_feeling' or just 'how_feeling'
-            parsed_data_type, parsed_key = self._parse_column(column)
-            data_types = get_data_types_to_return(parsed_data_type)
-            found_once = False  # we need to track this to make sure we found the key at least once
+        new_data = self.data.copy()
+        random.shuffle(new_data)
+        return Results(survey=self.survey, data=new_data, created_columns=None)
+
+    def sample(
+        self,
+        n: Optional[int] = None,
+        frac: Optional[float] = None,
+        with_replacement: bool = True,
+        seed: Optional[str] = None,
+    ) -> Results:
+        """Sample the results.
+
+        :param n: An integer representing the number of samples to take.
+        :param frac: A float representing the fraction of samples to take.
+        :param with_replacement: A boolean representing whether to sample with replacement.
+        :param seed: An integer representing the seed for the random number generator.
+
+        Example:
+
+        >>> r = Results.example()
+        >>> len(r.sample(2))
+        2
+        """
+        if seed:
+            random.seed(seed)
+
+        if n is None and frac is None:
+            raise Exception("You must specify either n or frac.")
+
+        if n is not None and frac is not None:
+            raise Exception("You cannot specify both n and frac.")
+
+        if frac is not None and n is None:
+            n = int(frac * len(self.data))
+
+        if with_replacement:
+            new_data = random.choices(self.data, k=n)
+        else:
+            new_data = random.sample(self.data, n)
+
+        return Results(survey=self.survey, data=new_data, created_columns=None)
+
+    def select(self, *columns: Union[str, list[str]]) -> Results:
+        """
+        Select data from the results and format it.
+
+        :param columns: A list of strings, each of which is a column name. The column name can be a single key, e.g. "how_feeling", or a dot-separated string, e.g. "answer.how_feeling".
+
+        Example:
+
+        >>> results = Results.example()
+        >>> results.select('how_feeling')
+        Dataset([{'answer.how_feeling': ['OK', 'Great', 'Terrible', 'OK']}])
+
+        >>> results.select('how_feeling', 'model', 'how_feeling')
+        Dataset([{'answer.how_feeling': ['OK', 'Great', 'Terrible', 'OK']}, {'answer.how_feeling': ['OK', 'Great', 'Terrible', 'OK']}, {'model.model': ['...', '...', '...', '...']}, {'answer.how_feeling': ['OK', 'Great', 'Terrible', 'OK']}, {'answer.how_feeling': ['OK', 'Great', 'Terrible', 'OK']}])
+
+        >>> from edsl import Results; r = Results.example(); r.select('answer.how_feeling_y')
+        Dataset([{'answer.how_feeling_yesterday': ['Great', 'Good', 'OK', 'Terrible']}])
+        """
 
-            for data_type in data_types:
-                # the keys for that data_type e.g.,# if data_type is 'answer', then the keys are 'how_feeling', 'how_feeling_comment', etc.
-                relevant_keys = self._data_type_to_keys[data_type]
+        from edsl.results.results_selector import Selector
 
-                for key in relevant_keys:
-                    if key == parsed_key or parsed_key == "*":
-                        found_once = True
-                        to_fetch[data_type].append(key)
-                        items_in_order.append(data_type + "." + key)
+        if len(self) == 0:
+            raise Exception("No data to select from---the Results object is empty.")
 
-            if not found_once:
-                raise Exception(f"Key {parsed_key} not found in data.")
+        selector = Selector(
+            known_data_types=self.known_data_types,
+            data_type_to_keys=self._data_type_to_keys,
+            key_to_data_type=self._key_to_data_type,
+            fetch_list_func=self._fetch_list,
+            columns=self.columns,
+        )
+        return selector.select(*columns)
+
+    def sort_by(self, *columns: str, reverse: bool = False) -> Results:
+        """Sort the results by one or more columns."""
+        import warnings
+
+        warnings.warn(
+            "sort_by is deprecated. Use order_by instead.", DeprecationWarning
+        )
+        return self.order_by(*columns, reverse=reverse)
+
+    def _parse_column(self, column: str) -> tuple[str, str]:
+        """Parse a column name into a data type and key."""
+        if "." in column:
+            return column.split(".")
+        return self._key_to_data_type[column], column
 
-        for data_type in to_fetch:
-            for key in to_fetch[data_type]:
-                entries = self._fetch_list(data_type, key)
-                new_data.append({data_type + "." + key: entries})
+    def order_by(self, *columns: str, reverse: bool = False) -> Results:
+        """Sort the results by one or more columns.
 
-        def sort_by_key_order(dictionary):
-            # Extract the single key from the dictionary
-            single_key = next(iter(dictionary))
-            # Return the index of this key in the list_of_keys
-            return items_in_order.index(single_key)
+        :param columns: One or more column names as strings.
+        :param reverse: A boolean that determines whether to sort in reverse order.
 
-        sorted(new_data, key=sort_by_key_order)
+        Each column name can be a single key, e.g. "how_feeling", or a dot-separated string, e.g. "answer.how_feeling".
 
-        return Dataset(new_data)
+        Example:
 
-    def sort_by(self, column, reverse=True) -> Results:
-        "Sorts the results by a column"
+        >>> r = Results.example()
+        >>> r.sort_by('how_feeling', reverse=False).select('how_feeling')
+        Dataset([{'answer.how_feeling': ['Great', 'OK', 'OK', 'Terrible']}])
 
-        data_type, key = self._parse_column(column)
+        >>> r.sort_by('how_feeling', reverse=True).select('how_feeling')
+        Dataset([{'answer.how_feeling': ['Terrible', 'OK', 'OK', 'Great']}])
+
+        """
 
         def to_numeric_if_possible(v):
             try:
@@ -400,66 +1044,144 @@ class Results(UserList, Mixins, Base):
             except:
                 return v
 
-        new_data = sorted(
-            self.data,
-            key=lambda x: to_numeric_if_possible(x.get_value(data_type, key)),
-            reverse=reverse,
-        )
+        def sort_key(item):
+            key_components = []
+            for col in columns:
+                data_type, key = self._parse_column(col)
+                value = item.get_value(data_type, key)
+                key_components.append(to_numeric_if_possible(value))
+            return tuple(key_components)
+
+        new_data = sorted(self.data, key=sort_key, reverse=reverse)
         return Results(survey=self.survey, data=new_data, created_columns=None)
 
-    def filter(self, expression) -> Results:
+    def filter(self, expression: str) -> Results:
         """
-        This filters a result based on the expression that is passed in.
-        >>> r = Results.create_example()
+        Filter based on the given expression and returns the filtered `Results`.
+
+        :param expression: A string expression that evaluates to a boolean. The expression is applied to each element in `Results` to determine whether it should be included in the filtered results.
+
+        The `expression` parameter is a string that must resolve to a boolean value when evaluated against each element in `Results`.
+        This expression is used to determine which elements to include in the returned `Results`.
+
+        Example usage: Create an example `Results` instance and apply filters to it:
+
+        >>> r = Results.example()
         >>> r.filter("how_feeling == 'Great'").select('how_feeling')
-        [{'answer.how_feeling': ['Great', 'Great']}]
-        >>> r.filter("how_feeling == 'Nothing'").select('how_feeling')
-        [{'answer.how_feeling': []}]
+        Dataset([{'answer.how_feeling': ['Great']}])
+
+        Example usage: Using an OR operator in the filter expression.
+
+        >>> r = Results.example().filter("how_feeling = 'Great'").select('how_feeling')
+        Traceback (most recent call last):
+        ...
+        edsl.exceptions.results.ResultsFilterError: You must use '==' instead of '=' in the filter expression.
+        ...
+
+        >>> r.filter("how_feeling == 'Great' or how_feeling == 'Terrible'").select('how_feeling')
+        Dataset([{'answer.how_feeling': ['Great', 'Terrible']}])
         """
 
-        def create_evaluator(result):
-            return EvalWithCompoundTypes(names=result.combined_dict)
+        def has_single_equals(string):
+            if "!=" in string:
+                return False
+            if "=" in string and not (
+                "==" in string or "<=" in string or ">=" in string
+            ):
+                return True
+
+        if has_single_equals(expression):
+            raise ResultsFilterError(
+                "You must use '==' instead of '=' in the filter expression."
+            )
 
         try:
-            new_data = [
-                result
-                for result in self.data
-                if create_evaluator(result).eval(expression)
-            ]
+            # iterates through all the results and evaluates the expression
+            new_data = []
+            for result in self.data:
+                evaluator = self._create_evaluator(result)
+                result.check_expression(expression)  # check expression
+                if evaluator.eval(expression):
+                    new_data.append(result)
+
+        except ValueError as e:
+            raise ResultsFilterError(
+                f"Error in filter. Exception:{e}",
+                f"The expression you provided was: {expression}",
+                "See https://docs.expectedparrot.com/en/latest/results.html#filtering-results for more details.",
+            )
         except Exception as e:
-            print(f"Exception:{e}")
+            raise ResultsFilterError(
+                f"""Error in filter. Exception:{e}.""",
+                f"""The expression you provided was: {expression}.""",
+                """Please make sure that the expression is a valid Python expression that evaluates to a boolean.""",
+                """For example, 'how_feeling == "Great"' is a valid expression, as is 'how_feeling in ["Great", "Terrible"]'., """,
+                """However, 'how_feeling = "Great"' is not a valid expression.""",
+                """See https://docs.expectedparrot.com/en/latest/results.html#filtering-results for more details.""",
+            )
+
+        if len(new_data) == 0:
+            import warnings
+
+            warnings.warn("No results remain after applying the filter.")
 
         return Results(survey=self.survey, data=new_data, created_columns=None)
 
     @classmethod
-    def example(cls, debug: bool = False) -> Results:
-        """
-        Returns an example Results object
-        - debug: if False, uses actual API calls
-        """
-        from edsl.jobs import Jobs
+    def example(cls, randomize: bool = False) -> Results:
+        """Return an example `Results` object.
+
+        Example usage:
+
+        >>> r = Results.example()
 
-        job = Jobs.example()
-        results = job.run()
+        :param debug: if False, uses actual API calls
+        """
+        from edsl.jobs.Jobs import Jobs
+        from edsl.data.Cache import Cache
+
+        c = Cache()
+        job = Jobs.example(randomize=randomize)
+        results = job.run(
+            cache=c,
+            stop_on_exception=True,
+            skip_retry=True,
+            raise_validation_errors=True,
+            disable_remote_cache=True,
+            disable_remote_inference=True,
+        )
         return results
 
     def rich_print(self):
-        """Displays an object as a table."""
-        with io.StringIO() as buf:
-            console = Console(file=buf, record=True)
+        """Display an object as a table."""
+        pass
 
-            for index, result in enumerate(self):
-                console.print(f"Result {index}")
-                console.print(result.rich_print())
+    def __str__(self):
+        data = self.to_dict()["data"]
+        return json.dumps(data, indent=4)
 
-            return console.export_text()
+    def show_exceptions(self, traceback=False):
+        """Print the exceptions."""
+        if hasattr(self, "task_history"):
+            self.task_history.show_exceptions(traceback)
+        else:
+            print("No exceptions to show.")
 
-    def __str__(self):
-        return self.rich_print()
+    def score(self, f: Callable) -> list:
+        """Score the results using in a function.
+
+        :param f: A function that takes values from a Resul object and returns a score.
+
+        >>> r = Results.example()
+        >>> def f(status): return 1 if status == 'Joyful' else 0
+        >>> r.score(f)
+        [1, 1, 0, 0]
+        """
+        return [r.score(f) for r in self.data]
 
 
 def main():  # pragma: no cover
-    """Calls the OpenAI API credits"""
+    """Call the OpenAI API credits."""
     from edsl.results.Results import Results
 
     results = Results.example(debug=True)
@@ -468,10 +1190,6 @@ def main():  # pragma: no cover
 
 
 if __name__ == "__main__":
-    print(Results.example())
-
-    r = Results.example()
-    # db_row = list(r[0].rows(1))
-    db_rows = list(r.rows())
+    import doctest
 
-    print(r.columns)
+    doctest.testmod(optionflags=doctest.ELLIPSIS)