edsl 0.1.39.dev1__py3-none-any.whl → 0.1.39.dev2__py3-none-any.whl

edsl/scenarios/ScenarioList.py

@@ -1,33 +1,78 @@
 """A list of Scenarios to be used in a survey."""
 
 from __future__ import annotations
-from typing import Any, Optional, Union, List, Callable
+from typing import (
+    Any,
+    Optional,
+    Union,
+    List,
+    Callable,
+    Literal,
+    TYPE_CHECKING,
+)
+
+try:
+    from typing import TypeAlias
+except ImportError:
+    from typing_extensions import TypeAlias
+
 import csv
 import random
-from collections import UserList, Counter
-from collections.abc import Iterable
-import urllib.parse
-import urllib.request
 from io import StringIO
-from collections import defaultdict
 import inspect
+from collections import UserList, defaultdict
+from collections.abc import Iterable
+
+if TYPE_CHECKING:
+    from urllib.parse import ParseResult
+    from edsl.results.Dataset import Dataset
+    from edsl.jobs.Jobs import Jobs
+    from edsl.surveys.Survey import Survey
+    from edsl.questions.QuestionBase import QuestionBase
 
-from simpleeval import EvalWithCompoundTypes
+
+from simpleeval import EvalWithCompoundTypes, NameNotDefined  # type: ignore
+
+from tabulate import tabulate_formats
 
 from edsl.Base import Base
-from edsl.utilities.decorators import add_edsl_version, remove_edsl_version
+from edsl.utilities.remove_edsl_version import remove_edsl_version
+
 from edsl.scenarios.Scenario import Scenario
 from edsl.scenarios.ScenarioListPdfMixin import ScenarioListPdfMixin
 from edsl.scenarios.ScenarioListExportMixin import ScenarioListExportMixin
-
 from edsl.utilities.naming_utilities import sanitize_string
-from edsl.utilities.utilities import is_valid_variable_name
+from edsl.utilities.is_valid_variable_name import is_valid_variable_name
+from edsl.exceptions.scenarios import ScenarioError
+
+from edsl.scenarios.DirectoryScanner import DirectoryScanner
 
 
 class ScenarioListMixin(ScenarioListPdfMixin, ScenarioListExportMixin):
     pass
 
 
+if TYPE_CHECKING:
+    from edsl.results.Dataset import Dataset
+
+TableFormat: TypeAlias = Literal[
+    "plain",
+    "simple",
+    "github",
+    "grid",
+    "fancy_grid",
+    "pipe",
+    "orgtbl",
+    "rst",
+    "mediawiki",
+    "html",
+    "latex",
+    "latex_raw",
+    "latex_booktabs",
+    "tsv",
+]
+
+
 class ScenarioList(Base, UserList, ScenarioListMixin):
     """Class for creating a list of scenarios to be used in a survey."""
 
@@ -35,7 +80,9 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
         "https://docs.expectedparrot.com/en/latest/scenarios.html#scenariolist"
     )
 
-    def __init__(self, data: Optional[list] = None, codebook: Optional[dict] = None):
+    def __init__(
+        self, data: Optional[list] = None, codebook: Optional[dict[str, str]] = None
+    ):
         """Initialize the ScenarioList class."""
         if data is not None:
             super().__init__(data)
@@ -57,12 +104,19 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
         """Check if the ScenarioList has Jinja braces."""
         return any([scenario.has_jinja_braces for scenario in self])
 
-    def convert_jinja_braces(self) -> ScenarioList:
+    def _convert_jinja_braces(self) -> ScenarioList:
         """Convert Jinja braces to Python braces."""
-        return ScenarioList([scenario.convert_jinja_braces() for scenario in self])
+        return ScenarioList([scenario._convert_jinja_braces() for scenario in self])
 
-    def give_valid_names(self) -> ScenarioList:
-        """Give valid names to the scenario keys.
+    def give_valid_names(self, existing_codebook: dict = None) -> ScenarioList:
+        """Give valid names to the scenario keys, using an existing codebook if provided.
+
+        Args:
+            existing_codebook (dict, optional): Existing mapping of original keys to valid names.
+                Defaults to None.
+
+        Returns:
+            ScenarioList: A new ScenarioList with valid variable names and updated codebook.
 
         >>> s = ScenarioList([Scenario({'a': 1, 'b': 2}), Scenario({'a': 1, 'b': 1})])
         >>> s.give_valid_names()
@@ -70,27 +124,38 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
         >>> s = ScenarioList([Scenario({'are you there John?': 1, 'b': 2}), Scenario({'a': 1, 'b': 1})])
         >>> s.give_valid_names()
         ScenarioList([Scenario({'john': 1, 'b': 2}), Scenario({'a': 1, 'b': 1})])
+        >>> s.give_valid_names({'are you there John?': 'custom_name'})
+        ScenarioList([Scenario({'custom_name': 1, 'b': 2}), Scenario({'a': 1, 'b': 1})])
         """
-        codebook = {}
-        new_scenaerios = []
+        codebook = existing_codebook.copy() if existing_codebook else {}
+        new_scenarios = []
+
         for scenario in self:
             new_scenario = {}
             for key in scenario:
-                if not is_valid_variable_name(key):
-                    if key in codebook:
-                        new_key = codebook[key]
-                    else:
-                        new_key = sanitize_string(key)
-                        if not is_valid_variable_name(new_key):
-                            new_key = f"var_{len(codebook)}"
-                        codebook[key] = new_key
-                    new_scenario[new_key] = scenario[key]
-                else:
+                if is_valid_variable_name(key):
                     new_scenario[key] = scenario[key]
-            new_scenaerios.append(Scenario(new_scenario))
-        return ScenarioList(new_scenaerios, codebook)
+                    continue
+
+                if key in codebook:
+                    new_key = codebook[key]
+                else:
+                    new_key = sanitize_string(key)
+                    if not is_valid_variable_name(new_key):
+                        new_key = f"var_{len(codebook)}"
+                    codebook[key] = new_key
+
+                new_scenario[new_key] = scenario[key]
+
+            new_scenarios.append(Scenario(new_scenario))
+
+        return ScenarioList(new_scenarios, codebook)
 
-    def unpivot(self, id_vars=None, value_vars=None):
+    def unpivot(
+        self,
+        id_vars: Optional[List[str]] = None,
+        value_vars: Optional[List[str]] = None,
+    ) -> ScenarioList:
         """
         Unpivot the ScenarioList, allowing for id variables to be specified.
 
@@ -121,7 +186,40 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
 
         return ScenarioList(new_scenarios)
 
-    def pivot(self, id_vars, var_name="variable", value_name="value"):
+    def sem_filter(self, language_predicate: str) -> ScenarioList:
+        """Filter the ScenarioList based on a language predicate.
+
+        :param language_predicate: The language predicate to use.
+
+        Inspired by:
+        @misc{patel2024semanticoperators,
+            title={Semantic Operators: A Declarative Model for Rich, AI-based Analytics Over Text Data},
+            author={Liana Patel and Siddharth Jha and Parth Asawa and Melissa Pan and Carlos Guestrin and Matei Zaharia},
+            year={2024},
+            eprint={2407.11418},
+            archivePrefix={arXiv},
+            primaryClass={cs.DB},
+            url={https://arxiv.org/abs/2407.11418},
+            }
+        """
+        from edsl import QuestionYesNo
+
+        new_scenario_list = self.duplicate()
+        q = QuestionYesNo(
+            question_text=language_predicate, question_name="binary_outcome"
+        )
+        results = q.by(new_scenario_list).run(verbose=False)
+        new_scenario_list = new_scenario_list.add_list(
+            "criteria", results.select("binary_outcome").to_list()
+        )
+        return new_scenario_list.filter("criteria == 'Yes'").drop("criteria")
+
+    def pivot(
+        self,
+        id_vars: List[str] = None,
+        var_name="variable",
+        value_name="value",
+    ) -> ScenarioList:
         """
         Pivot the ScenarioList from long to wide format.
 
@@ -163,15 +261,15 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
 
         return ScenarioList(pivoted_scenarios)
 
-    def group_by(self, id_vars, variables, func):
+    def group_by(
+        self, id_vars: List[str], variables: List[str], func: Callable
+    ) -> ScenarioList:
         """
         Group the ScenarioList by id_vars and apply a function to the specified variables.
 
-        Parameters:
-        id_vars (list): Fields to use as identifier variables for grouping
-        variables (list): Fields to pass to the aggregation function
-        func (callable): Function to apply to the grouped variables.
-                        Should accept lists of values for each variable.
+        :param id_vars: Fields to use as identifier variables
+        :param variables: Fields to group and aggregate
+        :param func: Function to apply to the grouped variables
 
         Returns:
         ScenarioList: A new ScenarioList with the grouped and aggregated results
@@ -191,12 +289,12 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
         # Check if the function is compatible with the specified variables
         func_params = inspect.signature(func).parameters
         if len(func_params) != len(variables):
-            raise ValueError(
+            raise ScenarioError(
                 f"Function {func.__name__} expects {len(func_params)} arguments, but {len(variables)} variables were provided"
             )
 
         # Group the scenarios
-        grouped = defaultdict(lambda: defaultdict(list))
+        grouped: dict[str, list] = defaultdict(lambda: defaultdict(list))
         for scenario in self:
             key = tuple(scenario[id_var] for id_var in id_vars)
             for var in variables:
@@ -208,10 +306,12 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
             try:
                 aggregated = func(*[group[var] for var in variables])
             except Exception as e:
-                raise ValueError(f"Error applying function to group {key}: {str(e)}")
+                raise ScenarioError(f"Error applying function to group {key}: {str(e)}")
 
             if not isinstance(aggregated, dict):
-                raise ValueError(f"Function {func.__name__} must return a dictionary")
+                raise ScenarioError(
+                    f"Function {func.__name__} must return a dictionary"
+                )
 
             new_scenario = dict(zip(id_vars, key))
             new_scenario.update(aggregated)
@@ -278,50 +378,18 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
         """
         return self.__mul__(other)
 
-    def shuffle(self, seed: Optional[str] = "edsl") -> ScenarioList:
+    def shuffle(self, seed: Optional[str] = None) -> ScenarioList:
         """Shuffle the ScenarioList.
 
         >>> s = ScenarioList.from_list("a", [1,2,3,4])
-        >>> s.shuffle()
-        ScenarioList([Scenario({'a': 3}), Scenario({'a': 4}), Scenario({'a': 1}), Scenario({'a': 2})])
+        >>> s.shuffle(seed = "1234")
+        ScenarioList([Scenario({'a': 1}), Scenario({'a': 4}), Scenario({'a': 3}), Scenario({'a': 2})])
         """
-        random.seed(seed)
-        random.shuffle(self.data)
-        return self
-
-    def _repr_html_(self):
-        """Return an HTML representation of the AgentList."""
-        # return (
-        #     str(self.summary(format="html")) + "<br>" + str(self.table(tablefmt="html"))
-        # )
-        footer = f"<a href={self.__documentation__}>(docs)</a>"
-        return str(self.summary(format="html")) + footer
-
-    # def _repr_html_(self) -> str:
-    # from edsl.utilities.utilities import data_to_html
-
-    # data = self.to_dict()
-    # _ = data.pop("edsl_version")
-    # _ = data.pop("edsl_class_name")
-    # for s in data["scenarios"]:
-    #     _ = s.pop("edsl_version")
-    #     _ = s.pop("edsl_class_name")
-    # for scenario in data["scenarios"]:
-    #     for key, value in scenario.items():
-    #         if hasattr(value, "to_dict"):
-    #             data[key] = value.to_dict()
-    # return data_to_html(data)
-
-    # def tally(self, field) -> dict:
-    #     """Return a tally of the values in the field.
-
-    #     Example:
-
-    #     >>> s = ScenarioList([Scenario({'a': 1, 'b': 1}), Scenario({'a': 1, 'b': 2})])
-    #     >>> s.tally('b')
-    #     {1: 1, 2: 1}
-    #     """
-    #     return dict(Counter([scenario[field] for scenario in self]))
+        sl = self.duplicate()
+        if seed:
+            random.seed(seed)
+        random.shuffle(sl.data)
+        return sl
 
     def sample(self, n: int, seed: Optional[str] = None) -> ScenarioList:
         """Return a random sample from the ScenarioList
@@ -333,16 +401,22 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
         if seed:
             random.seed(seed)
 
-        return ScenarioList(random.sample(self.data, n))
+        sl = self.duplicate()
+        return ScenarioList(random.sample(sl.data, n))
 
-    def expand(self, expand_field: str, number_field=False) -> ScenarioList:
+    def expand(self, expand_field: str, number_field: bool = False) -> ScenarioList:
         """Expand the ScenarioList by a field.
 
+        :param expand_field: The field to expand.
+        :param number_field: Whether to add a field with the index of the value
+
         Example:
 
         >>> s = ScenarioList( [ Scenario({'a':1, 'b':[1,2]}) ] )
         >>> s.expand('b')
         ScenarioList([Scenario({'a': 1, 'b': 1}), Scenario({'a': 1, 'b': 2})])
+        >>> s.expand('b', number_field=True)
+        ScenarioList([Scenario({'a': 1, 'b': 1, 'b_number': 1}), Scenario({'a': 1, 'b': 2, 'b_number': 2})])
         """
         new_scenarios = []
         for scenario in self:
@@ -357,12 +431,11 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
                 new_scenarios.append(new_scenario)
         return ScenarioList(new_scenarios)
 
-    def concatenate(self, fields: List[str], separator: str = ";") -> "ScenarioList":
+    def concatenate(self, fields: List[str], separator: str = ";") -> ScenarioList:
         """Concatenate specified fields into a single field.
 
-        Args:
-            fields (List[str]): List of field names to concatenate.
-            separator (str, optional): Separator to use between field values. Defaults to ";".
+        :param fields: The fields to concatenate.
+        :param separator: The separator to use.
 
         Returns:
             ScenarioList: A new ScenarioList with concatenated fields.
@@ -392,11 +465,17 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
     ) -> ScenarioList:
         """Unpack a dictionary field into separate fields.
 
+        :param field: The field to unpack.
+        :param prefix: An optional prefix to add to the new fields.
+        :param drop_field: Whether to drop the original field.
+
         Example:
 
         >>> s = ScenarioList([Scenario({'a': 1, 'b': {'c': 2, 'd': 3}})])
         >>> s.unpack_dict('b')
         ScenarioList([Scenario({'a': 1, 'b': {'c': 2, 'd': 3}, 'c': 2, 'd': 3})])
+        >>> s.unpack_dict('b', prefix='new_')
+        ScenarioList([Scenario({'a': 1, 'b': {'c': 2, 'd': 3}, 'new_c': 2, 'new_d': 3})])
         """
         new_scenarios = []
         for scenario in self:
@@ -414,7 +493,17 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
     def transform(
         self, field: str, func: Callable, new_name: Optional[str] = None
     ) -> ScenarioList:
-        """Transform a field using a function."""
+        """Transform a field using a function.
+
+        :param field: The field to transform.
+        :param func: The function to apply to the field.
+        :param new_name: An optional new name for the transformed field.
+
+        >>> s = ScenarioList([Scenario({'a': 1, 'b': 2}), Scenario({'a': 1, 'b': 1})])
+        >>> s.transform('b', lambda x: x + 1)
+        ScenarioList([Scenario({'a': 1, 'b': 3}), Scenario({'a': 1, 'b': 2})])
+
+        """
         new_scenarios = []
         for scenario in self:
             new_scenario = scenario.copy()
@@ -428,6 +517,9 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
         """
         Return a new ScenarioList with a new variable added.
 
+        :param new_var_string: A string with the new variable assignment.
+        :param functions_dict: A dictionary of functions to use in the assignment.
+
         Example:
 
         >>> s = ScenarioList([Scenario({'a': 1, 'b': 2}), Scenario({'a': 1, 'b': 1})])
@@ -436,7 +528,7 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
 
         """
         if "=" not in new_var_string:
-            raise Exception(
+            raise ScenarioError(
                 f"Mutate requires an '=' in the string, but '{new_var_string}' doesn't have one."
             )
         raw_var_name, expression = new_var_string.split("=", 1)
@@ -444,7 +536,7 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
         from edsl.utilities.utilities import is_valid_variable_name
 
         if not is_valid_variable_name(var_name):
-            raise Exception(f"{var_name} is not a valid variable name.")
+            raise ScenarioError(f"{var_name} is not a valid variable name.")
 
         # create the evaluator
         functions_dict = functions_dict or {}
@@ -462,13 +554,15 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
         try:
             new_data = [new_scenario(s, var_name) for s in self]
         except Exception as e:
-            raise Exception(f"Error in mutate. Exception:{e}")
+            raise ScenarioError(f"Error in mutate. Exception:{e}")
 
         return ScenarioList(new_data)
 
     def order_by(self, *fields: str, reverse: bool = False) -> ScenarioList:
         """Order the scenarios by one or more fields.
 
+        :param fields: The fields to order by.
+        :param reverse: Whether to reverse the order.
         Example:
 
         >>> s = ScenarioList([Scenario({'a': 1, 'b': 2}), Scenario({'a': 1, 'b': 1})])
@@ -481,16 +575,41 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
 
         return ScenarioList(sorted(self, key=get_sort_key, reverse=reverse))
 
+    def duplicate(self) -> ScenarioList:
+        """Return a copy of the ScenarioList.
+
+        >>> sl = ScenarioList.example()
+        >>> sl_copy = sl.duplicate()
+        >>> sl == sl_copy
+        True
+        >>> sl is sl_copy
+        False
+        """
+        return ScenarioList([scenario.copy() for scenario in self])
+
     def filter(self, expression: str) -> ScenarioList:
         """
         Filter a list of scenarios based on an expression.
 
+        :param expression: The expression to filter by.
+
         Example:
 
         >>> s = ScenarioList([Scenario({'a': 1, 'b': 1}), Scenario({'a': 1, 'b': 2})])
         >>> s.filter("b == 2")
         ScenarioList([Scenario({'a': 1, 'b': 2})])
         """
+        sl = self.duplicate()
+        base_keys = set(self[0].keys())
+        keys = set()
+        for scenario in sl:
+            keys.update(scenario.keys())
+        if keys != base_keys:
+            import warnings
+
+            warnings.warn(
+                "Ragged ScenarioList detected (different keys for different scenario entries). This may cause unexpected behavior."
+            )
 
         def create_evaluator(scenario: Scenario):
             """Create an evaluator for the given result.
@@ -500,14 +619,22 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
 
         try:
             # iterates through all the results and evaluates the expression
-            new_data = [
-                scenario
-                for scenario in self.data
-                if create_evaluator(scenario).eval(expression)
-            ]
+            new_data = []
+            for scenario in sl:
+                if create_evaluator(scenario).eval(expression):
+                    new_data.append(scenario)
+        except NameNotDefined as e:
+            available_fields = ", ".join(self.data[0].keys() if self.data else [])
+            raise ScenarioError(
+                f"Error in filter: '{e}'\n"
+                f"The expression '{expression}' refers to a field that does not exist.\n"
+                f"Scenario: {scenario}\n"
+                f"Available fields: {available_fields}\n"
+                "Check your filter expression or consult the documentation: "
+                "https://docs.expectedparrot.com/en/latest/scenarios.html#module-edsl.scenarios.Scenario"
+            ) from None
         except Exception as e:
-            print(f"Exception:{e}")
-            raise Exception(f"Error in filter. Exception:{e}")
+            raise ScenarioError(f"Error in filter. Exception:{e}")
 
         return ScenarioList(new_data)
 
@@ -519,30 +646,26 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
         :param urls: A list of URLs.
         :param field_name: The name of the field to store the text from the URLs.
 
-
         """
         return ScenarioList([Scenario.from_url(url, field_name) for url in urls])
 
-    def select(self, *fields) -> ScenarioList:
+    def select(self, *fields: str) -> ScenarioList:
         """
         Selects scenarios with only the references fields.
 
+        :param fields: The fields to select.
+
         Example:
 
         >>> s = ScenarioList([Scenario({'a': 1, 'b': 1}), Scenario({'a': 1, 'b': 2})])
         >>> s.select('a')
         ScenarioList([Scenario({'a': 1}), Scenario({'a': 1})])
         """
-        if len(fields) == 1:
-            fields_to_select = [list(fields)[0]]
-        else:
-            fields_to_select = list(fields)
+        from edsl.scenarios.ScenarioSelector import ScenarioSelector
 
-        return ScenarioList(
-            [scenario.select(fields_to_select) for scenario in self.data]
-        )
+        return ScenarioSelector(self).select(*fields)
 
-    def drop(self, *fields) -> ScenarioList:
+    def drop(self, *fields: str) -> ScenarioList:
         """Drop fields from the scenarios.
 
         Example:
@@ -551,18 +674,22 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
         >>> s.drop('a')
         ScenarioList([Scenario({'b': 1}), Scenario({'b': 2})])
         """
-        return ScenarioList([scenario.drop(fields) for scenario in self.data])
+        sl = self.duplicate()
+        return ScenarioList([scenario.drop(fields) for scenario in sl])
 
-    def keep(self, *fields) -> ScenarioList:
+    def keep(self, *fields: str) -> ScenarioList:
         """Keep only the specified fields in the scenarios.
 
+        :param fields: The fields to keep.
+
         Example:
 
         >>> s = ScenarioList([Scenario({'a': 1, 'b': 1}), Scenario({'a': 1, 'b': 2})])
         >>> s.keep('a')
         ScenarioList([Scenario({'a': 1}), Scenario({'a': 1})])
         """
-        return ScenarioList([scenario.keep(fields) for scenario in self.data])
+        sl = self.duplicate()
+        return ScenarioList([scenario.keep(fields) for scenario in sl])
 
     @classmethod
     def from_list(
@@ -570,6 +697,10 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
     ) -> ScenarioList:
         """Create a ScenarioList from a list of values.
 
+        :param name: The name of the field.
+        :param values: The list of values.
+        :param func: An optional function to apply to the values.
+
         Example:
 
         >>> ScenarioList.from_list('name', ['Alice', 'Bob'])
@@ -579,7 +710,12 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
             func = lambda x: x
         return cls([Scenario({name: func(value)}) for value in values])
 
-    def table(self, *fields, tablefmt=None, pretty_labels=None) -> str:
+    def table(
+        self,
+        *fields: str,
+        tablefmt: Optional[TableFormat] = None,
+        pretty_labels: Optional[dict[str, str]] = None,
+    ) -> str:
         """Return the ScenarioList as a table."""
 
         from tabulate import tabulate_formats
@@ -594,26 +730,41 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
         )
 
     def tree(self, node_list: Optional[List[str]] = None) -> str:
-        """Return the ScenarioList as a tree."""
+        """Return the ScenarioList as a tree.
+
+        :param node_list: The list of nodes to include in the tree.
+        """
         return self.to_dataset().tree(node_list)
 
-    def _summary(self):
+    def _summary(self) -> dict:
+        """Return a summary of the ScenarioList.
+
+        >>> ScenarioList.example()._summary()
+        {'scenarios': 2, 'keys': ['persona']}
+        """
         d = {
-            "EDSL Class name": "ScenarioList",
-            "# Scenarios": len(self),
-            "Scenario Keys": list(self.parameters),
+            "scenarios": len(self),
+            "keys": list(self.parameters),
         }
         return d
 
-    def reorder_keys(self, new_order):
+    def reorder_keys(self, new_order: List[str]) -> ScenarioList:
         """Reorder the keys in the scenarios.
 
+        :param new_order: The new order of the keys.
+
         Example:
 
         >>> s = ScenarioList([Scenario({'a': 1, 'b': 2}), Scenario({'a': 3, 'b': 4})])
         >>> s.reorder_keys(['b', 'a'])
         ScenarioList([Scenario({'b': 2, 'a': 1}), Scenario({'b': 4, 'a': 3})])
+        >>> s.reorder_keys(['a', 'b', 'c'])
+        Traceback (most recent call last):
+        ...
+        AssertionError
         """
+        assert set(new_order) == set(self.parameters)
+
         new_scenarios = []
         for scenario in self:
             new_scenario = Scenario({key: scenario[key] for key in new_order})
@@ -622,6 +773,8 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
 
     def to_dataset(self) -> "Dataset":
         """
+        Convert the ScenarioList to a Dataset.
+
         >>> s = ScenarioList.from_list("a", [1,2,3])
         >>> s.to_dataset()
         Dataset([{'a': [1, 2, 3]}])
@@ -631,8 +784,14 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
         """
         from edsl.results.Dataset import Dataset
 
-        keys = self[0].keys()
-        data = [{key: [scenario[key] for scenario in self.data]} for key in keys]
+        keys = list(self[0].keys())
+        for scenario in self:
+            new_keys = list(scenario.keys())
+            if new_keys != keys:
+                keys = list(set(keys + new_keys))
+        data = [
+            {key: [scenario.get(key, None) for scenario in self.data]} for key in keys
+        ]
         return Dataset(data)
 
     def unpack(
@@ -664,7 +823,14 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
             new_scenarios.append(new_scenario)
         return ScenarioList(new_scenarios)
 
-    def add_list(self, name, values) -> ScenarioList:
+    @classmethod
+    def from_list_of_tuples(self, *names: str, values: List[Tuple]) -> ScenarioList:
+        sl = ScenarioList.from_list(names[0], [value[0] for value in values])
+        for index, name in enumerate(names[1:]):
+            sl = sl.add_list(name, [value[index + 1] for value in values])
+        return sl
+
+    def add_list(self, name: str, values: List[Any]) -> ScenarioList:
         """Add a list of values to a ScenarioList.
 
         Example:
@@ -673,12 +839,10 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
         >>> s.add_list('age', [30, 25])
         ScenarioList([Scenario({'name': 'Alice', 'age': 30}), Scenario({'name': 'Bob', 'age': 25})])
         """
+        sl = self.duplicate()
         for i, value in enumerate(values):
-            if i < len(self):
-                self[i][name] = value
-            else:
-                self.append(Scenario({name: value}))
-        return self
+            sl[i][name] = value
+        return sl
 
     def add_value(self, name: str, value: Any) -> ScenarioList:
         """Add a value to all scenarios in a ScenarioList.
@@ -689,13 +853,16 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
         >>> s.add_value('age', 30)
         ScenarioList([Scenario({'name': 'Alice', 'age': 30}), Scenario({'name': 'Bob', 'age': 30})])
         """
-        for scenario in self:
+        sl = self.duplicate()
+        for scenario in sl:
             scenario[name] = value
-        return self
+        return sl
 
     def rename(self, replacement_dict: dict) -> ScenarioList:
         """Rename the fields in the scenarios.
 
+        :param replacement_dict: A dictionary with the old names as keys and the new names as values.
+
         Example:
 
         >>> s = ScenarioList([Scenario({'name': 'Alice', 'age': 30}), Scenario({'name': 'Bob', 'age': 25})])
@@ -710,8 +877,26 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
             new_list.append(new_obj)
         return new_list
 
+    ## NEEDS TO BE FIXED
+    # def new_column_names(self, new_names: List[str]) -> ScenarioList:
+    #     """Rename the fields in the scenarios.
+
+    #     Example:
+
+    #     >>> s = ScenarioList([Scenario({'name': 'Alice', 'age': 30}), Scenario({'name': 'Bob', 'age': 25})])
+    #     >>> s.new_column_names(['first_name', 'years'])
+    #     ScenarioList([Scenario({'first_name': 'Alice', 'years': 30}), Scenario({'first_name': 'Bob', 'years': 25})])
+
+    #     """
+    #     new_list = ScenarioList([])
+    #     for obj in self:
+    #         new_obj = obj.new_column_names(new_names)
+    #         new_list.append(new_obj)
+    #     return new_list
+
     @classmethod
     def from_sqlite(cls, filepath: str, table: str):
+        """Create a ScenarioList from a SQLite database."""
         import sqlite3
 
         with sqlite3.connect(filepath) as conn:
@@ -855,6 +1040,9 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
     def to_key_value(self, field: str, value=None) -> Union[dict, set]:
         """Return the set of values in the field.
 
+        :param field: The field to extract values from.
+        :param value: An optional field to use as the value in the key-value pair.
+
         Example:
 
         >>> s = ScenarioList([Scenario({'name': 'Alice'}), Scenario({'name': 'Bob'})])
@@ -974,56 +1162,42 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
 
     @classmethod
     def from_delimited_file(
-        cls, source: Union[str, urllib.parse.ParseResult], delimiter: str = ","
+        cls, source: Union[str, "ParseResult"], delimiter: str = ","
     ) -> ScenarioList:
-        """Create a ScenarioList from a delimited file (CSV/TSV) or URL.
-
-        Args:
-            source: A string representing either a local file path or a URL to a delimited file,
-                    or a urllib.parse.ParseResult object for a URL.
-            delimiter: The delimiter used in the file. Defaults to ',' for CSV files.
-                    Use '\t' for TSV files.
-
-        Returns:
-            ScenarioList: A ScenarioList object containing the data from the file.
-
-        Example:
-            # For CSV files
-
-            >>> with open('data.csv', 'w') as f:
-            ...     _ = f.write('name,age\\nAlice,30\\nBob,25\\n')
-            >>> scenario_list = ScenarioList.from_delimited_file('data.csv')
-
-            # For TSV files
-            >>> with open('data.tsv', 'w') as f:
-            ...     _ = f.write('name\\tage\\nAlice\t30\\nBob\t25\\n')
-            >>> scenario_list = ScenarioList.from_delimited_file('data.tsv', delimiter='\\t')
-
-        """
+        """Create a ScenarioList from a delimited file (CSV/TSV) or URL."""
+        import requests
         from edsl.scenarios.Scenario import Scenario
+        from urllib.parse import urlparse
+        from urllib.parse import ParseResult
+
+        headers = {
+            "Accept": "text/csv,application/csv,text/plain",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
+        }
 
         def is_url(source):
             try:
-                result = urllib.parse.urlparse(source)
+                result = urlparse(source)
                 return all([result.scheme, result.netloc])
             except ValueError:
                 return False
 
-        if isinstance(source, str) and is_url(source):
-            with urllib.request.urlopen(source) as response:
-                file_content = response.read().decode("utf-8")
-            file_obj = StringIO(file_content)
-        elif isinstance(source, urllib.parse.ParseResult):
-            with urllib.request.urlopen(source.geturl()) as response:
-                file_content = response.read().decode("utf-8")
-            file_obj = StringIO(file_content)
-        else:
-            file_obj = open(source, "r")
-
         try:
+            if isinstance(source, str) and is_url(source):
+                response = requests.get(source, headers=headers)
+                response.raise_for_status()
+                file_obj = StringIO(response.text)
+            elif isinstance(source, ParseResult):
+                response = requests.get(source.geturl(), headers=headers)
+                response.raise_for_status()
+                file_obj = StringIO(response.text)
+            else:
+                file_obj = open(source, "r")
+
             reader = csv.reader(file_obj, delimiter=delimiter)
             header = next(reader)
             observations = [Scenario(dict(zip(header, row))) for row in reader]
+
         finally:
             file_obj.close()
 
@@ -1031,7 +1205,7 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
 
     # Convenience methods for specific file types
     @classmethod
-    def from_csv(cls, source: Union[str, urllib.parse.ParseResult]) -> ScenarioList:
+    def from_csv(cls, source: Union[str, "ParseResult"]) -> ScenarioList:
         """Create a ScenarioList from a CSV file or URL."""
         return cls.from_delimited_file(source, delimiter=",")
 
@@ -1052,71 +1226,13 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
 
         sj = ScenarioJoin(self, other)
         return sj.left_join(by)
-        # # Validate join keys
-        # if not by:
-        #     raise ValueError(
-        #         "Join keys cannot be empty. Please specify at least one key to join on."
-        #     )
-
-        # # Convert single string to list for consistent handling
-        # by_keys = [by] if isinstance(by, str) else by
-
-        # # Verify all join keys exist in both ScenarioLists
-        # left_keys = set(next(iter(self)).keys()) if self else set()
-        # right_keys = set(next(iter(other)).keys()) if other else set()
-
-        # missing_left = set(by_keys) - left_keys
-        # missing_right = set(by_keys) - right_keys
-        # if missing_left or missing_right:
-        #     missing = missing_left | missing_right
-        #     raise ValueError(f"Join key(s) {missing} not found in both ScenarioLists")
-
-        # # Create lookup dictionary from the other ScenarioList
-        # def get_key_tuple(scenario: Scenario, keys: list[str]) -> tuple:
-        #     return tuple(scenario[k] for k in keys)
-
-        # other_dict = {get_key_tuple(scenario, by_keys): scenario for scenario in other}
-
-        # # Collect all possible keys (like SQL combining all columns)
-        # all_keys = set()
-        # for scenario in self:
-        #     all_keys.update(scenario.keys())
-        # for scenario in other:
-        #     all_keys.update(scenario.keys())
-
-        # new_scenarios = []
-        # for scenario in self:
-        #     new_scenario = {
-        #         key: None for key in all_keys
-        #     }  # Start with nulls (like SQL)
-        #     new_scenario.update(scenario)  # Add all left values
-
-        #     key_tuple = get_key_tuple(scenario, by_keys)
-        #     if matching_scenario := other_dict.get(key_tuple):
-        #         # Check for overlapping keys with different values
-        #         overlapping_keys = set(scenario.keys()) & set(matching_scenario.keys())
-        #         for key in overlapping_keys:
-        #             if key not in by_keys and scenario[key] != matching_scenario[key]:
-        #                 join_conditions = [f"{k}='{scenario[k]}'" for k in by_keys]
-        #                 print(
-        #                     f"Warning: Conflicting values for key '{key}' where {' AND '.join(join_conditions)}. "
-        #                     f"Keeping left value: {scenario[key]} (discarding: {matching_scenario[key]})"
-        #                 )
-
-        #         # Only update with non-overlapping keys from matching scenario
-        #         new_keys = set(matching_scenario.keys()) - set(scenario.keys())
-        #         new_scenario.update({k: matching_scenario[k] for k in new_keys})
-
-        #     new_scenarios.append(Scenario(new_scenario))
-
-        # return ScenarioList(new_scenarios)
 
     @classmethod
-    def from_tsv(cls, source: Union[str, urllib.parse.ParseResult]) -> ScenarioList:
+    def from_tsv(cls, source: Union[str, "ParseResult"]) -> ScenarioList:
         """Create a ScenarioList from a TSV file or URL."""
         return cls.from_delimited_file(source, delimiter="\t")
 
-    def to_dict(self, sort=False, add_edsl_version=True) -> dict:
+    def to_dict(self, sort: bool = False, add_edsl_version: bool = True) -> dict:
         """
         >>> s = ScenarioList([Scenario({'food': 'wood chips'}), Scenario({'food': 'wood-fired pizza'})])
         >>> s.to_dict()
@@ -1135,6 +1251,27 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
             d["edsl_class_name"] = self.__class__.__name__
         return d
 
+    def to(self, survey: Union["Survey", "QuestionBase"]) -> "Jobs":
+        """Create a Jobs object from a ScenarioList and a Survey object.
+
+        :param survey: The Survey object to use for the Jobs object.
+
+        Example:
+        >>> from edsl import Survey
+        >>> from edsl.jobs.Jobs import Jobs
+        >>> from edsl import ScenarioList
+        >>> isinstance(ScenarioList.example().to(Survey.example()), Jobs)
+        True
+        """
+        from edsl.surveys.Survey import Survey
+        from edsl.questions.QuestionBase import QuestionBase
+        from edsl.jobs.Jobs import Jobs
+
+        if isinstance(survey, QuestionBase):
+            return Survey([survey]).by(self)
+        else:
+            return survey.by(self)
+
     @classmethod
     def gen(cls, scenario_dicts_list: List[dict]) -> ScenarioList:
         """Create a `ScenarioList` from a list of dictionaries.
@@ -1160,15 +1297,12 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
     @classmethod
     def from_nested_dict(cls, data: dict) -> ScenarioList:
         """Create a `ScenarioList` from a nested dictionary."""
-        from edsl.scenarios.Scenario import Scenario
-
         s = ScenarioList()
         for key, value in data.items():
             s.add_list(key, value)
         return s
 
     def code(self) -> str:
-        ## TODO: Refactor to only use the questions actually in the survey
         """Create the Python code representation of a survey."""
         header_lines = [
             "from edsl.scenarios.Scenario import Scenario",
@@ -1191,16 +1325,16 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
         """
         return cls([Scenario.example(randomize), Scenario.example(randomize)])
 
-    def rich_print(self) -> None:
-        """Display an object as a table."""
-        from rich.table import Table
+    # def rich_print(self) -> None:
+    #     """Display an object as a table."""
+    #     from rich.table import Table
 
-        table = Table(title="ScenarioList")
-        table.add_column("Index", style="bold")
-        table.add_column("Scenario")
-        for i, s in enumerate(self):
-            table.add_row(str(i), s.rich_print())
-        return table
+    #     table = Table(title="ScenarioList")
+    #     table.add_column("Index", style="bold")
+    #     table.add_column("Scenario")
+    #     for i, s in enumerate(self):
+    #         table.add_row(str(i), s.rich_print())
+    #     return table
 
     def __getitem__(self, key: Union[int, slice]) -> Any:
         """Return the item at the given index.
@@ -1246,9 +1380,18 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
                     f"The 'name' field is reserved for the agent's name---putting this value in {proposed_agent_name}"
                 )
                 new_scenario[proposed_agent_name] = name
-                agents.append(Agent(traits=new_scenario, name=name))
+                new_agent = Agent(traits=new_scenario, name=name)
+            if "agent_parameters" in new_scenario:
+                agent_parameters = new_scenario.pop("agent_parameters")
+                instruction = agent_parameters.get("instruction", None)
+                name = agent_parameters.get("name", None)
+                new_agent = Agent(
+                    traits=new_scenario, name=name, instruction=instruction
+                )
             else:
-                agents.append(Agent(traits=new_scenario))
+                new_agent = Agent(traits=new_scenario)
+
+            agents.append(new_agent)
 
         return AgentList(agents)