edsl 0.1.39__py3-none-any.whl → 0.1.39.dev1__py3-none-any.whl

edsl/scenarios/ScenarioList.py

@@ -1,78 +1,33 @@
 """A list of Scenarios to be used in a survey."""
 
 from __future__ import annotations
-from typing import (
-    Any,
-    Optional,
-    Union,
-    List,
-    Callable,
-    Literal,
-    TYPE_CHECKING,
-)
-
-try:
-    from typing import TypeAlias
-except ImportError:
-    from typing_extensions import TypeAlias
-
+from typing import Any, Optional, Union, List, Callable
 import csv
 import random
+from collections import UserList, Counter
+from collections.abc import Iterable
+import urllib.parse
+import urllib.request
 from io import StringIO
+from collections import defaultdict
 import inspect
-from collections import UserList, defaultdict
-from collections.abc import Iterable
-
-if TYPE_CHECKING:
-    from urllib.parse import ParseResult
-    from edsl.results.Dataset import Dataset
-    from edsl.jobs.Jobs import Jobs
-    from edsl.surveys.Survey import Survey
-    from edsl.questions.QuestionBase import QuestionBase
-
 
-from simpleeval import EvalWithCompoundTypes, NameNotDefined  # type: ignore
-
-from tabulate import tabulate_formats
+from simpleeval import EvalWithCompoundTypes
 
 from edsl.Base import Base
-from edsl.utilities.remove_edsl_version import remove_edsl_version
-
+from edsl.utilities.decorators import add_edsl_version, remove_edsl_version
 from edsl.scenarios.Scenario import Scenario
 from edsl.scenarios.ScenarioListPdfMixin import ScenarioListPdfMixin
 from edsl.scenarios.ScenarioListExportMixin import ScenarioListExportMixin
-from edsl.utilities.naming_utilities import sanitize_string
-from edsl.utilities.is_valid_variable_name import is_valid_variable_name
-from edsl.exceptions.scenarios import ScenarioError
 
-from edsl.scenarios.directory_scanner import DirectoryScanner
+from edsl.utilities.naming_utilities import sanitize_string
+from edsl.utilities.utilities import is_valid_variable_name
 
 
 class ScenarioListMixin(ScenarioListPdfMixin, ScenarioListExportMixin):
     pass
 
 
-if TYPE_CHECKING:
-    from edsl.results.Dataset import Dataset
-
-TableFormat: TypeAlias = Literal[
-    "plain",
-    "simple",
-    "github",
-    "grid",
-    "fancy_grid",
-    "pipe",
-    "orgtbl",
-    "rst",
-    "mediawiki",
-    "html",
-    "latex",
-    "latex_raw",
-    "latex_booktabs",
-    "tsv",
-]
-
-
 class ScenarioList(Base, UserList, ScenarioListMixin):
     """Class for creating a list of scenarios to be used in a survey."""
 
@@ -80,9 +35,7 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
         "https://docs.expectedparrot.com/en/latest/scenarios.html#scenariolist"
     )
 
-    def __init__(
-        self, data: Optional[list] = None, codebook: Optional[dict[str, str]] = None
-    ):
+    def __init__(self, data: Optional[list] = None, codebook: Optional[dict] = None):
         """Initialize the ScenarioList class."""
         if data is not None:
             super().__init__(data)
@@ -104,19 +57,12 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
         """Check if the ScenarioList has Jinja braces."""
         return any([scenario.has_jinja_braces for scenario in self])
 
-    def _convert_jinja_braces(self) -> ScenarioList:
+    def convert_jinja_braces(self) -> ScenarioList:
         """Convert Jinja braces to Python braces."""
-        return ScenarioList([scenario._convert_jinja_braces() for scenario in self])
-
-    def give_valid_names(self, existing_codebook: dict = None) -> ScenarioList:
-        """Give valid names to the scenario keys, using an existing codebook if provided.
+        return ScenarioList([scenario.convert_jinja_braces() for scenario in self])
 
-        Args:
-            existing_codebook (dict, optional): Existing mapping of original keys to valid names.
-                Defaults to None.
-
-        Returns:
-            ScenarioList: A new ScenarioList with valid variable names and updated codebook.
+    def give_valid_names(self) -> ScenarioList:
+        """Give valid names to the scenario keys.
 
         >>> s = ScenarioList([Scenario({'a': 1, 'b': 2}), Scenario({'a': 1, 'b': 1})])
         >>> s.give_valid_names()
@@ -124,38 +70,27 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
         >>> s = ScenarioList([Scenario({'are you there John?': 1, 'b': 2}), Scenario({'a': 1, 'b': 1})])
         >>> s.give_valid_names()
         ScenarioList([Scenario({'john': 1, 'b': 2}), Scenario({'a': 1, 'b': 1})])
-        >>> s.give_valid_names({'are you there John?': 'custom_name'})
-        ScenarioList([Scenario({'custom_name': 1, 'b': 2}), Scenario({'a': 1, 'b': 1})])
         """
-        codebook = existing_codebook.copy() if existing_codebook else {}
-        new_scenarios = []
-
+        codebook = {}
+        new_scenaerios = []
         for scenario in self:
             new_scenario = {}
             for key in scenario:
-                if is_valid_variable_name(key):
-                    new_scenario[key] = scenario[key]
-                    continue
-
-                if key in codebook:
-                    new_key = codebook[key]
+                if not is_valid_variable_name(key):
+                    if key in codebook:
+                        new_key = codebook[key]
+                    else:
+                        new_key = sanitize_string(key)
+                        if not is_valid_variable_name(new_key):
+                            new_key = f"var_{len(codebook)}"
+                        codebook[key] = new_key
+                    new_scenario[new_key] = scenario[key]
                 else:
-                    new_key = sanitize_string(key)
-                    if not is_valid_variable_name(new_key):
-                        new_key = f"var_{len(codebook)}"
-                    codebook[key] = new_key
-
-                new_scenario[new_key] = scenario[key]
-
-            new_scenarios.append(Scenario(new_scenario))
-
-        return ScenarioList(new_scenarios, codebook)
+                    new_scenario[key] = scenario[key]
+            new_scenaerios.append(Scenario(new_scenario))
+        return ScenarioList(new_scenaerios, codebook)
 
-    def unpivot(
-        self,
-        id_vars: Optional[List[str]] = None,
-        value_vars: Optional[List[str]] = None,
-    ) -> ScenarioList:
+    def unpivot(self, id_vars=None, value_vars=None):
         """
         Unpivot the ScenarioList, allowing for id variables to be specified.
 
@@ -186,40 +121,7 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
 
         return ScenarioList(new_scenarios)
 
-    def sem_filter(self, language_predicate: str) -> ScenarioList:
-        """Filter the ScenarioList based on a language predicate.
-
-        :param language_predicate: The language predicate to use.
-
-        Inspired by:
-        @misc{patel2024semanticoperators,
-            title={Semantic Operators: A Declarative Model for Rich, AI-based Analytics Over Text Data},
-            author={Liana Patel and Siddharth Jha and Parth Asawa and Melissa Pan and Carlos Guestrin and Matei Zaharia},
-            year={2024},
-            eprint={2407.11418},
-            archivePrefix={arXiv},
-            primaryClass={cs.DB},
-            url={https://arxiv.org/abs/2407.11418},
-            }
-        """
-        from edsl import QuestionYesNo
-
-        new_scenario_list = self.duplicate()
-        q = QuestionYesNo(
-            question_text=language_predicate, question_name="binary_outcome"
-        )
-        results = q.by(new_scenario_list).run(verbose=False)
-        new_scenario_list = new_scenario_list.add_list(
-            "criteria", results.select("binary_outcome").to_list()
-        )
-        return new_scenario_list.filter("criteria == 'Yes'").drop("criteria")
-
-    def pivot(
-        self,
-        id_vars: List[str] = None,
-        var_name="variable",
-        value_name="value",
-    ) -> ScenarioList:
+    def pivot(self, id_vars, var_name="variable", value_name="value"):
         """
         Pivot the ScenarioList from long to wide format.
 
@@ -261,15 +163,15 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
 
         return ScenarioList(pivoted_scenarios)
 
-    def group_by(
-        self, id_vars: List[str], variables: List[str], func: Callable
-    ) -> ScenarioList:
+    def group_by(self, id_vars, variables, func):
         """
         Group the ScenarioList by id_vars and apply a function to the specified variables.
 
-        :param id_vars: Fields to use as identifier variables
-        :param variables: Fields to group and aggregate
-        :param func: Function to apply to the grouped variables
+        Parameters:
+        id_vars (list): Fields to use as identifier variables for grouping
+        variables (list): Fields to pass to the aggregation function
+        func (callable): Function to apply to the grouped variables.
+                        Should accept lists of values for each variable.
 
         Returns:
         ScenarioList: A new ScenarioList with the grouped and aggregated results
@@ -289,12 +191,12 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
         # Check if the function is compatible with the specified variables
         func_params = inspect.signature(func).parameters
         if len(func_params) != len(variables):
-            raise ScenarioError(
+            raise ValueError(
                 f"Function {func.__name__} expects {len(func_params)} arguments, but {len(variables)} variables were provided"
             )
 
         # Group the scenarios
-        grouped: dict[str, list] = defaultdict(lambda: defaultdict(list))
+        grouped = defaultdict(lambda: defaultdict(list))
         for scenario in self:
             key = tuple(scenario[id_var] for id_var in id_vars)
             for var in variables:
@@ -306,12 +208,10 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
             try:
                 aggregated = func(*[group[var] for var in variables])
             except Exception as e:
-                raise ScenarioError(f"Error applying function to group {key}: {str(e)}")
+                raise ValueError(f"Error applying function to group {key}: {str(e)}")
 
             if not isinstance(aggregated, dict):
-                raise ScenarioError(
-                    f"Function {func.__name__} must return a dictionary"
-                )
+                raise ValueError(f"Function {func.__name__} must return a dictionary")
 
             new_scenario = dict(zip(id_vars, key))
             new_scenario.update(aggregated)
@@ -378,18 +278,50 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
         """
         return self.__mul__(other)
 
-    def shuffle(self, seed: Optional[str] = None) -> ScenarioList:
+    def shuffle(self, seed: Optional[str] = "edsl") -> ScenarioList:
         """Shuffle the ScenarioList.
 
         >>> s = ScenarioList.from_list("a", [1,2,3,4])
-        >>> s.shuffle(seed = "1234")
-        ScenarioList([Scenario({'a': 1}), Scenario({'a': 4}), Scenario({'a': 3}), Scenario({'a': 2})])
+        >>> s.shuffle()
+        ScenarioList([Scenario({'a': 3}), Scenario({'a': 4}), Scenario({'a': 1}), Scenario({'a': 2})])
         """
-        sl = self.duplicate()
-        if seed:
-            random.seed(seed)
-        random.shuffle(sl.data)
-        return sl
+        random.seed(seed)
+        random.shuffle(self.data)
+        return self
+
+    def _repr_html_(self):
+        """Return an HTML representation of the AgentList."""
+        # return (
+        #     str(self.summary(format="html")) + "<br>" + str(self.table(tablefmt="html"))
+        # )
+        footer = f"<a href={self.__documentation__}>(docs)</a>"
+        return str(self.summary(format="html")) + footer
+
+    # def _repr_html_(self) -> str:
+    # from edsl.utilities.utilities import data_to_html
+
+    # data = self.to_dict()
+    # _ = data.pop("edsl_version")
+    # _ = data.pop("edsl_class_name")
+    # for s in data["scenarios"]:
+    #     _ = s.pop("edsl_version")
+    #     _ = s.pop("edsl_class_name")
+    # for scenario in data["scenarios"]:
+    #     for key, value in scenario.items():
+    #         if hasattr(value, "to_dict"):
+    #             data[key] = value.to_dict()
+    # return data_to_html(data)
+
+    # def tally(self, field) -> dict:
+    #     """Return a tally of the values in the field.
+
+    #     Example:
+
+    #     >>> s = ScenarioList([Scenario({'a': 1, 'b': 1}), Scenario({'a': 1, 'b': 2})])
+    #     >>> s.tally('b')
+    #     {1: 1, 2: 1}
+    #     """
+    #     return dict(Counter([scenario[field] for scenario in self]))
 
     def sample(self, n: int, seed: Optional[str] = None) -> ScenarioList:
         """Return a random sample from the ScenarioList
@@ -401,22 +333,16 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
         if seed:
             random.seed(seed)
 
-        sl = self.duplicate()
-        return ScenarioList(random.sample(sl.data, n))
+        return ScenarioList(random.sample(self.data, n))
 
-    def expand(self, expand_field: str, number_field: bool = False) -> ScenarioList:
+    def expand(self, expand_field: str, number_field=False) -> ScenarioList:
         """Expand the ScenarioList by a field.
 
-        :param expand_field: The field to expand.
-        :param number_field: Whether to add a field with the index of the value
-
         Example:
 
         >>> s = ScenarioList( [ Scenario({'a':1, 'b':[1,2]}) ] )
         >>> s.expand('b')
         ScenarioList([Scenario({'a': 1, 'b': 1}), Scenario({'a': 1, 'b': 2})])
-        >>> s.expand('b', number_field=True)
-        ScenarioList([Scenario({'a': 1, 'b': 1, 'b_number': 1}), Scenario({'a': 1, 'b': 2, 'b_number': 2})])
         """
         new_scenarios = []
         for scenario in self:
@@ -431,11 +357,12 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
                 new_scenarios.append(new_scenario)
         return ScenarioList(new_scenarios)
 
-    def concatenate(self, fields: List[str], separator: str = ";") -> ScenarioList:
+    def concatenate(self, fields: List[str], separator: str = ";") -> "ScenarioList":
         """Concatenate specified fields into a single field.
 
-        :param fields: The fields to concatenate.
-        :param separator: The separator to use.
+        Args:
+            fields (List[str]): List of field names to concatenate.
+            separator (str, optional): Separator to use between field values. Defaults to ";".
 
         Returns:
             ScenarioList: A new ScenarioList with concatenated fields.
@@ -465,17 +392,11 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
     ) -> ScenarioList:
         """Unpack a dictionary field into separate fields.
 
-        :param field: The field to unpack.
-        :param prefix: An optional prefix to add to the new fields.
-        :param drop_field: Whether to drop the original field.
-
         Example:
 
         >>> s = ScenarioList([Scenario({'a': 1, 'b': {'c': 2, 'd': 3}})])
         >>> s.unpack_dict('b')
         ScenarioList([Scenario({'a': 1, 'b': {'c': 2, 'd': 3}, 'c': 2, 'd': 3})])
-        >>> s.unpack_dict('b', prefix='new_')
-        ScenarioList([Scenario({'a': 1, 'b': {'c': 2, 'd': 3}, 'new_c': 2, 'new_d': 3})])
         """
         new_scenarios = []
         for scenario in self:
@@ -493,17 +414,7 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
     def transform(
         self, field: str, func: Callable, new_name: Optional[str] = None
     ) -> ScenarioList:
-        """Transform a field using a function.
-
-        :param field: The field to transform.
-        :param func: The function to apply to the field.
-        :param new_name: An optional new name for the transformed field.
-
-        >>> s = ScenarioList([Scenario({'a': 1, 'b': 2}), Scenario({'a': 1, 'b': 1})])
-        >>> s.transform('b', lambda x: x + 1)
-        ScenarioList([Scenario({'a': 1, 'b': 3}), Scenario({'a': 1, 'b': 2})])
-
-        """
+        """Transform a field using a function."""
         new_scenarios = []
         for scenario in self:
             new_scenario = scenario.copy()
@@ -517,9 +428,6 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
         """
         Return a new ScenarioList with a new variable added.
 
-        :param new_var_string: A string with the new variable assignment.
-        :param functions_dict: A dictionary of functions to use in the assignment.
-
         Example:
 
         >>> s = ScenarioList([Scenario({'a': 1, 'b': 2}), Scenario({'a': 1, 'b': 1})])
@@ -528,7 +436,7 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
 
         """
         if "=" not in new_var_string:
-            raise ScenarioError(
+            raise Exception(
                 f"Mutate requires an '=' in the string, but '{new_var_string}' doesn't have one."
             )
         raw_var_name, expression = new_var_string.split("=", 1)
@@ -536,7 +444,7 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
         from edsl.utilities.utilities import is_valid_variable_name
 
         if not is_valid_variable_name(var_name):
-            raise ScenarioError(f"{var_name} is not a valid variable name.")
+            raise Exception(f"{var_name} is not a valid variable name.")
 
         # create the evaluator
         functions_dict = functions_dict or {}
@@ -554,15 +462,13 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
         try:
             new_data = [new_scenario(s, var_name) for s in self]
         except Exception as e:
-            raise ScenarioError(f"Error in mutate. Exception:{e}")
+            raise Exception(f"Error in mutate. Exception:{e}")
 
         return ScenarioList(new_data)
 
     def order_by(self, *fields: str, reverse: bool = False) -> ScenarioList:
         """Order the scenarios by one or more fields.
 
-        :param fields: The fields to order by.
-        :param reverse: Whether to reverse the order.
         Example:
 
         >>> s = ScenarioList([Scenario({'a': 1, 'b': 2}), Scenario({'a': 1, 'b': 1})])
@@ -575,41 +481,16 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
 
         return ScenarioList(sorted(self, key=get_sort_key, reverse=reverse))
 
-    def duplicate(self) -> ScenarioList:
-        """Return a copy of the ScenarioList.
-
-        >>> sl = ScenarioList.example()
-        >>> sl_copy = sl.duplicate()
-        >>> sl == sl_copy
-        True
-        >>> sl is sl_copy
-        False
-        """
-        return ScenarioList([scenario.copy() for scenario in self])
-
     def filter(self, expression: str) -> ScenarioList:
         """
         Filter a list of scenarios based on an expression.
 
-        :param expression: The expression to filter by.
-
         Example:
 
         >>> s = ScenarioList([Scenario({'a': 1, 'b': 1}), Scenario({'a': 1, 'b': 2})])
         >>> s.filter("b == 2")
         ScenarioList([Scenario({'a': 1, 'b': 2})])
         """
-        sl = self.duplicate()
-        base_keys = set(self[0].keys())
-        keys = set()
-        for scenario in sl:
-            keys.update(scenario.keys())
-        if keys != base_keys:
-            import warnings
-
-            warnings.warn(
-                "Ragged ScenarioList detected (different keys for different scenario entries). This may cause unexpected behavior."
-            )
 
         def create_evaluator(scenario: Scenario):
             """Create an evaluator for the given result.
@@ -619,22 +500,14 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
 
         try:
             # iterates through all the results and evaluates the expression
-            new_data = []
-            for scenario in sl:
-                if create_evaluator(scenario).eval(expression):
-                    new_data.append(scenario)
-        except NameNotDefined as e:
-            available_fields = ", ".join(self.data[0].keys() if self.data else [])
-            raise ScenarioError(
-                f"Error in filter: '{e}'\n"
-                f"The expression '{expression}' refers to a field that does not exist.\n"
-                f"Scenario: {scenario}\n"
-                f"Available fields: {available_fields}\n"
-                "Check your filter expression or consult the documentation: "
-                "https://docs.expectedparrot.com/en/latest/scenarios.html#module-edsl.scenarios.Scenario"
-            ) from None
+            new_data = [
+                scenario
+                for scenario in self.data
+                if create_evaluator(scenario).eval(expression)
+            ]
         except Exception as e:
-            raise ScenarioError(f"Error in filter. Exception:{e}")
+            print(f"Exception:{e}")
+            raise Exception(f"Error in filter. Exception:{e}")
 
         return ScenarioList(new_data)
 
@@ -646,26 +519,30 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
         :param urls: A list of URLs.
         :param field_name: The name of the field to store the text from the URLs.
 
+
         """
         return ScenarioList([Scenario.from_url(url, field_name) for url in urls])
 
-    def select(self, *fields: str) -> ScenarioList:
+    def select(self, *fields) -> ScenarioList:
         """
         Selects scenarios with only the references fields.
 
-        :param fields: The fields to select.
-
         Example:
 
         >>> s = ScenarioList([Scenario({'a': 1, 'b': 1}), Scenario({'a': 1, 'b': 2})])
         >>> s.select('a')
         ScenarioList([Scenario({'a': 1}), Scenario({'a': 1})])
         """
-        from edsl.scenarios.scenario_selector import ScenarioSelector
+        if len(fields) == 1:
+            fields_to_select = [list(fields)[0]]
+        else:
+            fields_to_select = list(fields)
 
-        return ScenarioSelector(self).select(*fields)
+        return ScenarioList(
+            [scenario.select(fields_to_select) for scenario in self.data]
+        )
 
-    def drop(self, *fields: str) -> ScenarioList:
+    def drop(self, *fields) -> ScenarioList:
         """Drop fields from the scenarios.
 
         Example:
@@ -674,22 +551,18 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
         >>> s.drop('a')
         ScenarioList([Scenario({'b': 1}), Scenario({'b': 2})])
         """
-        sl = self.duplicate()
-        return ScenarioList([scenario.drop(fields) for scenario in sl])
+        return ScenarioList([scenario.drop(fields) for scenario in self.data])
 
-    def keep(self, *fields: str) -> ScenarioList:
+    def keep(self, *fields) -> ScenarioList:
         """Keep only the specified fields in the scenarios.
 
-        :param fields: The fields to keep.
-
         Example:
 
         >>> s = ScenarioList([Scenario({'a': 1, 'b': 1}), Scenario({'a': 1, 'b': 2})])
         >>> s.keep('a')
         ScenarioList([Scenario({'a': 1}), Scenario({'a': 1})])
         """
-        sl = self.duplicate()
-        return ScenarioList([scenario.keep(fields) for scenario in sl])
+        return ScenarioList([scenario.keep(fields) for scenario in self.data])
 
     @classmethod
     def from_list(
@@ -697,10 +570,6 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
     ) -> ScenarioList:
         """Create a ScenarioList from a list of values.
 
-        :param name: The name of the field.
-        :param values: The list of values.
-        :param func: An optional function to apply to the values.
-
         Example:
 
         >>> ScenarioList.from_list('name', ['Alice', 'Bob'])
@@ -710,12 +579,7 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
             func = lambda x: x
         return cls([Scenario({name: func(value)}) for value in values])
 
-    def table(
-        self,
-        *fields: str,
-        tablefmt: Optional[TableFormat] = None,
-        pretty_labels: Optional[dict[str, str]] = None,
-    ) -> str:
+    def table(self, *fields, tablefmt=None, pretty_labels=None) -> str:
         """Return the ScenarioList as a table."""
 
         from tabulate import tabulate_formats
@@ -730,41 +594,26 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
         )
 
     def tree(self, node_list: Optional[List[str]] = None) -> str:
-        """Return the ScenarioList as a tree.
-
-        :param node_list: The list of nodes to include in the tree.
-        """
+        """Return the ScenarioList as a tree."""
         return self.to_dataset().tree(node_list)
 
-    def _summary(self) -> dict:
-        """Return a summary of the ScenarioList.
-
-        >>> ScenarioList.example()._summary()
-        {'scenarios': 2, 'keys': ['persona']}
-        """
+    def _summary(self):
         d = {
-            "scenarios": len(self),
-            "keys": list(self.parameters),
+            "EDSL Class name": "ScenarioList",
+            "# Scenarios": len(self),
+            "Scenario Keys": list(self.parameters),
         }
         return d
 
-    def reorder_keys(self, new_order: List[str]) -> ScenarioList:
+    def reorder_keys(self, new_order):
         """Reorder the keys in the scenarios.
 
-        :param new_order: The new order of the keys.
-
         Example:
 
         >>> s = ScenarioList([Scenario({'a': 1, 'b': 2}), Scenario({'a': 3, 'b': 4})])
         >>> s.reorder_keys(['b', 'a'])
         ScenarioList([Scenario({'b': 2, 'a': 1}), Scenario({'b': 4, 'a': 3})])
-        >>> s.reorder_keys(['a', 'b', 'c'])
-        Traceback (most recent call last):
-        ...
-        AssertionError
         """
-        assert set(new_order) == set(self.parameters)
-
         new_scenarios = []
         for scenario in self:
             new_scenario = Scenario({key: scenario[key] for key in new_order})
@@ -773,8 +622,6 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
 
     def to_dataset(self) -> "Dataset":
         """
-        Convert the ScenarioList to a Dataset.
-
         >>> s = ScenarioList.from_list("a", [1,2,3])
         >>> s.to_dataset()
         Dataset([{'a': [1, 2, 3]}])
@@ -784,14 +631,8 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
         """
         from edsl.results.Dataset import Dataset
 
-        keys = list(self[0].keys())
-        for scenario in self:
-            new_keys = list(scenario.keys())
-            if new_keys != keys:
-                keys = list(set(keys + new_keys))
-        data = [
-            {key: [scenario.get(key, None) for scenario in self.data]} for key in keys
-        ]
+        keys = self[0].keys()
+        data = [{key: [scenario[key] for scenario in self.data]} for key in keys]
         return Dataset(data)
 
     def unpack(
@@ -823,14 +664,7 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
             new_scenarios.append(new_scenario)
         return ScenarioList(new_scenarios)
 
-    @classmethod
-    def from_list_of_tuples(self, *names: str, values: List[Tuple]) -> ScenarioList:
-        sl = ScenarioList.from_list(names[0], [value[0] for value in values])
-        for index, name in enumerate(names[1:]):
-            sl = sl.add_list(name, [value[index + 1] for value in values])
-        return sl
-
-    def add_list(self, name: str, values: List[Any]) -> ScenarioList:
+    def add_list(self, name, values) -> ScenarioList:
         """Add a list of values to a ScenarioList.
 
         Example:
@@ -839,25 +673,12 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
         >>> s.add_list('age', [30, 25])
         ScenarioList([Scenario({'name': 'Alice', 'age': 30}), Scenario({'name': 'Bob', 'age': 25})])
         """
-        sl = self.duplicate()
-        if len(values) != len(sl):
-            raise ScenarioError(
-                f"Length of values ({len(values)}) does not match length of ScenarioList ({len(sl)})"
-            )
         for i, value in enumerate(values):
-            sl[i][name] = value
-        return sl
-
-    @classmethod
-    def create_empty_scenario_list(cls, n: int) -> ScenarioList:
-        """Create an empty ScenarioList with n scenarios.
-
-        Example:
-
-        >>> ScenarioList.create_empty_scenario_list(3)
-        ScenarioList([Scenario({}), Scenario({}), Scenario({})])
-        """
-        return ScenarioList([Scenario({}) for _ in range(n)])
+            if i < len(self):
+                self[i][name] = value
+            else:
+                self.append(Scenario({name: value}))
+        return self
 
     def add_value(self, name: str, value: Any) -> ScenarioList:
         """Add a value to all scenarios in a ScenarioList.
@@ -868,16 +689,13 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
         >>> s.add_value('age', 30)
         ScenarioList([Scenario({'name': 'Alice', 'age': 30}), Scenario({'name': 'Bob', 'age': 30})])
         """
-        sl = self.duplicate()
-        for scenario in sl:
+        for scenario in self:
             scenario[name] = value
-        return sl
+        return self
 
     def rename(self, replacement_dict: dict) -> ScenarioList:
         """Rename the fields in the scenarios.
 
-        :param replacement_dict: A dictionary with the old names as keys and the new names as values.
-
         Example:
 
         >>> s = ScenarioList([Scenario({'name': 'Alice', 'age': 30}), Scenario({'name': 'Bob', 'age': 25})])
@@ -892,26 +710,8 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
             new_list.append(new_obj)
         return new_list
 
-    ## NEEDS TO BE FIXED
-    # def new_column_names(self, new_names: List[str]) -> ScenarioList:
-    #     """Rename the fields in the scenarios.
-
-    #     Example:
-
-    #     >>> s = ScenarioList([Scenario({'name': 'Alice', 'age': 30}), Scenario({'name': 'Bob', 'age': 25})])
-    #     >>> s.new_column_names(['first_name', 'years'])
-    #     ScenarioList([Scenario({'first_name': 'Alice', 'years': 30}), Scenario({'first_name': 'Bob', 'years': 25})])
-
-    #     """
-    #     new_list = ScenarioList([])
-    #     for obj in self:
-    #         new_obj = obj.new_column_names(new_names)
-    #         new_list.append(new_obj)
-    #     return new_list
-
     @classmethod
     def from_sqlite(cls, filepath: str, table: str):
-        """Create a ScenarioList from a SQLite database."""
         import sqlite3
 
         with sqlite3.connect(filepath) as conn:
@@ -1055,9 +855,6 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
     def to_key_value(self, field: str, value=None) -> Union[dict, set]:
         """Return the set of values in the field.
 
-        :param field: The field to extract values from.
-        :param value: An optional field to use as the value in the key-value pair.
-
         Example:
 
         >>> s = ScenarioList([Scenario({'name': 'Alice'}), Scenario({'name': 'Bob'})])
@@ -1177,42 +974,56 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
 
     @classmethod
     def from_delimited_file(
-        cls, source: Union[str, "ParseResult"], delimiter: str = ","
+        cls, source: Union[str, urllib.parse.ParseResult], delimiter: str = ","
     ) -> ScenarioList:
-        """Create a ScenarioList from a delimited file (CSV/TSV) or URL."""
-        import requests
-        from edsl.scenarios.Scenario import Scenario
-        from urllib.parse import urlparse
-        from urllib.parse import ParseResult
+        """Create a ScenarioList from a delimited file (CSV/TSV) or URL.
 
-        headers = {
-            "Accept": "text/csv,application/csv,text/plain",
-            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
-        }
+        Args:
+            source: A string representing either a local file path or a URL to a delimited file,
+                    or a urllib.parse.ParseResult object for a URL.
+            delimiter: The delimiter used in the file. Defaults to ',' for CSV files.
+                    Use '\t' for TSV files.
+
+        Returns:
+            ScenarioList: A ScenarioList object containing the data from the file.
+
+        Example:
+            # For CSV files
+
+            >>> with open('data.csv', 'w') as f:
+            ...     _ = f.write('name,age\\nAlice,30\\nBob,25\\n')
+            >>> scenario_list = ScenarioList.from_delimited_file('data.csv')
+
+            # For TSV files
+            >>> with open('data.tsv', 'w') as f:
+            ...     _ = f.write('name\\tage\\nAlice\t30\\nBob\t25\\n')
+            >>> scenario_list = ScenarioList.from_delimited_file('data.tsv', delimiter='\\t')
+
+        """
+        from edsl.scenarios.Scenario import Scenario
 
         def is_url(source):
             try:
-                result = urlparse(source)
+                result = urllib.parse.urlparse(source)
                 return all([result.scheme, result.netloc])
             except ValueError:
                 return False
 
-        try:
-            if isinstance(source, str) and is_url(source):
-                response = requests.get(source, headers=headers)
-                response.raise_for_status()
-                file_obj = StringIO(response.text)
-            elif isinstance(source, ParseResult):
-                response = requests.get(source.geturl(), headers=headers)
-                response.raise_for_status()
-                file_obj = StringIO(response.text)
-            else:
-                file_obj = open(source, "r")
+        if isinstance(source, str) and is_url(source):
+            with urllib.request.urlopen(source) as response:
+                file_content = response.read().decode("utf-8")
+            file_obj = StringIO(file_content)
+        elif isinstance(source, urllib.parse.ParseResult):
+            with urllib.request.urlopen(source.geturl()) as response:
+                file_content = response.read().decode("utf-8")
+            file_obj = StringIO(file_content)
+        else:
+            file_obj = open(source, "r")
 
+        try:
             reader = csv.reader(file_obj, delimiter=delimiter)
             header = next(reader)
             observations = [Scenario(dict(zip(header, row))) for row in reader]
-
         finally:
             file_obj.close()
 
@@ -1220,7 +1031,7 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
 
     # Convenience methods for specific file types
     @classmethod
-    def from_csv(cls, source: Union[str, "ParseResult"]) -> ScenarioList:
+    def from_csv(cls, source: Union[str, urllib.parse.ParseResult]) -> ScenarioList:
         """Create a ScenarioList from a CSV file or URL."""
         return cls.from_delimited_file(source, delimiter=",")
 
@@ -1237,17 +1048,75 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
         >>> s3 == ScenarioList([Scenario({'age': 30, 'location': 'New York', 'name': 'Alice'}), Scenario({'age': 25, 'location': None, 'name': 'Bob'})])
         True
         """
-        from edsl.scenarios.scenario_join import ScenarioJoin
+        from edsl.scenarios.ScenarioJoin import ScenarioJoin
 
         sj = ScenarioJoin(self, other)
         return sj.left_join(by)
+        # # Validate join keys
+        # if not by:
+        #     raise ValueError(
+        #         "Join keys cannot be empty. Please specify at least one key to join on."
+        #     )
+
+        # # Convert single string to list for consistent handling
+        # by_keys = [by] if isinstance(by, str) else by
+
+        # # Verify all join keys exist in both ScenarioLists
+        # left_keys = set(next(iter(self)).keys()) if self else set()
+        # right_keys = set(next(iter(other)).keys()) if other else set()
+
+        # missing_left = set(by_keys) - left_keys
+        # missing_right = set(by_keys) - right_keys
+        # if missing_left or missing_right:
+        #     missing = missing_left | missing_right
+        #     raise ValueError(f"Join key(s) {missing} not found in both ScenarioLists")
+
+        # # Create lookup dictionary from the other ScenarioList
+        # def get_key_tuple(scenario: Scenario, keys: list[str]) -> tuple:
+        #     return tuple(scenario[k] for k in keys)
+
+        # other_dict = {get_key_tuple(scenario, by_keys): scenario for scenario in other}
+
+        # # Collect all possible keys (like SQL combining all columns)
+        # all_keys = set()
+        # for scenario in self:
+        #     all_keys.update(scenario.keys())
+        # for scenario in other:
+        #     all_keys.update(scenario.keys())
+
+        # new_scenarios = []
+        # for scenario in self:
+        #     new_scenario = {
+        #         key: None for key in all_keys
+        #     }  # Start with nulls (like SQL)
+        #     new_scenario.update(scenario)  # Add all left values
+
+        #     key_tuple = get_key_tuple(scenario, by_keys)
+        #     if matching_scenario := other_dict.get(key_tuple):
+        #         # Check for overlapping keys with different values
+        #         overlapping_keys = set(scenario.keys()) & set(matching_scenario.keys())
+        #         for key in overlapping_keys:
+        #             if key not in by_keys and scenario[key] != matching_scenario[key]:
+        #                 join_conditions = [f"{k}='{scenario[k]}'" for k in by_keys]
+        #                 print(
+        #                     f"Warning: Conflicting values for key '{key}' where {' AND '.join(join_conditions)}. "
+        #                     f"Keeping left value: {scenario[key]} (discarding: {matching_scenario[key]})"
+        #                 )
+
+        #         # Only update with non-overlapping keys from matching scenario
+        #         new_keys = set(matching_scenario.keys()) - set(scenario.keys())
+        #         new_scenario.update({k: matching_scenario[k] for k in new_keys})
+
+        #     new_scenarios.append(Scenario(new_scenario))
+
+        # return ScenarioList(new_scenarios)
 
     @classmethod
-    def from_tsv(cls, source: Union[str, "ParseResult"]) -> ScenarioList:
+    def from_tsv(cls, source: Union[str, urllib.parse.ParseResult]) -> ScenarioList:
         """Create a ScenarioList from a TSV file or URL."""
         return cls.from_delimited_file(source, delimiter="\t")
 
-    def to_dict(self, sort: bool = False, add_edsl_version: bool = True) -> dict:
+    def to_dict(self, sort=False, add_edsl_version=True) -> dict:
         """
         >>> s = ScenarioList([Scenario({'food': 'wood chips'}), Scenario({'food': 'wood-fired pizza'})])
         >>> s.to_dict()
@@ -1259,7 +1128,6 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
         else:
             data = self
         d = {"scenarios": [s.to_dict(add_edsl_version=add_edsl_version) for s in data]}
-
         if add_edsl_version:
             from edsl import __version__
 
@@ -1267,27 +1135,6 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
             d["edsl_class_name"] = self.__class__.__name__
         return d
 
-    def to(self, survey: Union["Survey", "QuestionBase"]) -> "Jobs":
-        """Create a Jobs object from a ScenarioList and a Survey object.
-
-        :param survey: The Survey object to use for the Jobs object.
-
-        Example:
-        >>> from edsl import Survey
-        >>> from edsl.jobs.Jobs import Jobs
-        >>> from edsl import ScenarioList
-        >>> isinstance(ScenarioList.example().to(Survey.example()), Jobs)
-        True
-        """
-        from edsl.surveys.Survey import Survey
-        from edsl.questions.QuestionBase import QuestionBase
-        from edsl.jobs.Jobs import Jobs
-
-        if isinstance(survey, QuestionBase):
-            return Survey([survey]).by(self)
-        else:
-            return survey.by(self)
-
     @classmethod
     def gen(cls, scenario_dicts_list: List[dict]) -> ScenarioList:
         """Create a `ScenarioList` from a list of dictionaries.
@@ -1312,25 +1159,16 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
 
     @classmethod
     def from_nested_dict(cls, data: dict) -> ScenarioList:
-        """Create a `ScenarioList` from a nested dictionary.
-
-        >>> data = {"headline": ["Armistice Signed, War Over: Celebrations Erupt Across City"], "date": ["1918-11-11"], "author": ["Jane Smith"]}
-        >>> ScenarioList.from_nested_dict(data)
-        ScenarioList([Scenario({'headline': 'Armistice Signed, War Over: Celebrations Erupt Across City', 'date': '1918-11-11', 'author': 'Jane Smith'})])
-
-        """
-        length_of_first_list = len(next(iter(data.values())))
-        s = ScenarioList.create_empty_scenario_list(n=length_of_first_list)
+        """Create a `ScenarioList` from a nested dictionary."""
+        from edsl.scenarios.Scenario import Scenario
 
-        if any(len(v) != length_of_first_list for v in data.values()):
-            raise ValueError(
-                "All lists in the dictionary must be of the same length.",
-            )
-        for key, list_of_values in data.items():
-            s = s.add_list(key, list_of_values)
+        s = ScenarioList()
+        for key, value in data.items():
+            s.add_list(key, value)
         return s
 
     def code(self) -> str:
+        ## TODO: Refactor to only use the questions actually in the survey
         """Create the Python code representation of a survey."""
         header_lines = [
             "from edsl.scenarios.Scenario import Scenario",
@@ -1353,16 +1191,16 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
         """
         return cls([Scenario.example(randomize), Scenario.example(randomize)])
 
-    # def rich_print(self) -> None:
-    #     """Display an object as a table."""
-    #     from rich.table import Table
+    def rich_print(self) -> None:
+        """Display an object as a table."""
+        from rich.table import Table
 
-    #     table = Table(title="ScenarioList")
-    #     table.add_column("Index", style="bold")
-    #     table.add_column("Scenario")
-    #     for i, s in enumerate(self):
-    #         table.add_row(str(i), s.rich_print())
-    #     return table
+        table = Table(title="ScenarioList")
+        table.add_column("Index", style="bold")
+        table.add_column("Scenario")
+        for i, s in enumerate(self):
+            table.add_row(str(i), s.rich_print())
+        return table
 
     def __getitem__(self, key: Union[int, slice]) -> Any:
         """Return the item at the given index.
@@ -1408,18 +1246,9 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
                     f"The 'name' field is reserved for the agent's name---putting this value in {proposed_agent_name}"
                 )
                 new_scenario[proposed_agent_name] = name
-                new_agent = Agent(traits=new_scenario, name=name)
-            if "agent_parameters" in new_scenario:
-                agent_parameters = new_scenario.pop("agent_parameters")
-                instruction = agent_parameters.get("instruction", None)
-                name = agent_parameters.get("name", None)
-                new_agent = Agent(
-                    traits=new_scenario, name=name, instruction=instruction
-                )
+                agents.append(Agent(traits=new_scenario, name=name))
             else:
-                new_agent = Agent(traits=new_scenario)
-
-            agents.append(new_agent)
+                agents.append(Agent(traits=new_scenario))
 
         return AgentList(agents)