edsl 0.1.46__py3-none-any.whl → 0.1.48__py3-none-any.whl

edsl/scenarios/{ScenarioList.py → scenario_list.py}

@@ -1,4 +1,21 @@
-"""A list of Scenarios to be used in a survey."""
+"""
+ScenarioList provides a collection of Scenario objects with advanced operations.
+
+The ScenarioList module extends the functionality of a simple list of Scenario objects,
+providing powerful operations for data manipulation, filtering, transformation, and analysis.
+It serves as a bridge between individual Scenarios and higher-level EDSL components like
+Surveys and Jobs.
+
+Key features include:
+- Collection operations (filtering, sorting, sampling, and iteration)
+- Data manipulation (transformation, joining, grouping, pivoting)
+- Format conversion (to/from pandas, CSV, Excel, etc.)
+- Advanced selection and retrieval mechanisms
+- Integration with other EDSL components
+
+ScenarioList is a core component in the EDSL framework for creating, managing, and
+manipulating collections of Scenarios for experiments, surveys, and data processing tasks.
+"""
 
 from __future__ import annotations
 from typing import (
@@ -10,12 +27,7 @@ from typing import (
     Literal,
     TYPE_CHECKING,
 )
-
-try:
-    from typing import TypeAlias
-except ImportError:
-    from typing_extensions import TypeAlias
-
+import warnings
 import csv
 import random
 from io import StringIO
@@ -23,37 +35,34 @@ import inspect
 from collections import UserList, defaultdict
 from collections.abc import Iterable
 
-if TYPE_CHECKING:
-    from urllib.parse import ParseResult
-    from edsl.results.Dataset import Dataset
-    from edsl.jobs.Jobs import Jobs
-    from edsl.surveys.Survey import Survey
-    from edsl.questions.QuestionBase import QuestionBase
-
-
 from simpleeval import EvalWithCompoundTypes, NameNotDefined  # type: ignore
-
 from tabulate import tabulate_formats
 
-from edsl.Base import Base
-from edsl.utilities.remove_edsl_version import remove_edsl_version
+try:
+    from typing import TypeAlias
+except ImportError:
+    from typing_extensions import TypeAlias
 
-from edsl.scenarios.Scenario import Scenario
-from edsl.scenarios.ScenarioListPdfMixin import ScenarioListPdfMixin
-from edsl.scenarios.ScenarioListExportMixin import ScenarioListExportMixin
-from edsl.utilities.naming_utilities import sanitize_string
-from edsl.utilities.is_valid_variable_name import is_valid_variable_name
-from edsl.exceptions.scenarios import ScenarioError
+if TYPE_CHECKING:
+    from urllib.parse import ParseResult
+    from ..dataset import Dataset
+    from ..jobs import Jobs
+    from ..surveys import Survey
+    from ..questions import QuestionBase
 
-from edsl.scenarios.directory_scanner import DirectoryScanner
 
+from ..base import Base
+from ..utilities import remove_edsl_version, sanitize_string, is_valid_variable_name, dict_hash
+from ..dataset import ScenarioListOperationsMixin
 
-class ScenarioListMixin(ScenarioListPdfMixin, ScenarioListExportMixin):
-    pass
+from .exceptions import ScenarioError
+from .scenario import Scenario
+from .directory_scanner import DirectoryScanner
+from .scenario_list_pdf_tools import PdfTools
 
 
 if TYPE_CHECKING:
-    from edsl.results.Dataset import Dataset
+    from ..dataset import Dataset
 
 TableFormat: TypeAlias = Literal[
     "plain",
@@ -72,9 +81,42 @@ TableFormat: TypeAlias = Literal[
     "tsv",
 ]
 
-
-class ScenarioList(Base, UserList, ScenarioListMixin):
-    """Class for creating a list of scenarios to be used in a survey."""
+class ScenarioList(Base, UserList, ScenarioListOperationsMixin):
+    """
+    A collection of Scenario objects with advanced operations for manipulation and analysis.
+    
+    ScenarioList extends Python's UserList to provide specialized functionality for
+    working with collections of Scenario objects. It inherits from Base to integrate
+    with EDSL's object model and from ScenarioListOperationsMixin to provide
+    powerful data manipulation capabilities.
+    
+    The class provides methods for filtering, sorting, joining, transforming, and
+    analyzing collections of Scenarios. It's designed to work seamlessly with other
+    EDSL components like Surveys, Jobs, and Questions.
+    
+    Attributes:
+        data (list): The underlying list of Scenario objects.
+        codebook (dict): Optional metadata describing the fields in the scenarios.
+        
+    Examples:
+        Create a ScenarioList from Scenario objects:
+        >>> from edsl.scenarios import Scenario, ScenarioList
+        >>> s1 = Scenario({"product": "apple", "price": 1.99})
+        >>> s2 = Scenario({"product": "banana", "price": 0.99})
+        >>> sl = ScenarioList([s1, s2])
+        
+        Filter scenarios based on a condition:
+        >>> cheap_fruits = sl.filter("price < 1.50")
+        >>> len(cheap_fruits)
+        1
+        >>> cheap_fruits[0]["product"]
+        'banana'
+        
+        Add a new column based on existing data:
+        >>> sl_with_tax = sl.mutate("tax = price * 0.08")
+        >>> sl_with_tax[0]["tax"]
+        0.1592
+    """
 
     __documentation__ = (
         "https://docs.expectedparrot.com/en/latest/scenarios.html#scenariolist"
@@ -83,7 +125,24 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
     def __init__(
         self, data: Optional[list] = None, codebook: Optional[dict[str, str]] = None
     ):
-        """Initialize the ScenarioList class."""
+        """
+        Initialize a new ScenarioList with optional data and codebook.
+        
+        Args:
+            data: A list of Scenario objects. If None, an empty list is used.
+            codebook: A dictionary mapping field names to descriptions or metadata.
+                     Used for documentation and to provide context for fields.
+                     
+        Examples:
+            >>> sl = ScenarioList()  # Empty list
+            >>> s1 = Scenario({"product": "apple"})
+            >>> s2 = Scenario({"product": "banana"})
+            >>> sl = ScenarioList([s1, s2])  # With data
+            
+            >>> # With a codebook
+            >>> codebook = {"product": "Fruit name", "price": "Price in USD"}
+            >>> sl = ScenarioList([s1, s2], codebook=codebook)
+        """
         if data is not None:
             super().__init__(data)
         else:
@@ -91,21 +150,85 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
         self.codebook = codebook or {}
 
     def unique(self) -> ScenarioList:
-        """Return a list of unique scenarios.
-
-        >>> s = ScenarioList([Scenario({'a': 1}), Scenario({'a': 1}), Scenario({'a': 2})])
-        >>> s.unique()
-        ScenarioList([Scenario({'a': 1}), Scenario({'a': 2})])
+        """
+        Return a new ScenarioList containing only unique Scenario objects.
+        
+        This method removes duplicate Scenario objects based on their hash values,
+        which are determined by their content. Two Scenarios with identical key-value
+        pairs will have the same hash and be considered duplicates.
+        
+        Returns:
+            A new ScenarioList containing only unique Scenario objects.
+            
+        Examples:
+            >>> from edsl.scenarios import Scenario, ScenarioList
+            >>> s1 = Scenario({"a": 1})
+            >>> s2 = Scenario({"a": 1})  # Same content as s1
+            >>> s3 = Scenario({"a": 2})
+            >>> sl = ScenarioList([s1, s2, s3])
+            >>> unique_sl = sl.unique()
+            >>> len(unique_sl)
+            2
+            >>> unique_sl
+            ScenarioList([Scenario({'a': 1}), Scenario({'a': 2})])
+            
+        Notes:
+            - The order of scenarios in the result is not guaranteed due to the use of sets
+            - Uniqueness is determined by the Scenario's __hash__ method
+            - The original ScenarioList is not modified
         """
         return ScenarioList(list(set(self)))
 
     @property
     def has_jinja_braces(self) -> bool:
-        """Check if the ScenarioList has Jinja braces."""
+        """
+        Check if any Scenario in the list contains values with Jinja template braces.
+        
+        This property checks all Scenarios in the list to determine if any contain
+        string values with Jinja template syntax ({{ and }}). This is important for
+        rendering templates and avoiding conflicts with other templating systems.
+        
+        Returns:
+            True if any Scenario contains values with Jinja braces, False otherwise.
+            
+        Examples:
+            >>> from edsl.scenarios import Scenario, ScenarioList
+            >>> s1 = Scenario({"text": "Plain text"})
+            >>> s2 = Scenario({"text": "Template with {{variable}}"})
+            >>> sl1 = ScenarioList([s1])
+            >>> sl1.has_jinja_braces
+            False
+            >>> sl2 = ScenarioList([s1, s2])
+            >>> sl2.has_jinja_braces
+            True
+        """
         return any([scenario.has_jinja_braces for scenario in self])
 
     def _convert_jinja_braces(self) -> ScenarioList:
-        """Convert Jinja braces to Python braces."""
+        """
+        Convert Jinja braces to alternative symbols in all Scenarios in the list.
+        
+        This method creates a new ScenarioList where all Jinja template braces
+        ({{ and }}) in string values are converted to alternative symbols (<< and >>).
+        This is useful when you need to prevent template processing or avoid conflicts
+        with other templating systems.
+        
+        Returns:
+            A new ScenarioList with converted braces in all Scenarios.
+            
+        Examples:
+            >>> from edsl.scenarios import Scenario, ScenarioList
+            >>> s = Scenario({"text": "Template with {{variable}}"})
+            >>> sl = ScenarioList([s])
+            >>> converted = sl._convert_jinja_braces()
+            >>> converted[0]["text"]
+            'Template with <<variable>>'
+            
+        Notes:
+            - The original ScenarioList is not modified
+            - This is primarily intended for internal use
+            - The default replacement symbols are << and >>
+        """
         return ScenarioList([scenario._convert_jinja_braces() for scenario in self])
 
     def give_valid_names(self, existing_codebook: dict = None) -> ScenarioList:
@@ -202,7 +325,7 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
             url={https://arxiv.org/abs/2407.11418},
             }
         """
-        from edsl import QuestionYesNo
+        from ..questions import QuestionYesNo
 
         new_scenario_list = self.duplicate()
         q = QuestionYesNo(
@@ -341,8 +464,6 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
         >>> hash(s)
         1262252885757976162
         """
-        from edsl.utilities.utilities import dict_hash
-
         return dict_hash(self.to_dict(sort=True, add_edsl_version=False))
 
     def __eq__(self, other: Any) -> bool:
@@ -360,7 +481,7 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
         ScenarioList([Scenario({'a': 1, 'b': 3}), Scenario({'a': 1, 'b': 4}), Scenario({'a': 2, 'b': 3}), Scenario({'a': 2, 'b': 4})])
         """
         from itertools import product
-        from edsl import Scenario
+        from .scenario import Scenario
         if isinstance(other, Scenario):
             other = ScenarioList([other])
         elif not isinstance(other, ScenarioList):
@@ -436,12 +557,14 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
                 new_scenarios.append(new_scenario)
         return ScenarioList(new_scenarios)
 
-    def _concatenate(self, fields: List[str], output_type: str = "string", separator: str = ";") -> ScenarioList:
+    def _concatenate(self, fields: List[str], output_type: str = "string", separator: str = ";", new_field_name: Optional[str] = None) -> ScenarioList:
         """Private method to handle concatenation logic for different output types.
         
         :param fields: The fields to concatenate.
         :param output_type: The type of output ("string", "list", or "set").
         :param separator: The separator to use for string concatenation.
+        :param new_field_name: Optional custom name for the concatenated field.
+                             If None, defaults to "concat_field1_field2_..."
         
         Returns:
             ScenarioList: A new ScenarioList with concatenated fields.
@@ -461,17 +584,17 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
                     values.append(new_scenario[field])
                     del new_scenario[field]
 
-            new_field_name = f"concat_{'_'.join(fields)}"
+            field_name = new_field_name if new_field_name is not None else f"concat_{'_'.join(fields)}"
             
             if output_type == "string":
                 # Convert all values to strings and join with separator
-                new_scenario[new_field_name] = separator.join(str(v) for v in values)
+                new_scenario[field_name] = separator.join(str(v) for v in values)
             elif output_type == "list":
                 # Keep as a list
-                new_scenario[new_field_name] = values
+                new_scenario[field_name] = values
             elif output_type == "set":
                 # Convert to a set (removes duplicates)
-                new_scenario[new_field_name] = set(values)
+                new_scenario[field_name] = set(values)
             else:
                 raise ValueError(f"Invalid output_type: {output_type}. Must be 'string', 'list', or 'set'.")
                 
@@ -479,11 +602,12 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
 
         return ScenarioList(new_scenarios)
 
-    def concatenate(self, fields: List[str], separator: str = ";") -> ScenarioList:
+    def concatenate(self, fields: List[str], separator: str = ";", new_field_name: Optional[str] = None) -> ScenarioList:
         """Concatenate specified fields into a single string field.
 
         :param fields: The fields to concatenate.
         :param separator: The separator to use.
+        :param new_field_name: Optional custom name for the concatenated field.
 
         Returns:
             ScenarioList: A new ScenarioList with concatenated fields.
@@ -492,13 +616,16 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
             >>> s = ScenarioList([Scenario({'a': 1, 'b': 2, 'c': 3}), Scenario({'a': 4, 'b': 5, 'c': 6})])
             >>> s.concatenate(['a', 'b', 'c'])
             ScenarioList([Scenario({'concat_a_b_c': '1;2;3'}), Scenario({'concat_a_b_c': '4;5;6'})])
+            >>> s.concatenate(['a', 'b', 'c'], new_field_name='combined')
+            ScenarioList([Scenario({'combined': '1;2;3'}), Scenario({'combined': '4;5;6'})])
         """
-        return self._concatenate(fields, output_type="string", separator=separator)
+        return self._concatenate(fields, output_type="string", separator=separator, new_field_name=new_field_name)
 
-    def concatenate_to_list(self, fields: List[str]) -> ScenarioList:
+    def concatenate_to_list(self, fields: List[str], new_field_name: Optional[str] = None) -> ScenarioList:
         """Concatenate specified fields into a single list field.
 
         :param fields: The fields to concatenate.
+        :param new_field_name: Optional custom name for the concatenated field.
 
         Returns:
             ScenarioList: A new ScenarioList with fields concatenated into a list.
@@ -507,13 +634,16 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
             >>> s = ScenarioList([Scenario({'a': 1, 'b': 2, 'c': 3}), Scenario({'a': 4, 'b': 5, 'c': 6})])
             >>> s.concatenate_to_list(['a', 'b', 'c'])
             ScenarioList([Scenario({'concat_a_b_c': [1, 2, 3]}), Scenario({'concat_a_b_c': [4, 5, 6]})])
+            >>> s.concatenate_to_list(['a', 'b', 'c'], new_field_name='values')
+            ScenarioList([Scenario({'values': [1, 2, 3]}), Scenario({'values': [4, 5, 6]})])
         """
-        return self._concatenate(fields, output_type="list")
+        return self._concatenate(fields, output_type="list", new_field_name=new_field_name)
 
-    def concatenate_to_set(self, fields: List[str]) -> ScenarioList:
+    def concatenate_to_set(self, fields: List[str], new_field_name: Optional[str] = None) -> ScenarioList:
         """Concatenate specified fields into a single set field.
 
         :param fields: The fields to concatenate.
+        :param new_field_name: Optional custom name for the concatenated field.
 
         Returns:
             ScenarioList: A new ScenarioList with fields concatenated into a set.
@@ -522,11 +652,10 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
             >>> s = ScenarioList([Scenario({'a': 1, 'b': 2, 'c': 3}), Scenario({'a': 4, 'b': 5, 'c': 6})])
             >>> s.concatenate_to_set(['a', 'b', 'c'])
             ScenarioList([Scenario({'concat_a_b_c': {1, 2, 3}}), Scenario({'concat_a_b_c': {4, 5, 6}})])
-            >>> s = ScenarioList([Scenario({'a': 1, 'b': 1, 'c': 3})])
-            >>> s.concatenate_to_set(['a', 'b', 'c'])
-            ScenarioList([Scenario({'concat_a_b_c': {1, 3}})])
+            >>> s.concatenate_to_set(['a', 'b', 'c'], new_field_name='unique_values')
+            ScenarioList([Scenario({'unique_values': {1, 2, 3}}), Scenario({'unique_values': {4, 5, 6}})])
         """
-        return self._concatenate(fields, output_type="set")
+        return self._concatenate(fields, output_type="set", new_field_name=new_field_name)
 
     def unpack_dict(
         self, field: str, prefix: Optional[str] = None, drop_field: bool = False
@@ -601,7 +730,6 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
             )
         raw_var_name, expression = new_var_string.split("=", 1)
         var_name = raw_var_name.strip()
-        from edsl.utilities.utilities import is_valid_variable_name
 
         if not is_valid_variable_name(var_name):
             raise ScenarioError(f"{var_name} is not a valid variable name.")
@@ -729,7 +857,7 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
         >>> s.select('a')
         ScenarioList([Scenario({'a': 1}), Scenario({'a': 1})])
         """
-        from edsl.scenarios.scenario_selector import ScenarioSelector
+        from .scenario_selector import ScenarioSelector
 
         return ScenarioSelector(self).select(*fields)
 
@@ -850,7 +978,7 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
         >>> s.to_dataset()
         Dataset([{'a': [1, 2, 3]}, {'b': [4, 5, 6]}])
         """
-        from edsl.results.Dataset import Dataset
+        from ..dataset import Dataset
 
         keys = list(self[0].keys())
         for scenario in self:
@@ -959,6 +1087,8 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
             new_list.append(new_obj)
         return new_list
 
+    
+
     def replace_names(self, new_names: list) -> ScenarioList:
         """Replace the field names in the scenarios with a new list of names.
 
@@ -1156,6 +1286,7 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
         
         return scenario_list
 
+    @classmethod
     def from_wikipedia(cls, url: str, table_index: int = 0):
         """
         Extracts a table from a Wikipedia page.
@@ -1230,13 +1361,19 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
 
     @classmethod
     def from_excel(
-        cls, filename: str, sheet_name: Optional[str] = None
+        cls, filename: str, sheet_name: Optional[str] = None, skip_rows: Optional[List[int]] = None, use_codebook: bool = False
     ) -> ScenarioList:
         """Create a ScenarioList from an Excel file.
 
         If the Excel file contains multiple sheets and no sheet_name is provided,
         the method will print the available sheets and require the user to specify one.
 
+        Args:
+            filename (str): Path to the Excel file
+            sheet_name (Optional[str]): Name of the sheet to load. If None and multiple sheets exist,
+                                      will raise an error listing available sheets.
+            skip_rows (Optional[List[int]]): List of row indices to skip (0-based). If None, all rows are included.
+
         Example:
 
         >>> import tempfile
@@ -1244,30 +1381,33 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
         >>> import pandas as pd
         >>> with tempfile.NamedTemporaryFile(delete=False, suffix='.xlsx') as f:
         ...     df1 = pd.DataFrame({
-        ...         'name': ['Alice', 'Bob'],
-        ...         'age': [30, 25],
-        ...         'location': ['New York', 'Los Angeles']
+        ...         'name': ['Alice', 'Bob', 'Charlie'],
+        ...         'age': [30, 25, 35],
+        ...         'location': ['New York', 'Los Angeles', 'Chicago']
         ...     })
         ...     df2 = pd.DataFrame({
-        ...         'name': ['Charlie', 'David'],
-        ...         'age': [35, 40],
-        ...         'location': ['Chicago', 'Boston']
+        ...         'name': ['David', 'Eve'],
+        ...         'age': [40, 45],
+        ...         'location': ['Boston', 'Seattle']
         ...     })
         ...     with pd.ExcelWriter(f.name) as writer:
         ...         df1.to_excel(writer, sheet_name='Sheet1', index=False)
         ...         df2.to_excel(writer, sheet_name='Sheet2', index=False)
         ...     temp_filename = f.name
+        >>> # Load all rows
         >>> scenario_list = ScenarioList.from_excel(temp_filename, sheet_name='Sheet1')
         >>> len(scenario_list)
+        3
+        >>> # Skip the second row (index 1)
+        >>> scenario_list = ScenarioList.from_excel(temp_filename, sheet_name='Sheet1', skip_rows=[1])
+        >>> len(scenario_list)
         2
         >>> scenario_list[0]['name']
         'Alice'
-        >>> scenario_list = ScenarioList.from_excel(temp_filename)  # Should raise an error and list sheets
-        Traceback (most recent call last):
-        ...
-        ValueError: Please provide a sheet name to load data from.
+        >>> scenario_list[1]['name']
+        'Charlie'
         """
-        from edsl.scenarios.Scenario import Scenario
+        from .scenario import Scenario
         import pandas as pd
 
         # Get all sheets
@@ -1287,11 +1427,28 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
         # Load the specified or determined sheet
         df = pd.read_excel(filename, sheet_name=sheet_name)
 
+        # Skip specified rows if any
+        if skip_rows:
+            df = df.drop(skip_rows)
+            # Reset index to ensure continuous indexing
+            df = df.reset_index(drop=True)
+
+        if use_codebook:
+            codebook = {f"col_{i}": col for i, col in enumerate(df.columns)}
+            koobedoc = {col:f"col_{i}" for i, col in enumerate(df.columns)}
+
         observations = []
         for _, row in df.iterrows():
-            observations.append(Scenario(row.to_dict()))
+            if use_codebook:
+                observations.append(Scenario({koobedoc.get(k):v for k,v in row.to_dict().items()}))
+            else:
+                observations.append(Scenario(row.to_dict()))
 
-        return cls(observations)
+
+        if use_codebook:    
+            return cls(observations, codebook=codebook)
+        else:
+            return cls(observations)
 
     @classmethod
     def from_google_sheet(cls, url: str, sheet_name: str = None, column_names: Optional[List[str]]= None) -> ScenarioList:
@@ -1359,7 +1516,7 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
     ) -> ScenarioList:
         """Create a ScenarioList from a delimited file (CSV/TSV) or URL."""
         import requests
-        from edsl.scenarios.Scenario import Scenario
+        from .scenario import Scenario
         from urllib.parse import urlparse
         from urllib.parse import ParseResult
 
@@ -1415,7 +1572,7 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
         >>> s3 == ScenarioList([Scenario({'age': 30, 'location': 'New York', 'name': 'Alice'}), Scenario({'age': 25, 'location': None, 'name': 'Bob'})])
         True
         """
-        from edsl.scenarios.scenario_join import ScenarioJoin
+        from .scenario_join import ScenarioJoin
 
         sj = ScenarioJoin(self, other)
         return sj.left_join(by)
@@ -1439,7 +1596,7 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
         d = {"scenarios": [s.to_dict(add_edsl_version=add_edsl_version) for s in data]}
 
         if add_edsl_version:
-            from edsl import __version__
+            from .. import __version__
 
             d["edsl_version"] = __version__
             d["edsl_class_name"] = self.__class__.__name__
@@ -1451,15 +1608,13 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
         :param survey: The Survey object to use for the Jobs object.
 
         Example:
-        >>> from edsl import Survey
-        >>> from edsl.jobs.Jobs import Jobs
-        >>> from edsl import ScenarioList
+        >>> from edsl import Survey, Jobs, ScenarioList
         >>> isinstance(ScenarioList.example().to(Survey.example()), Jobs)
         True
         """
-        from edsl.surveys.Survey import Survey
-        from edsl.questions.QuestionBase import QuestionBase
-        from edsl.jobs.Jobs import Jobs
+        from ..surveys import Survey
+        from ..questions import QuestionBase
+        from ..jobs import Jobs
 
         if isinstance(survey, QuestionBase):
             return Survey([survey]).by(self)
@@ -1476,7 +1631,7 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
         ScenarioList([Scenario({'name': 'Alice'}), Scenario({'name': 'Bob'})])
 
         """
-        from edsl.scenarios.Scenario import Scenario
+        from .scenario import Scenario
 
         return cls([Scenario(s) for s in scenario_dicts_list])
 
@@ -1484,7 +1639,7 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
     @remove_edsl_version
     def from_dict(cls, data) -> ScenarioList:
         """Create a `ScenarioList` from a dictionary."""
-        from edsl.scenarios.Scenario import Scenario
+        from .scenario import Scenario
 
         return cls([Scenario.from_dict(s) for s in data["scenarios"]])
 
@@ -1511,8 +1666,8 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
     def code(self) -> str:
         """Create the Python code representation of a survey."""
         header_lines = [
-            "from edsl.scenarios.Scenario import Scenario",
-            "from edsl.scenarios.ScenarioList import ScenarioList",
+            "from edsl.scenarios import Scenario",
+            "from edsl.scenarios import ScenarioList",
         ]
         lines = ["\n".join(header_lines)]
         names = []
@@ -1531,17 +1686,6 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
         """
         return cls([Scenario.example(randomize), Scenario.example(randomize)])
 
-    # def rich_print(self) -> None:
-    #     """Display an object as a table."""
-    #     from rich.table import Table
-
-    #     table = Table(title="ScenarioList")
-    #     table.add_column("Index", style="bold")
-    #     table.add_column("Scenario")
-    #     for i, s in enumerate(self):
-    #         table.add_row(str(i), s.rich_print())
-    #     return table
-
     def __getitem__(self, key: Union[int, slice]) -> Any:
         """Return the item at the given index.
 
@@ -1570,9 +1714,7 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
         >>> s.to_agent_list()
         AgentList([Agent(traits = {'age': 22, 'hair': 'brown', 'height': 5.5}), Agent(traits = {'age': 22, 'hair': 'brown', 'height': 5.5})])
         """
-        from edsl.agents.AgentList import AgentList
-        from edsl.agents.Agent import Agent
-        import warnings
+        from ..agents import AgentList, Agent
 
         agents = []
         for scenario in self:
@@ -1629,12 +1771,14 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
             new_scenarios.extend(replacement_scenarios)
         return ScenarioList(new_scenarios)
 
-    def collapse(self, field: str) -> ScenarioList:
+    def collapse(self, field: str, separator: Optional[str] = None, add_count: bool = False) -> ScenarioList:
         """Collapse a ScenarioList by grouping on all fields except the specified one,
         collecting the values of the specified field into a list.
 
         Args:
             field: The field to collapse (whose values will be collected into lists)
+            separator: Optional string to join the values with instead of keeping as a list
+            add_count: If True, adds a field showing the number of collapsed rows
 
         Returns:
             ScenarioList: A new ScenarioList with the specified field collapsed into lists
@@ -1642,12 +1786,11 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
         Example:
         >>> s = ScenarioList([
         ...     Scenario({'category': 'fruit', 'color': 'red', 'item': 'apple'}),
-        ...     Scenario({'category': 'fruit', 'color': 'yellow', 'item': 'banana'}),
         ...     Scenario({'category': 'fruit', 'color': 'red', 'item': 'cherry'}),
         ...     Scenario({'category': 'vegetable', 'color': 'green', 'item': 'spinach'})
         ... ])
-        >>> s.collapse('item')
-        ScenarioList([Scenario({'category': 'fruit', 'color': 'red', 'item': ['apple', 'cherry']}), Scenario({'category': 'fruit', 'color': 'yellow', 'item': ['banana']}), Scenario({'category': 'vegetable', 'color': 'green', 'item': ['spinach']})])
+        >>> s.collapse('item', add_count=True)
+        ScenarioList([Scenario({'category': 'fruit', 'color': 'red', 'item': ['apple', 'cherry'], 'num_collapsed_rows': 2}), Scenario({'category': 'vegetable', 'color': 'green', 'item': ['spinach'], 'num_collapsed_rows': 1})])
         """
         if not self:
             return ScenarioList([])
@@ -1667,11 +1810,186 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
         result = []
         for key, values in grouped.items():
             new_scenario = dict(zip(id_vars, key))
-            new_scenario[field] = values
+            if separator:
+                new_scenario[field] = separator.join(values)
+            else:
+                new_scenario[field] = values
+            if add_count:
+                new_scenario['num_collapsed_rows'] = len(values)
+            result.append(Scenario(new_scenario))
+        
+        return ScenarioList(result)
+
+    def create_comparisons(
+        self, 
+        bidirectional: bool = False, 
+        num_options: int = 2,
+        option_prefix: str = "option_",
+        use_alphabet: bool = False
+    ) -> ScenarioList:
+        """Create a new ScenarioList with comparisons between scenarios.
+        
+        Each scenario in the result contains multiple original scenarios as dictionaries,
+        allowing for side-by-side comparison.
+        
+        Args:
+            bidirectional (bool): If True, include both (A,B) and (B,A) comparisons.
+                If False, only include (A,B) where A comes before B in the original list.
+            num_options (int): Number of scenarios to include in each comparison.
+                Default is 2 for pairwise comparisons.
+            option_prefix (str): Prefix for the keys in the resulting scenarios.
+                Default is "option_", resulting in keys like "option_1", "option_2", etc.
+                Ignored if use_alphabet is True.
+            use_alphabet (bool): If True, use letters as keys (A, B, C, etc.) instead of
+                the option_prefix with numbers.
+        
+        Returns:
+            ScenarioList: A new ScenarioList where each scenario contains multiple original
+                scenarios as dictionaries.
+        
+        Example:
+            >>> s = ScenarioList([
+            ...     Scenario({'id': 1, 'text': 'Option A'}),
+            ...     Scenario({'id': 2, 'text': 'Option B'}),
+            ...     Scenario({'id': 3, 'text': 'Option C'})
+            ... ])
+            >>> s.create_comparisons(use_alphabet=True)
+            ScenarioList([Scenario({'A': {'id': 1, 'text': 'Option A'}, 'B': {'id': 2, 'text': 'Option B'}}), Scenario({'A': {'id': 1, 'text': 'Option A'}, 'B': {'id': 3, 'text': 'Option C'}}), Scenario({'A': {'id': 2, 'text': 'Option B'}, 'B': {'id': 3, 'text': 'Option C'}})])
+            >>> s.create_comparisons(num_options=3, use_alphabet=True)
+            ScenarioList([Scenario({'A': {'id': 1, 'text': 'Option A'}, 'B': {'id': 2, 'text': 'Option B'}, 'C': {'id': 3, 'text': 'Option C'}})])
+        """
+        from itertools import combinations, permutations
+        import string
+        
+        if num_options < 2:
+            raise ValueError("num_options must be at least 2")
+        
+        if num_options > len(self):
+            raise ValueError(f"num_options ({num_options}) cannot exceed the number of scenarios ({len(self)})")
+        
+        if use_alphabet and num_options > 26:
+            raise ValueError("When using alphabet labels, num_options cannot exceed 26 (the number of letters in the English alphabet)")
+        
+        # Convert each scenario to a dictionary
+        scenario_dicts = [scenario.to_dict(add_edsl_version=False) for scenario in self]
+        
+        # Generate combinations or permutations based on bidirectional flag
+        if bidirectional:
+            # For bidirectional, use permutations to get all ordered arrangements
+            if num_options == 2:
+                # For pairwise, we can use permutations with r=2
+                scenario_groups = permutations(scenario_dicts, 2)
+            else:
+                # For more than 2 options with bidirectional=True, 
+                # we need all permutations of the specified size
+                scenario_groups = permutations(scenario_dicts, num_options)
+        else:
+            # For unidirectional, use combinations to get unordered groups
+            scenario_groups = combinations(scenario_dicts, num_options)
+        
+        # Create new scenarios with the combinations
+        result = []
+        for group in scenario_groups:
+            new_scenario = {}
+            for i, scenario_dict in enumerate(group):
+                if use_alphabet:
+                    # Use uppercase letters (A, B, C, etc.)
+                    key = string.ascii_uppercase[i]
+                else:
+                    # Use the option prefix with numbers (option_1, option_2, etc.)
+                    key = f"{option_prefix}{i+1}"
+                new_scenario[key] = scenario_dict
             result.append(Scenario(new_scenario))
         
         return ScenarioList(result)
 
+    @classmethod
+    def from_parquet(cls, filepath: str) -> ScenarioList:
+        """Create a ScenarioList from a Parquet file.
+        
+        Args:
+            filepath (str): Path to the Parquet file
+            
+        Returns:
+            ScenarioList: A ScenarioList containing the data from the Parquet file
+            
+        Example:
+        >>> import pandas as pd
+        >>> import tempfile
+        >>> df = pd.DataFrame({'name': ['Alice', 'Bob'], 'age': [30, 25]})
+        >>> # The following would create and read a parquet file if dependencies are installed:
+        >>> # with tempfile.NamedTemporaryFile(suffix='.parquet', delete=False) as f:
+        >>> #     df.to_parquet(f.name)
+        >>> #     scenario_list = ScenarioList.from_parquet(f.name)
+        >>> # Instead, we'll demonstrate the equivalent result:
+        >>> scenario_list = ScenarioList.from_pandas(df)
+        >>> len(scenario_list)
+        2
+        >>> scenario_list[0]['name']
+        'Alice'
+        """
+        import pandas as pd
+        
+        try:
+            # Try to read the Parquet file with pandas
+            df = pd.read_parquet(filepath)
+        except ImportError as e:
+            # Handle missing dependencies with a helpful error message
+            if "pyarrow" in str(e) or "fastparquet" in str(e):
+                raise ImportError(
+                    "Missing dependencies for Parquet support. Please install either pyarrow or fastparquet:\n"
+                    "  pip install pyarrow\n"
+                    "  or\n"
+                    "  pip install fastparquet"
+                ) from e
+            else:
+                raise
+        
+        # Convert the DataFrame to a ScenarioList
+        return cls.from_pandas(df)
+
+    def replace_values(self, replacements:dict) -> "ScenarioList":
+        """
+        Create new scenarios with values replaced according to the provided replacement dictionary.
+        
+        Args:
+            replacements (dict): Dictionary of values to replace {old_value: new_value}
+        
+        Returns:
+            ScenarioList: A new ScenarioList with replaced values
+        
+        Examples:
+            >>> scenarios = ScenarioList([
+            ...     Scenario({'a': 'nan', 'b': 1}),
+            ...     Scenario({'a': 2, 'b': 'nan'})
+            ... ])
+            >>> replaced = scenarios.replace_values({'nan': None})
+            >>> print(replaced)
+            ScenarioList([Scenario({'a': None, 'b': 1}), Scenario({'a': 2, 'b': None})])
+            >>> # Original scenarios remain unchanged
+            >>> print(scenarios)
+            ScenarioList([Scenario({'a': 'nan', 'b': 1}), Scenario({'a': 2, 'b': 'nan'})])
+        """
+        new_scenarios = []
+        for scenario in self:
+            new_scenario = {}
+            for key, value in scenario.items():
+                if str(value) in replacements:
+                    new_scenario[key] = replacements[str(value)]
+                else:
+                    new_scenario[key] = value
+            new_scenarios.append(Scenario(new_scenario))
+        return ScenarioList(new_scenarios)
+
+    @classmethod
+    def from_pdf(cls, filename_or_url, collapse_pages=False):
+        return PdfTools.from_pdf(filename_or_url, collapse_pages)
+    
+    @classmethod
+    def from_pdf_to_image(cls, pdf_path, image_format="jpeg"):
+        return PdfTools.from_pdf_to_image(pdf_path, image_format)
+
+
 
 if __name__ == "__main__":
     import doctest