edsl 0.1.44__py3-none-any.whl → 0.1.46__py3-none-any.whl

edsl/questions/question_base_gen_mixin.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 import copy
 import itertools
-from typing import Optional, List, Callable, Type, TYPE_CHECKING
+from typing import Optional, List, Callable, Type, TYPE_CHECKING, Union
 
 if TYPE_CHECKING:
     from edsl.questions.QuestionBase import QuestionBase
@@ -9,7 +9,11 @@ if TYPE_CHECKING:
 
 
 class QuestionBaseGenMixin:
-    """Mixin for QuestionBase."""
+    """Mixin for QuestionBase.
+    
+    This mostly has functions that are used to generate new questions from existing ones.
+    
+    """
 
     def copy(self) -> QuestionBase:
         """Return a deep copy of the question.
@@ -85,48 +89,112 @@ class QuestionBaseGenMixin:
         lp = LoopProcessor(self)
         return lp.process_templates(scenario_list)
 
-    def render(self, replacement_dict: dict) -> "QuestionBase":
-        """Render the question components as jinja2 templates with the replacement dictionary.
-
-        :param replacement_dict: The dictionary of values to replace in the question components.
+    class MaxTemplateNestingExceeded(Exception):
+        """Raised when template rendering exceeds maximum allowed nesting level."""
+        pass
 
+    def render(self, replacement_dict: dict, return_dict: bool = False) -> Union["QuestionBase", dict]:
+        """Render the question components as jinja2 templates with the replacement dictionary.
+        Handles nested template variables by recursively rendering until all variables are resolved.
+        
+        Raises:
+            MaxTemplateNestingExceeded: If template nesting exceeds MAX_NESTING levels
+        
         >>> from edsl.questions.QuestionFreeText import QuestionFreeText
         >>> q = QuestionFreeText(question_name = "color", question_text = "What is your favorite {{ thing }}?")
         >>> q.render({"thing": "color"})
         Question('free_text', question_name = \"""color\""", question_text = \"""What is your favorite color?\""")
 
+        >>> from edsl.questions.QuestionMultipleChoice import QuestionMultipleChoice
+        >>> q = QuestionMultipleChoice(question_name = "color", question_text = "What is your favorite {{ thing }}?", question_options = ["red", "blue", "green"])
+        >>> from edsl.scenarios.Scenario import Scenario
+        >>> q.render(Scenario({"thing": "color"})).data
+        {'question_name': 'color', 'question_text': 'What is your favorite color?', 'question_options': ['red', 'blue', 'green']}
+
+        >>> from edsl.questions.QuestionMultipleChoice import QuestionMultipleChoice
+        >>> q = QuestionMultipleChoice(question_name = "color", question_text = "What is your favorite {{ thing }}?", question_options = ["red", "blue", "green"])
+        >>> q.render({"thing": 1}).data
+        {'question_name': 'color', 'question_text': 'What is your favorite 1?', 'question_options': ['red', 'blue', 'green']}
+
+
+        >>> from edsl.questions.QuestionMultipleChoice import QuestionMultipleChoice
+        >>> from edsl.scenarios.Scenario import Scenario
+        >>> q = QuestionMultipleChoice(question_name = "color", question_text = "What is your favorite {{ thing }}?", question_options = ["red", "blue", "green"])
+        >>> q.render(Scenario({"thing": "color of {{ object }}", "object":"water"})).data
+        {'question_name': 'color', 'question_text': 'What is your favorite color of water?', 'question_options': ['red', 'blue', 'green']}
+
+        
+        >>> from edsl.questions.QuestionFreeText import QuestionFreeText
+        >>> q = QuestionFreeText(question_name = "infinite", question_text = "This has {{ a }}")
+        >>> q.render({"a": "{{ b }}", "b": "{{ a }}"}) # doctest: +IGNORE_EXCEPTION_DETAIL
+        Traceback (most recent call last):
+            ...
+        edsl.questions.question_base_gen_mixin.QuestionBaseGenMixin.MaxTemplateNestingExceeded:...
         """
-        from jinja2 import Environment
+        from jinja2 import Environment, meta
         from edsl.scenarios.Scenario import Scenario
 
+        MAX_NESTING = 10  # Maximum allowed nesting levels
+        
         strings_only_replacement_dict = {
             k: v for k, v in replacement_dict.items() if not isinstance(v, Scenario)
         }
 
+        strings_only_replacement_dict['scenario'] = strings_only_replacement_dict
+
+        def _has_unrendered_variables(template_str: str, env: Environment) -> bool:
+            """Check if the template string has any unrendered variables."""
+            if not isinstance(template_str, str):
+                return False
+            ast = env.parse(template_str)
+            return bool(meta.find_undeclared_variables(ast))
+
         def render_string(value: str) -> str:
             if value is None or not isinstance(value, str):
                 return value
-            else:
-                try:
-                    return (
-                        Environment()
-                        .from_string(value)
-                        .render(strings_only_replacement_dict)
-                    )
-                except Exception as e:
-                    #breakpoint()
-                    import warnings
-
-                    warnings.warn("Failed to render string: " + value)
-                    # breakpoint()
-                    return value
-
-        return self.apply_function(render_string)
-
+            
+            try:
+                env = Environment()
+                result = value
+                nesting_count = 0
+                
+                while _has_unrendered_variables(result, env):
+                    if nesting_count >= MAX_NESTING:
+                        raise self.MaxTemplateNestingExceeded(
+                            f"Template rendering exceeded {MAX_NESTING} levels of nesting. "
+                            f"Current value: {result}"
+                        )
+                    
+                    template = env.from_string(result)
+                    new_result = template.render(strings_only_replacement_dict)
+                    if new_result == result:  # Break if no changes made
+                        break
+                    result = new_result
+                    nesting_count += 1
+                
+                return result
+            except self.MaxTemplateNestingExceeded:
+                raise
+            except Exception as e:
+                import warnings
+                warnings.warn("Failed to render string: " + value)
+                return value
+        if return_dict:
+            return self._apply_function_dict(render_string)
+        else:
+            return self.apply_function(render_string)
+      
     def apply_function(
-        self, func: Callable, exclude_components: List[str] = None
+        self, func: Callable, exclude_components: Optional[List[str]] = None
     ) -> QuestionBase:
-        """Apply a function to the question parts
+        from edsl.questions.QuestionBase import QuestionBase
+        d = self._apply_function_dict(func, exclude_components)
+        return QuestionBase.from_dict(d)
+
+    def _apply_function_dict(
+        self, func: Callable, exclude_components: Optional[List[str]] = None
+    ) -> dict:
+        """Apply a function to the question parts, excluding certain components.
 
         :param func: The function to apply to the question parts.
         :param exclude_components: The components to exclude from the function application.
@@ -141,7 +209,6 @@ class QuestionBaseGenMixin:
         Question('free_text', question_name = \"""COLOR\""", question_text = \"""WHAT IS YOUR FAVORITE COLOR?\""")
 
         """
-        from edsl.questions.QuestionBase import QuestionBase
 
         if exclude_components is None:
             exclude_components = ["question_name", "question_type"]
@@ -160,10 +227,10 @@ class QuestionBaseGenMixin:
                 d[key] = value
                 continue
             d[key] = func(value)
-        return QuestionBase.from_dict(d)
+        return d
 
 
 if __name__ == "__main__":
     import doctest
 
-    doctest.testmod()
+    doctest.testmod(optionflags=doctest.ELLIPSIS)

edsl/results/Dataset.py

@@ -15,6 +15,7 @@ from edsl.Base import PersistenceMixin, HashingMixin
 
 from edsl.results.smart_objects import FirstObject
 
+from edsl.results.ResultsGGMixin import GGPlotMethod
 
 class Dataset(UserList, ResultsExportMixin, PersistenceMixin, HashingMixin):
     """A class to represent a dataset of observations."""
@@ -26,6 +27,20 @@ class Dataset(UserList, ResultsExportMixin, PersistenceMixin, HashingMixin):
         super().__init__(data)
         self.print_parameters = print_parameters
 
+
+    def ggplot2(
+        self,
+        ggplot_code: str,
+        shape="wide",
+        sql: str = None,
+        remove_prefix: bool = True,
+        debug: bool = False,
+        height=4,
+        width=6,
+        factor_orders: Optional[dict] = None,
+    ):
+        return GGPlotMethod(self).ggplot2(ggplot_code, shape, sql, remove_prefix, debug, height, width, factor_orders)
+
     def __len__(self) -> int:
         """Return the number of observations in the dataset.
 
@@ -580,6 +595,56 @@ class Dataset(UserList, ResultsExportMixin, PersistenceMixin, HashingMixin):
         result = cls([{col: df[col].tolist()} for col in df.columns])
         return result
 
+    def to_docx(self, output_file: str, title: str = None) -> None:
+        """
+        Convert the dataset to a Word document.
+        
+        Args:
+            output_file (str): Path to save the Word document
+            title (str, optional): Title for the document
+        """
+        from docx import Document
+        from docx.shared import Inches
+        from docx.enum.text import WD_ALIGN_PARAGRAPH
+
+        # Create document
+        doc = Document()
+        
+        # Add title if provided
+        if title:
+            title_heading = doc.add_heading(title, level=1)
+            title_heading.alignment = WD_ALIGN_PARAGRAPH.CENTER
+
+        # Get headers and data
+        headers, data = self._tabular()
+        
+        # Create table
+        table = doc.add_table(rows=len(data) + 1, cols=len(headers))
+        table.style = 'Table Grid'
+        
+        # Add headers
+        for j, header in enumerate(headers):
+            cell = table.cell(0, j)
+            cell.text = str(header)
+        
+        # Add data
+        for i, row in enumerate(data):
+            for j, cell_content in enumerate(row):
+                cell = table.cell(i + 1, j)
+                cell.text = str(cell_content) if cell_content is not None else ""
+        
+        # Adjust column widths
+        for column in table.columns:
+            max_width = 0
+            for cell in column.cells:
+                text_width = len(str(cell.text))
+                max_width = max(max_width, text_width)
+            for cell in column.cells:
+                cell.width = Inches(min(max_width * 0.1 + 0.5, 6))
+        
+        # Save the document
+        doc.save(output_file)
+
 
 if __name__ == "__main__":
     import doctest

edsl/results/DatasetExportMixin.py

@@ -7,6 +7,7 @@ from typing import Optional, Tuple, Union, List
 
 from edsl.results.file_exports import CSVExport, ExcelExport, JSONLExport, SQLiteExport
 
+
 class DatasetExportMixin:
     """Mixin class for exporting Dataset objects."""
 
@@ -82,7 +83,8 @@ class DatasetExportMixin:
             else:
                 if len(values) != _num_observations:
                     raise ValueError(
-                        "The number of observations is not consistent across columns."
+                        f"The number of observations is not consistent across columns. "
+                        f"Column '{key}' has {len(values)} observations, but previous columns had {_num_observations} observations."
                     )
 
         return _num_observations
@@ -219,7 +221,9 @@ class DatasetExportMixin:
         )
         return exporter.export()
 
-    def _db(self, remove_prefix: bool = True, shape: str = "wide") -> "sqlalchemy.engine.Engine":
+    def _db(
+        self, remove_prefix: bool = True, shape: str = "wide"
+    ) -> "sqlalchemy.engine.Engine":
         """Create a SQLite database in memory and return the connection.
 
         Args:
@@ -229,7 +233,7 @@ class DatasetExportMixin:
         Returns:
             A database connection
         >>> from sqlalchemy import text
-        >>> from edsl import Results 
+        >>> from edsl import Results
         >>> engine = Results.example()._db()
         >>> len(engine.execute(text("SELECT * FROM self")).fetchall())
         4
@@ -247,16 +251,17 @@ class DatasetExportMixin:
 
         if shape == "long":
             # Melt the dataframe to convert it to long format
-            df = df.melt(
-                var_name='key', 
-                value_name='value'
-            )
+            df = df.melt(var_name="key", value_name="value")
             # Add a row number column for reference
-            df.insert(0, 'row_number', range(1, len(df) + 1))
-            
+            df.insert(0, "row_number", range(1, len(df) + 1))
+
             # Split the key into data_type and key
-            df['data_type'] = df['key'].apply(lambda x: x.split('.')[0] if '.' in x else None)
-            df['key'] = df['key'].apply(lambda x: '.'.join(x.split('.')[1:]) if '.' in x else x)
+            df["data_type"] = df["key"].apply(
+                lambda x: x.split(".")[0] if "." in x else None
+            )
+            df["key"] = df["key"].apply(
+                lambda x: ".".join(x.split(".")[1:]) if "." in x else x
+            )
 
         df.to_sql(
             "self",
@@ -276,27 +281,27 @@ class DatasetExportMixin:
     ) -> Union["pd.DataFrame", str]:
         """Execute a SQL query and return the results as a DataFrame.
 
-        Args:
-            query: The SQL query to execute
-            shape: The shape of the data in the database (wide or long)
-            remove_prefix: Whether to remove the prefix from the column names
-            transpose: Whether to transpose the DataFrame
-            transpose_by: The column to use as the index when transposing
-            csv: Whether to return the DataFrame as a CSV string
-            to_list: Whether to return the results as a list
-            to_latex: Whether to return the results as LaTeX
-            filename: Optional filename to save the results to
-
-        Returns:
-            DataFrame, CSV string, list, or LaTeX string depending on parameters
-
-       Examples:
-           >>> from edsl import Results
-           >>> r = Results.example(); 
-           >>> len(r.sql("SELECT * FROM self", shape = "wide"))
-           4
-           >>> len(r.sql("SELECT * FROM self", shape = "long"))
-           172
+         Args:
+             query: The SQL query to execute
+             shape: The shape of the data in the database (wide or long)
+             remove_prefix: Whether to remove the prefix from the column names
+             transpose: Whether to transpose the DataFrame
+             transpose_by: The column to use as the index when transposing
+             csv: Whether to return the DataFrame as a CSV string
+             to_list: Whether to return the results as a list
+             to_latex: Whether to return the results as LaTeX
+             filename: Optional filename to save the results to
+
+         Returns:
+             DataFrame, CSV string, list, or LaTeX string depending on parameters
+
+        Examples:
+            >>> from edsl import Results
+            >>> r = Results.example();
+            >>> len(r.sql("SELECT * FROM self", shape = "wide"))
+            4
+            >>> len(r.sql("SELECT * FROM self", shape = "long"))
+            172
         """
         import pandas as pd
 
@@ -538,6 +543,116 @@ class DatasetExportMixin:
 
         if return_link:
             return filename
+        
+    def report(self, *fields: Optional[str], top_n: Optional[int] = None, 
+               header_fields: Optional[List[str]] = None, divider: bool = True,
+               return_string: bool = False) -> Optional[str]:
+        """Takes the fields in order and returns a report of the results by iterating through rows.
+        The row number is printed as # Observation: <row number>
+        The name of the field is used as markdown header at level "##" 
+        The content of that field is then printed. 
+        Then the next field and so on. 
+        Once that row is done, a new line is printed and the next row is shown.
+        If in a jupyter notebook, the report is displayed as markdown.
+        
+        Args:
+            *fields: The fields to include in the report. If none provided, all fields are used.
+            top_n: Optional limit on the number of observations to include.
+            header_fields: Optional list of fields to include in the main header instead of as sections.
+            divider: If True, adds a horizontal rule between observations for better visual separation.
+            return_string: If True, returns the markdown string. If False (default in notebooks),
+                          only displays the markdown without returning.
+            
+        Returns:
+            A string containing the markdown report if return_string is True, otherwise None.
+            
+        Examples:
+            >>> from edsl.results import Results
+            >>> r = Results.example()
+            >>> report = r.select('how_feeling', 'how_feeling_yesterday').report(return_string=True)
+            >>> "# Observation: 1" in report
+            True
+            >>> "## answer.how_feeling" in report
+            True
+            >>> report = r.select('how_feeling').report(header_fields=['answer.how_feeling'], return_string=True)
+            >>> "# Observation: 1 (`how_feeling`: OK)" in report
+            True
+        """
+        from edsl.utilities.utilities import is_notebook
+        
+        # If no fields specified, use all columns
+        if not fields:
+            fields = self.relevant_columns()
+        
+        # Initialize header_fields if not provided
+        if header_fields is None:
+            header_fields = []
+        
+        # Validate all fields
+        all_fields = list(fields) + [f for f in header_fields if f not in fields]
+        for field in all_fields:
+            if field not in self.relevant_columns():
+                raise ValueError(f"Field '{field}' not found in dataset")
+        
+        # Get data for each field
+        field_data = {}
+        for field in all_fields:
+            for entry in self:
+                if field in entry:
+                    field_data[field] = entry[field]
+                    break
+        
+        # Number of observations to process
+        num_obs = self.num_observations()
+        if top_n is not None:
+            num_obs = min(num_obs, top_n)
+        
+        # Build the report
+        report_lines = []
+        for i in range(num_obs):
+            # Create header with observation number and any header fields
+            header = f"# Observation: {i+1}"
+            if header_fields:
+                header_parts = []
+                for field in header_fields:
+                    value = field_data[field][i]
+                    # Get the field name without prefix for cleaner display
+                    display_name = field.split('.')[-1] if '.' in field else field
+                    # Format with backticks for monospace
+                    header_parts.append(f"`{display_name}`: {value}")
+                if header_parts:
+                    header += f" ({', '.join(header_parts)})"
+            report_lines.append(header)
+            
+            # Add the remaining fields
+            for field in fields:
+                if field not in header_fields:
+                    report_lines.append(f"## {field}")
+                    value = field_data[field][i]
+                    if isinstance(value, list) or isinstance(value, dict):
+                        import json
+                        report_lines.append(f"```\n{json.dumps(value, indent=2)}\n```")
+                    else:
+                        report_lines.append(str(value))
+            
+            # Add divider between observations if requested
+            if divider and i < num_obs - 1:
+                report_lines.append("\n---\n")
+            else:
+                report_lines.append("")  # Empty line between observations
+        
+        report_text = "\n".join(report_lines)
+        
+        # In notebooks, display as markdown and optionally return
+        is_nb = is_notebook()
+        if is_nb:
+            from IPython.display import Markdown, display
+            display(Markdown(report_text))
+        
+        # Return the string if requested or if not in a notebook
+        if return_string or not is_nb:
+            return report_text
+        return None
 
     def tally(
         self, *fields: Optional[str], top_n: Optional[int] = None, output="Dataset"
@@ -616,6 +731,179 @@ class DatasetExportMixin:
             keys.append("count")
             return sl.reorder_keys(keys).to_dataset()
 
+    def flatten(self, field, keep_original=False):
+        """
+        Flatten a field containing a list of dictionaries into separate fields.
+
+        >>> from edsl.results.Dataset import Dataset
+        >>> Dataset([{'a': [{'a': 1, 'b': 2}]}, {'c': [5] }]).flatten('a')
+        Dataset([{'c': [5]}, {'a.a': [1]}, {'a.b': [2]}])
+
+
+        >>> Dataset([{'answer.example': [{'a': 1, 'b': 2}]}, {'c': [5] }]).flatten('answer.example')
+        Dataset([{'c': [5]}, {'answer.example.a': [1]}, {'answer.example.b': [2]}])
+
+
+        Args:
+            field: The field to flatten
+            keep_original: If True, keeps the original field in the dataset
+
+        Returns:
+            A new dataset with the flattened fields
+        """
+        from edsl.results.Dataset import Dataset
+
+        # Ensure the dataset isn't empty
+        if not self.data:
+            return self.copy()
+        
+        # Find all columns that contain the field
+        matching_entries = []
+        for entry in self.data:
+            col_name = next(iter(entry.keys()))
+            if field == col_name or (
+                '.' in col_name and 
+                (col_name.endswith('.' + field) or col_name.startswith(field + '.'))
+            ):
+                matching_entries.append(entry)
+        
+        # Check if the field is ambiguous
+        if len(matching_entries) > 1:
+            matching_cols = [next(iter(entry.keys())) for entry in matching_entries]
+            raise ValueError(
+                f"Ambiguous field name '{field}'. It matches multiple columns: {matching_cols}. "
+                f"Please specify the full column name to flatten."
+            )
+
+        # Get the number of observations
+        num_observations = self.num_observations()
+
+        # Find the column to flatten
+        field_entry = None
+        for entry in self.data:
+            if field in entry:
+                field_entry = entry
+                break
+
+        if field_entry is None:
+            warnings.warn(
+                f"Field '{field}' not found in dataset, returning original dataset"
+            )
+            return self.copy()
+
+        # Create new dictionary for flattened data
+        flattened_data = []
+
+        # Copy all existing columns except the one we're flattening (if keep_original is False)
+        for entry in self.data:
+            col_name = next(iter(entry.keys()))
+            if col_name != field or keep_original:
+                flattened_data.append(entry.copy())
+
+        # Get field data and make sure it's valid
+        field_values = field_entry[field]
+        if not all(isinstance(item, dict) for item in field_values if item is not None):
+            warnings.warn(
+                f"Field '{field}' contains non-dictionary values that cannot be flattened"
+            )
+            return self.copy()
+
+        # Collect all unique keys across all dictionaries
+        all_keys = set()
+        for item in field_values:
+            if isinstance(item, dict):
+                all_keys.update(item.keys())
+
+        # Create new columns for each key
+        for key in sorted(all_keys):  # Sort for consistent output
+            new_values = []
+            for i in range(num_observations):
+                value = None
+                if i < len(field_values) and isinstance(field_values[i], dict):
+                    value = field_values[i].get(key, None)
+                new_values.append(value)
+
+            # Add this as a new column
+            flattened_data.append({f"{field}.{key}": new_values})
+
+        # Return a new Dataset with the flattened data
+        return Dataset(flattened_data)
+
+    def unpack_list(
+        self,
+        field: str,
+        new_names: Optional[List[str]] = None,
+        keep_original: bool = True,
+    ) -> "Dataset":
+        """Unpack list columns into separate columns with provided names or numeric suffixes.
+
+        For example, if a dataset contains:
+        [{'data': [[1, 2, 3], [4, 5, 6]], 'other': ['x', 'y']}]
+
+        After d.unpack_list('data'), it should become:
+        [{'other': ['x', 'y'], 'data_1': [1, 4], 'data_2': [2, 5], 'data_3': [3, 6]}]
+
+        Args:
+            field: The field containing lists to unpack
+            new_names: Optional list of names for the unpacked fields. If None, uses numeric suffixes.
+            keep_original: If True, keeps the original field in the dataset
+
+        Returns:
+            A new Dataset with unpacked columns
+
+        Examples:
+            >>> from edsl.results.Dataset import Dataset
+            >>> d = Dataset([{'data': [[1, 2, 3], [4, 5, 6]]}])
+            >>> d.unpack_list('data')
+            Dataset([{'data': [[1, 2, 3], [4, 5, 6]]}, {'data_1': [1, 4]}, {'data_2': [2, 5]}, {'data_3': [3, 6]}])
+
+            >>> d.unpack_list('data', new_names=['first', 'second', 'third'])
+            Dataset([{'data': [[1, 2, 3], [4, 5, 6]]}, {'first': [1, 4]}, {'second': [2, 5]}, {'third': [3, 6]}])
+        """
+        from edsl.results.Dataset import Dataset
+
+        # Create a copy of the dataset
+        result = Dataset(self.data.copy())
+
+        # Find the field in the dataset
+        field_index = None
+        for i, entry in enumerate(result.data):
+            if field in entry:
+                field_index = i
+                break
+
+        if field_index is None:
+            raise ValueError(f"Field '{field}' not found in dataset")
+
+        field_data = result.data[field_index][field]
+
+        # Check if values are lists
+        if not all(isinstance(v, list) for v in field_data):
+            raise ValueError(f"Field '{field}' does not contain lists in all entries")
+
+        # Get the maximum length of lists
+        max_len = max(len(v) for v in field_data)
+
+        # Create new fields for each index
+        for i in range(max_len):
+            if new_names and i < len(new_names):
+                new_field = new_names[i]
+            else:
+                new_field = f"{field}_{i+1}"
+
+            # Extract the i-th element from each list
+            new_values = []
+            for item in field_data:
+                new_values.append(item[i] if i < len(item) else None)
+
+            result.data.append({new_field: new_values})
+
+        # Remove the original field if keep_original is False
+        if not keep_original:
+            result.data.pop(field_index)
+
+        return result
+
 
 if __name__ == "__main__":
     import doctest