edsl 0.1.43__py3-none-any.whl → 0.1.45__py3-none-any.whl

edsl/results/Dataset.py

@@ -15,6 +15,7 @@ from edsl.Base import PersistenceMixin, HashingMixin
 
 from edsl.results.smart_objects import FirstObject
 
+from edsl.results.ResultsGGMixin import GGPlotMethod
 
 class Dataset(UserList, ResultsExportMixin, PersistenceMixin, HashingMixin):
     """A class to represent a dataset of observations."""
@@ -26,6 +27,20 @@ class Dataset(UserList, ResultsExportMixin, PersistenceMixin, HashingMixin):
         super().__init__(data)
         self.print_parameters = print_parameters
 
+
+    def ggplot2(
+        self,
+        ggplot_code: str,
+        shape="wide",
+        sql: str = None,
+        remove_prefix: bool = True,
+        debug: bool = False,
+        height=4,
+        width=6,
+        factor_orders: Optional[dict] = None,
+    ):
+        return GGPlotMethod(self).ggplot2(ggplot_code, shape, sql, remove_prefix, debug, height, width, factor_orders)
+
     def __len__(self) -> int:
         """Return the number of observations in the dataset.
 
@@ -580,6 +595,56 @@ class Dataset(UserList, ResultsExportMixin, PersistenceMixin, HashingMixin):
         result = cls([{col: df[col].tolist()} for col in df.columns])
         return result
 
+    def to_docx(self, output_file: str, title: str = None) -> None:
+        """
+        Convert the dataset to a Word document.
+        
+        Args:
+            output_file (str): Path to save the Word document
+            title (str, optional): Title for the document
+        """
+        from docx import Document
+        from docx.shared import Inches
+        from docx.enum.text import WD_ALIGN_PARAGRAPH
+
+        # Create document
+        doc = Document()
+        
+        # Add title if provided
+        if title:
+            title_heading = doc.add_heading(title, level=1)
+            title_heading.alignment = WD_ALIGN_PARAGRAPH.CENTER
+
+        # Get headers and data
+        headers, data = self._tabular()
+        
+        # Create table
+        table = doc.add_table(rows=len(data) + 1, cols=len(headers))
+        table.style = 'Table Grid'
+        
+        # Add headers
+        for j, header in enumerate(headers):
+            cell = table.cell(0, j)
+            cell.text = str(header)
+        
+        # Add data
+        for i, row in enumerate(data):
+            for j, cell_content in enumerate(row):
+                cell = table.cell(i + 1, j)
+                cell.text = str(cell_content) if cell_content is not None else ""
+        
+        # Adjust column widths
+        for column in table.columns:
+            max_width = 0
+            for cell in column.cells:
+                text_width = len(str(cell.text))
+                max_width = max(max_width, text_width)
+            for cell in column.cells:
+                cell.width = Inches(min(max_width * 0.1 + 0.5, 6))
+        
+        # Save the document
+        doc.save(output_file)
+
 
 if __name__ == "__main__":
     import doctest

edsl/results/DatasetExportMixin.py

@@ -7,6 +7,7 @@ from typing import Optional, Tuple, Union, List
 
 from edsl.results.file_exports import CSVExport, ExcelExport, JSONLExport, SQLiteExport
 
+
 class DatasetExportMixin:
     """Mixin class for exporting Dataset objects."""
 
@@ -82,7 +83,8 @@ class DatasetExportMixin:
             else:
                 if len(values) != _num_observations:
                     raise ValueError(
-                        "The number of observations is not consistent across columns."
+                        f"The number of observations is not consistent across columns. "
+                        f"Column '{key}' has {len(values)} observations, but previous columns had {_num_observations} observations."
                     )
 
         return _num_observations
@@ -219,7 +221,9 @@ class DatasetExportMixin:
         )
         return exporter.export()
 
-    def _db(self, remove_prefix: bool = True, shape: str = "wide") -> "sqlalchemy.engine.Engine":
+    def _db(
+        self, remove_prefix: bool = True, shape: str = "wide"
+    ) -> "sqlalchemy.engine.Engine":
         """Create a SQLite database in memory and return the connection.
 
         Args:
@@ -229,7 +233,7 @@ class DatasetExportMixin:
         Returns:
             A database connection
         >>> from sqlalchemy import text
-        >>> from edsl import Results 
+        >>> from edsl import Results
         >>> engine = Results.example()._db()
         >>> len(engine.execute(text("SELECT * FROM self")).fetchall())
         4
@@ -247,16 +251,17 @@ class DatasetExportMixin:
 
         if shape == "long":
             # Melt the dataframe to convert it to long format
-            df = df.melt(
-                var_name='key', 
-                value_name='value'
-            )
+            df = df.melt(var_name="key", value_name="value")
             # Add a row number column for reference
-            df.insert(0, 'row_number', range(1, len(df) + 1))
-            
+            df.insert(0, "row_number", range(1, len(df) + 1))
+
             # Split the key into data_type and key
-            df['data_type'] = df['key'].apply(lambda x: x.split('.')[0] if '.' in x else None)
-            df['key'] = df['key'].apply(lambda x: '.'.join(x.split('.')[1:]) if '.' in x else x)
+            df["data_type"] = df["key"].apply(
+                lambda x: x.split(".")[0] if "." in x else None
+            )
+            df["key"] = df["key"].apply(
+                lambda x: ".".join(x.split(".")[1:]) if "." in x else x
+            )
 
         df.to_sql(
             "self",
@@ -276,27 +281,27 @@ class DatasetExportMixin:
     ) -> Union["pd.DataFrame", str]:
         """Execute a SQL query and return the results as a DataFrame.
 
-        Args:
-            query: The SQL query to execute
-            shape: The shape of the data in the database (wide or long)
-            remove_prefix: Whether to remove the prefix from the column names
-            transpose: Whether to transpose the DataFrame
-            transpose_by: The column to use as the index when transposing
-            csv: Whether to return the DataFrame as a CSV string
-            to_list: Whether to return the results as a list
-            to_latex: Whether to return the results as LaTeX
-            filename: Optional filename to save the results to
-
-        Returns:
-            DataFrame, CSV string, list, or LaTeX string depending on parameters
-
-       Examples:
-           >>> from edsl import Results
-           >>> r = Results.example(); 
-           >>> len(r.sql("SELECT * FROM self", shape = "wide"))
-           4
-           >>> len(r.sql("SELECT * FROM self", shape = "long"))
-           172
+         Args:
+             query: The SQL query to execute
+             shape: The shape of the data in the database (wide or long)
+             remove_prefix: Whether to remove the prefix from the column names
+             transpose: Whether to transpose the DataFrame
+             transpose_by: The column to use as the index when transposing
+             csv: Whether to return the DataFrame as a CSV string
+             to_list: Whether to return the results as a list
+             to_latex: Whether to return the results as LaTeX
+             filename: Optional filename to save the results to
+
+         Returns:
+             DataFrame, CSV string, list, or LaTeX string depending on parameters
+
+        Examples:
+            >>> from edsl import Results
+            >>> r = Results.example();
+            >>> len(r.sql("SELECT * FROM self", shape = "wide"))
+            4
+            >>> len(r.sql("SELECT * FROM self", shape = "long"))
+            172
         """
         import pandas as pd
 
@@ -538,6 +543,116 @@ class DatasetExportMixin:
 
         if return_link:
             return filename
+        
+    def report(self, *fields: Optional[str], top_n: Optional[int] = None, 
+               header_fields: Optional[List[str]] = None, divider: bool = True,
+               return_string: bool = False) -> Optional[str]:
+        """Takes the fields in order and returns a report of the results by iterating through rows.
+        The row number is printed as # Observation: <row number>
+        The name of the field is used as markdown header at level "##" 
+        The content of that field is then printed. 
+        Then the next field and so on. 
+        Once that row is done, a new line is printed and the next row is shown.
+        If in a jupyter notebook, the report is displayed as markdown.
+        
+        Args:
+            *fields: The fields to include in the report. If none provided, all fields are used.
+            top_n: Optional limit on the number of observations to include.
+            header_fields: Optional list of fields to include in the main header instead of as sections.
+            divider: If True, adds a horizontal rule between observations for better visual separation.
+            return_string: If True, returns the markdown string. If False (default in notebooks),
+                          only displays the markdown without returning.
+            
+        Returns:
+            A string containing the markdown report if return_string is True, otherwise None.
+            
+        Examples:
+            >>> from edsl.results import Results
+            >>> r = Results.example()
+            >>> report = r.select('how_feeling', 'how_feeling_yesterday').report(return_string=True)
+            >>> "# Observation: 1" in report
+            True
+            >>> "## answer.how_feeling" in report
+            True
+            >>> report = r.select('how_feeling').report(header_fields=['answer.how_feeling'], return_string=True)
+            >>> "# Observation: 1 (`how_feeling`: OK)" in report
+            True
+        """
+        from edsl.utilities.utilities import is_notebook
+        
+        # If no fields specified, use all columns
+        if not fields:
+            fields = self.relevant_columns()
+        
+        # Initialize header_fields if not provided
+        if header_fields is None:
+            header_fields = []
+        
+        # Validate all fields
+        all_fields = list(fields) + [f for f in header_fields if f not in fields]
+        for field in all_fields:
+            if field not in self.relevant_columns():
+                raise ValueError(f"Field '{field}' not found in dataset")
+        
+        # Get data for each field
+        field_data = {}
+        for field in all_fields:
+            for entry in self:
+                if field in entry:
+                    field_data[field] = entry[field]
+                    break
+        
+        # Number of observations to process
+        num_obs = self.num_observations()
+        if top_n is not None:
+            num_obs = min(num_obs, top_n)
+        
+        # Build the report
+        report_lines = []
+        for i in range(num_obs):
+            # Create header with observation number and any header fields
+            header = f"# Observation: {i+1}"
+            if header_fields:
+                header_parts = []
+                for field in header_fields:
+                    value = field_data[field][i]
+                    # Get the field name without prefix for cleaner display
+                    display_name = field.split('.')[-1] if '.' in field else field
+                    # Format with backticks for monospace
+                    header_parts.append(f"`{display_name}`: {value}")
+                if header_parts:
+                    header += f" ({', '.join(header_parts)})"
+            report_lines.append(header)
+            
+            # Add the remaining fields
+            for field in fields:
+                if field not in header_fields:
+                    report_lines.append(f"## {field}")
+                    value = field_data[field][i]
+                    if isinstance(value, list) or isinstance(value, dict):
+                        import json
+                        report_lines.append(f"```\n{json.dumps(value, indent=2)}\n```")
+                    else:
+                        report_lines.append(str(value))
+            
+            # Add divider between observations if requested
+            if divider and i < num_obs - 1:
+                report_lines.append("\n---\n")
+            else:
+                report_lines.append("")  # Empty line between observations
+        
+        report_text = "\n".join(report_lines)
+        
+        # In notebooks, display as markdown and optionally return
+        is_nb = is_notebook()
+        if is_nb:
+            from IPython.display import Markdown, display
+            display(Markdown(report_text))
+        
+        # Return the string if requested or if not in a notebook
+        if return_string or not is_nb:
+            return report_text
+        return None
 
     def tally(
         self, *fields: Optional[str], top_n: Optional[int] = None, output="Dataset"
@@ -616,6 +731,158 @@ class DatasetExportMixin:
             keys.append("count")
             return sl.reorder_keys(keys).to_dataset()
 
+    def flatten(self, field, keep_original=False):
+        """
+        Flatten a field containing a list of dictionaries into separate fields.
+
+        For example, if a dataset contains:
+        [{'data': [{'a': 1}, {'b': 2}], 'other': ['x', 'y']}]
+
+        After d.flatten('data'), it should become:
+        [{'other': ['x', 'y'], 'data.a': [1, None], 'data.b': [None, 2]}]
+
+        Args:
+            field: The field to flatten
+            keep_original: If True, keeps the original field in the dataset
+
+        Returns:
+            A new dataset with the flattened fields
+        """
+        from edsl.results.Dataset import Dataset
+
+        # Ensure the dataset isn't empty
+        if not self.data:
+            return self.copy()
+
+        # Get the number of observations
+        num_observations = self.num_observations()
+
+        # Find the column to flatten
+        field_entry = None
+        for entry in self.data:
+            if field in entry:
+                field_entry = entry
+                break
+
+        if field_entry is None:
+            warnings.warn(
+                f"Field '{field}' not found in dataset, returning original dataset"
+            )
+            return self.copy()
+
+        # Create new dictionary for flattened data
+        flattened_data = []
+
+        # Copy all existing columns except the one we're flattening (if keep_original is False)
+        for entry in self.data:
+            col_name = next(iter(entry.keys()))
+            if col_name != field or keep_original:
+                flattened_data.append(entry.copy())
+
+        # Get field data and make sure it's valid
+        field_values = field_entry[field]
+        if not all(isinstance(item, dict) for item in field_values if item is not None):
+            warnings.warn(
+                f"Field '{field}' contains non-dictionary values that cannot be flattened"
+            )
+            return self.copy()
+
+        # Collect all unique keys across all dictionaries
+        all_keys = set()
+        for item in field_values:
+            if isinstance(item, dict):
+                all_keys.update(item.keys())
+
+        # Create new columns for each key
+        for key in sorted(all_keys):  # Sort for consistent output
+            new_values = []
+            for i in range(num_observations):
+                value = None
+                if i < len(field_values) and isinstance(field_values[i], dict):
+                    value = field_values[i].get(key, None)
+                new_values.append(value)
+
+            # Add this as a new column
+            flattened_data.append({f"{field}.{key}": new_values})
+
+        # Return a new Dataset with the flattened data
+        return Dataset(flattened_data)
+
+    def unpack_list(
+        self,
+        field: str,
+        new_names: Optional[List[str]] = None,
+        keep_original: bool = True,
+    ) -> "Dataset":
+        """Unpack list columns into separate columns with provided names or numeric suffixes.
+
+        For example, if a dataset contains:
+        [{'data': [[1, 2, 3], [4, 5, 6]], 'other': ['x', 'y']}]
+
+        After d.unpack_list('data'), it should become:
+        [{'other': ['x', 'y'], 'data_1': [1, 4], 'data_2': [2, 5], 'data_3': [3, 6]}]
+
+        Args:
+            field: The field containing lists to unpack
+            new_names: Optional list of names for the unpacked fields. If None, uses numeric suffixes.
+            keep_original: If True, keeps the original field in the dataset
+
+        Returns:
+            A new Dataset with unpacked columns
+
+        Examples:
+            >>> from edsl.results.Dataset import Dataset
+            >>> d = Dataset([{'data': [[1, 2, 3], [4, 5, 6]]}])
+            >>> d.unpack_list('data')
+            Dataset([{'data': [[1, 2, 3], [4, 5, 6]]}, {'data_1': [1, 4]}, {'data_2': [2, 5]}, {'data_3': [3, 6]}])
+
+            >>> d.unpack_list('data', new_names=['first', 'second', 'third'])
+            Dataset([{'data': [[1, 2, 3], [4, 5, 6]]}, {'first': [1, 4]}, {'second': [2, 5]}, {'third': [3, 6]}])
+        """
+        from edsl.results.Dataset import Dataset
+
+        # Create a copy of the dataset
+        result = Dataset(self.data.copy())
+
+        # Find the field in the dataset
+        field_index = None
+        for i, entry in enumerate(result.data):
+            if field in entry:
+                field_index = i
+                break
+
+        if field_index is None:
+            raise ValueError(f"Field '{field}' not found in dataset")
+
+        field_data = result.data[field_index][field]
+
+        # Check if values are lists
+        if not all(isinstance(v, list) for v in field_data):
+            raise ValueError(f"Field '{field}' does not contain lists in all entries")
+
+        # Get the maximum length of lists
+        max_len = max(len(v) for v in field_data)
+
+        # Create new fields for each index
+        for i in range(max_len):
+            if new_names and i < len(new_names):
+                new_field = new_names[i]
+            else:
+                new_field = f"{field}_{i+1}"
+
+            # Extract the i-th element from each list
+            new_values = []
+            for item in field_data:
+                new_values.append(item[i] if i < len(item) else None)
+
+            result.data.append({new_field: new_values})
+
+        # Remove the original field if keep_original is False
+        if not keep_original:
+            result.data.pop(field_index)
+
+        return result
+
 
 if __name__ == "__main__":
     import doctest

edsl/results/Result.py

@@ -439,6 +439,33 @@ class Result(Base, UserDict):
         from edsl.results.Results import Results
 
         return Results.example()[0]
+    
+    def score_with_answer_key(self, answer_key: dict) -> Union[int, float]:
+        """Score the result using an answer key.
+
+        :param answer_key: A dictionary that maps question_names to answers
+
+        >>> Result.example()['answer']
+        {'how_feeling': 'OK', 'how_feeling_yesterday': 'Great'}
+
+        >>> answer_key = {'how_feeling': 'OK', 'how_feeling_yesterday': 'Great'}
+        >>> Result.example().score_with_answer_key(answer_key)
+        {'correct': 2, 'incorrect': 0, 'missing': 0}
+        >>> answer_key = {'how_feeling': 'OK', 'how_feeling_yesterday': ['Great', 'Good']}
+        >>> Result.example().score_with_answer_key(answer_key)
+        {'correct': 2, 'incorrect': 0, 'missing': 0}
+        """
+        final_scores = {'correct': 0, 'incorrect': 0, 'missing': 0}
+        for question_name, answer in self.answer.items():
+            if question_name in answer_key:
+                if answer == answer_key[question_name] or answer in answer_key[question_name]:
+                    final_scores['correct'] += 1
+                else:
+                    final_scores['incorrect'] += 1
+            else:
+                final_scores['missing'] += 1
+
+        return final_scores
 
     def score(self, scoring_function: Callable) -> Union[int, float]:
         """Score the result using a passed-in scoring function.

edsl/results/Results.py

@@ -34,7 +34,7 @@ if TYPE_CHECKING:
     from simpleeval import EvalWithCompoundTypes
 
 from edsl.results.ResultsExportMixin import ResultsExportMixin
-from edsl.results.ResultsGGMixin import ResultsGGMixin
+from edsl.results.ResultsGGMixin import GGPlotMethod
 from edsl.results.results_fetch_mixin import ResultsFetchMixin
 from edsl.utilities.remove_edsl_version import remove_edsl_version
 
@@ -100,7 +100,7 @@ class NotReadyObject:
 class Mixins(
     ResultsExportMixin,
     ResultsFetchMixin,
-    ResultsGGMixin,
+#    ResultsGGMixin,
 ):
     def long(self):
         return self.table().long()
@@ -151,6 +151,19 @@ class Results(UserList, Mixins, Base):
         "cache_keys",
     ]
 
+    def ggplot2(
+        self,
+        ggplot_code: str,
+        shape="wide",
+        sql: str = None,
+        remove_prefix: bool = True,
+        debug: bool = False,
+        height=4,
+        width=6,
+        factor_orders: Optional[dict] = None,
+    ):
+        return GGPlotMethod(self).ggplot2(ggplot_code, shape, sql, remove_prefix, debug, height, width, factor_orders)
+
     @classmethod
     def from_job_info(cls, job_info: dict) -> Results:
         """
@@ -1277,6 +1290,13 @@ class Results(UserList, Mixins, Base):
         """
         return [r.score(f) for r in self.data]
     
+    def score_with_answer_key(self, answer_key: dict) -> list:
+        """Score the results using an answer key.
+
+        :param answer_key: A dictionary that maps answer values to scores.
+        """
+        return [r.score_with_answer_key(answer_key) for r in self.data]
+    
 
     def fetch_remote(self, job_info: "RemoteJobInfo") -> None:
         """
@@ -1326,7 +1346,7 @@ class Results(UserList, Mixins, Base):
         except Exception as e:
             raise ResultsError(f"Failed to fetch remote results: {str(e)}")
 
-    def fetch(self, polling_interval: float = 1.0) -> Results:
+    def fetch(self, polling_interval: [float, int] = 1.0) -> Results:
         """
         Polls the server for job completion and updates this Results instance with the completed data.
         
@@ -1346,6 +1366,7 @@ class Results(UserList, Mixins, Base):
             remote_job_data = JobsRemoteInferenceHandler.check_status(self.job_info.job_uuid)
             
             while remote_job_data.get("status") not in ["completed", "failed"]:
+                print("Waiting for remote job to complete...")
                 import time
                 time.sleep(polling_interval)
                 remote_job_data = JobsRemoteInferenceHandler.check_status(self.job_info.job_uuid)

edsl/results/ResultsGGMixin.py

@@ -75,7 +75,11 @@ class GGPlot:
                     
         return self._svg_data
 
-class ResultsGGMixin:
+class GGPlotMethod:
+
+    def __init__(self, results: 'Results'):
+        self.results = results
+    
     """Mixin class for ggplot2 plotting."""
 
     def ggplot2(
@@ -106,9 +110,9 @@ class ResultsGGMixin:
             sql = "select * from self"
 
         if shape == "long":
-            df = self.sql(sql, shape="long")
+            df = self.results.sql(sql, shape="long")
         elif shape == "wide":
-            df = self.sql(sql, remove_prefix=remove_prefix)
+            df = self.results.sql(sql, remove_prefix=remove_prefix)
 
         # Convert DataFrame to CSV format
         csv_data = df.to_csv().text

edsl/scenarios/DocumentChunker.py

@@ -85,6 +85,8 @@ class DocumentChunker:
             new_scenario = copy.deepcopy(self.scenario)
             new_scenario[field] = chunk
             new_scenario[field + "_chunk"] = i
+            new_scenario[field + "_char_count"] = len(chunk)
+            new_scenario[field + "_word_count"] = len(chunk.split())
             if include_original:
                 if hash_original:
                     new_scenario[field + "_original"] = hashlib.md5(

edsl/scenarios/FileStore.py

@@ -9,7 +9,8 @@ from edsl.scenarios.Scenario import Scenario
 from edsl.utilities.remove_edsl_version import remove_edsl_version
 
 from edsl.scenarios.file_methods import FileMethods
-
+from typing import Union
+from uuid import UUID
 
 class FileStore(Scenario):
     __documentation__ = "https://docs.expectedparrot.com/en/latest/filestore.html"
@@ -28,6 +29,12 @@ class FileStore(Scenario):
         if path is None and "filename" in kwargs:
             path = kwargs["filename"]
 
+        # Check if path is a URL and handle download
+        if path and (path.startswith('http://') or path.startswith('https://')):
+            temp_filestore = self.from_url(path, mime_type=mime_type)
+            path = temp_filestore._path
+            mime_type = temp_filestore.mime_type
+
         self._path = path  # Store the original path privately
         self._temp_path = None  # Track any generated temporary file
 
@@ -137,6 +144,10 @@ class FileStore(Scenario):
                 base64_encoded_data = base64.b64encode(binary_data)
                 self.binary = True
         # Convert the base64 bytes to a string
+        except FileNotFoundError:
+            print(f"File not found: {file_path}")
+            print("Current working directory:", os.getcwd())
+            raise
         base64_string = base64_encoded_data.decode("utf-8")
 
         return base64_string
@@ -262,7 +273,12 @@ class FileStore(Scenario):
         # raise TypeError("No text method found for this file type.")
 
     def push(
-        self, description: Optional[str] = None, visibility: str = "unlisted"
+        self, 
+        description: Optional[str] = None, 
+        alias: Optional[str] = None,
+        visibility: Optional[str] = "unlisted",
+        expected_parrot_url: Optional[str] = None,
+
     ) -> dict:
         """
         Push the object to Coop.
@@ -272,17 +288,22 @@ class FileStore(Scenario):
         scenario_version = Scenario.from_dict(self.to_dict())
         if description is None:
             description = "File: " + self.path
-        info = scenario_version.push(description=description, visibility=visibility)
+        info = scenario_version.push(description=description, visibility=visibility, expected_parrot_url=expected_parrot_url, alias=alias)
         return info
 
     @classmethod
-    def pull(cls, uuid: str, expected_parrot_url: Optional[str] = None) -> "FileStore":
+    def pull(cls, url_or_uuid: Union[str, UUID]) -> "FileStore":
         """
-        :param uuid: The UUID of the object to pull.
-        :param expected_parrot_url: The URL of the Parrot server to use.
-        :return: The object pulled from the Parrot server.
+        Pull a FileStore object from Coop.
+        
+        Args:
+            url_or_uuid: Either a UUID string or a URL pointing to the object
+            expected_parrot_url: Optional URL for the Parrot server
+        
+        Returns:
+            FileStore: The pulled FileStore object
         """
-        scenario_version = Scenario.pull(uuid, expected_parrot_url=expected_parrot_url)
+        scenario_version = Scenario.pull(url_or_uuid)
         return cls.from_dict(scenario_version.to_dict())
 
     @classmethod