edsl 0.1.33__py3-none-any.whl → 0.1.33.dev1__py3-none-any.whl

edsl/results/Results.py

@@ -17,7 +17,6 @@ from edsl.exceptions.results import (
     ResultsInvalidNameError,
     ResultsMutateError,
     ResultsFilterError,
-    ResultsDeserializationError,
 )
 
 from edsl.results.ResultsExportMixin import ResultsExportMixin
@@ -78,7 +77,6 @@ class Results(UserList, Mixins, Base):
         "question_options",
         "question_type",
         "comment",
-        "generated_tokens",
     ]
 
     def __init__(
@@ -110,81 +108,6 @@ class Results(UserList, Mixins, Base):
         if hasattr(self, "_add_output_functions"):
             self._add_output_functions()
 
-    def leaves(self):
-        leaves = []
-        for result in self:
-            leaves.extend(result.leaves())
-        return leaves
-
-    def tree(
-        self,
-        fold_attributes: Optional[List[str]] = None,
-        drop: Optional[List[str]] = None,
-        open_file=True,
-    ) -> dict:
-        """Return the results as a tree."""
-        from edsl.results.tree_explore import FoldableHTMLTableGenerator
-
-        if drop is None:
-            drop = []
-
-        valid_attributes = [
-            "model",
-            "scenario",
-            "agent",
-            "answer",
-            "question",
-            "iteration",
-        ]
-        if fold_attributes is None:
-            fold_attributes = []
-
-        for attribute in fold_attributes:
-            if attribute not in valid_attributes:
-                raise ValueError(
-                    f"Invalid fold attribute: {attribute}; must be in {valid_attributes}"
-                )
-        data = self.leaves()
-        generator = FoldableHTMLTableGenerator(data)
-        tree = generator.tree(fold_attributes=fold_attributes, drop=drop)
-        html_content = generator.generate_html(tree, fold_attributes)
-        import tempfile
-        from edsl.utilities.utilities import is_notebook
-
-        from IPython.display import display, HTML
-
-        if is_notebook():
-            import html
-            from IPython.display import display, HTML
-
-            height = 1000
-            width = 1000
-            escaped_output = html.escape(html_content)
-            # escaped_output = rendered_html
-            iframe = f""""
-            <iframe srcdoc="{ escaped_output }" style="width: {width}px; height: {height}px;"></iframe>
-            """
-            display(HTML(iframe))
-            return None
-
-        with tempfile.NamedTemporaryFile(suffix=".html", delete=False) as f:
-            f.write(html_content.encode())
-            print(f"HTML file has been generated: {f.name}")
-
-            if open_file:
-                import webbrowser
-                import time
-
-                time.sleep(1)  # Wait for 1 second
-                # webbrowser.open(f.name)
-                import os
-
-                filename = f.name
-                webbrowser.open(f"file://{os.path.abspath(filename)}")
-
-            else:
-                return html_content
-
     def code(self):
         raise NotImplementedError
 
@@ -245,9 +168,7 @@ class Results(UserList, Mixins, Base):
         )
 
     def __repr__(self) -> str:
-        import reprlib
-
-        return f"Results(data = {reprlib.repr(self.data)}, survey = {repr(self.survey)}, created_columns = {self.created_columns})"
+        return f"Results(data = {self.data}, survey = {repr(self.survey)}, created_columns = {self.created_columns})"
 
     def _repr_html_(self) -> str:
         from IPython.display import HTML
@@ -369,7 +290,8 @@ class Results(UserList, Mixins, Base):
                 ),
             )
         except Exception as e:
-            raise ResultsDeserializationError(f"Error in Results.from_dict: {e}")
+            print(e)
+            # breakpoint()
         return results
 
     ######################
@@ -473,7 +395,7 @@ class Results(UserList, Mixins, Base):
 
         >>> r = Results.example()
         >>> r.models[0]
-        Model(model_name = ...)
+        Model(model_name = 'gpt-4-1106-preview', temperature = 0.5, max_tokens = 1000, top_p = 1, frequency_penalty = 0, presence_penalty = 0, logprobs = False, top_logprobs = 3)
         """
         return [r.model for r in self.data]
 
@@ -555,6 +477,39 @@ class Results(UserList, Mixins, Base):
         )
         return sorted(list(all_keys))
 
+    def _parse_column(self, column: str) -> tuple[str, str]:
+        """
+        Parses a column name into a tuple containing a data type and a key.
+
+        >>> r = Results.example()
+        >>> r._parse_column("answer.how_feeling")
+        ('answer', 'how_feeling')
+
+        The standard way a column is specified is with a dot-separated string, e.g. _parse_column("agent.status")
+        But you can also specify a single key, e.g. "status", in which case it will look up the data type.
+        """
+        if "." in column:
+            data_type, key = column.split(".")
+        else:
+            try:
+                data_type, key = self._key_to_data_type[column], column
+            except KeyError:
+                import difflib
+
+                close_matches = difflib.get_close_matches(
+                    column, self._key_to_data_type.keys()
+                )
+                if close_matches:
+                    suggestions = ", ".join(close_matches)
+                    raise ResultsColumnNotFoundError(
+                        f"Column '{column}' not found in data. Did you mean: {suggestions}?"
+                    )
+                else:
+                    raise ResultsColumnNotFoundError(
+                        f"Column {column} not found in data"
+                    )
+        return data_type, key
+
     def first(self) -> "Result":
         """Return the first observation in the results.
 
@@ -677,11 +632,9 @@ class Results(UserList, Mixins, Base):
         """
         if functions_dict is None:
             functions_dict = {}
-        evaluator = EvalWithCompoundTypes(
+        return EvalWithCompoundTypes(
             names=result.combined_dict, functions=functions_dict
         )
-        evaluator.functions.update(int=int, float=float)
-        return evaluator
 
     def mutate(
         self, new_var_string: str, functions_dict: Optional[dict] = None
@@ -768,8 +721,8 @@ class Results(UserList, Mixins, Base):
 
     def sample(
         self,
-        n: Optional[int] = None,
-        frac: Optional[float] = None,
+        n: int = None,
+        frac: float = None,
         with_replacement: bool = True,
         seed: Optional[str] = "edsl",
     ) -> Results:
@@ -818,17 +771,13 @@ class Results(UserList, Mixins, Base):
         Dataset([{'answer.how_feeling': ['OK', 'Great', 'Terrible', 'OK']}])
 
         >>> results.select('how_feeling', 'model', 'how_feeling')
-        Dataset([{'answer.how_feeling': ['OK', 'Great', 'Terrible', 'OK']}, {'model.model': ['...', '...', '...', '...']}, {'answer.how_feeling': ['OK', 'Great', 'Terrible', 'OK']}])
-
-        >>> from edsl import Results; r = Results.example(); r.select('answer.how_feeling_y')
-        Dataset([{'answer.how_feeling_yesterday': ['Great', 'Good', 'OK', 'Terrible']}])
+        Dataset([{'answer.how_feeling': ['OK', 'Great', 'Terrible', 'OK']}, {'model.model': ['gpt-4-1106-preview', 'gpt-4-1106-preview', 'gpt-4-1106-preview', 'gpt-4-1106-preview']}, {'answer.how_feeling': ['OK', 'Great', 'Terrible', 'OK']}])
         """
 
-        # if len(self) == 0:
-        #    raise Exception("No data to select from---the Results object is empty.")
+        if len(self) == 0:
+            raise Exception("No data to select from---the Results object is empty.")
 
         if not columns or columns == ("*",) or columns == (None,):
-            # is the users passes nothing, then we'll return all the columns
             columns = ("*.*",)
 
         if isinstance(columns[0], list):
@@ -852,16 +801,6 @@ class Results(UserList, Mixins, Base):
         # iterate through the passed columns
         for column in columns:
             # a user could pass 'result.how_feeling' or just 'how_feeling'
-            matches = self._matching_columns(column)
-            if len(matches) > 1:
-                raise Exception(
-                    f"Column '{column}' is ambiguous. Did you mean one of {matches}?"
-                )
-            if len(matches) == 0 and ".*" not in column:
-                raise Exception(f"Column '{column}' not found in data.")
-            if len(matches) == 1:
-                column = matches[0]
-
             parsed_data_type, parsed_key = self._parse_column(column)
             data_types = get_data_types_to_return(parsed_data_type)
             found_once = False  # we need to track this to make sure we found the key at least once
@@ -904,21 +843,6 @@ class Results(UserList, Mixins, Base):
 
         return Dataset(sorted_new_data)
 
-    def select(self, *columns: Union[str, list[str]]) -> "Results":
-        from edsl.results.Selector import Selector
-
-        if len(self) == 0:
-            raise Exception("No data to select from---the Results object is empty.")
-
-        selector = Selector(
-            known_data_types=self.known_data_types,
-            data_type_to_keys=self._data_type_to_keys,
-            key_to_data_type=self._key_to_data_type,
-            fetch_list_func=self._fetch_list,
-            columns=self.columns,
-        )
-        return selector.select(*columns)
-
     def sort_by(self, *columns: str, reverse: bool = False) -> Results:
         import warnings
 
@@ -927,11 +851,6 @@ class Results(UserList, Mixins, Base):
         )
         return self.order_by(*columns, reverse=reverse)
 
-    def _parse_column(self, column: str) -> tuple[str, str]:
-        if "." in column:
-            return column.split(".")
-        return self._key_to_data_type[column], column
-
     def order_by(self, *columns: str, reverse: bool = False) -> Results:
         """Sort the results by one or more columns.
 
@@ -1029,9 +948,7 @@ class Results(UserList, Mixins, Base):
         def has_single_equals(string):
             if "!=" in string:
                 return False
-            if "=" in string and not (
-                "==" in string or "<=" in string or ">=" in string
-            ):
+            if "=" in string and not "==" in string:
                 return True
 
         if has_single_equals(expression):
@@ -1072,7 +989,7 @@ class Results(UserList, Mixins, Base):
         return Results(survey=self.survey, data=new_data, created_columns=None)
 
     @classmethod
-    def example(cls, randomize: bool = False) -> Results:
+    def example(cls, debug: bool = False, randomize: bool = False) -> Results:
         """Return an example `Results` object.
 
         Example usage:
@@ -1086,13 +1003,7 @@ class Results(UserList, Mixins, Base):
 
         c = Cache()
         job = Jobs.example(randomize=randomize)
-        results = job.run(
-            cache=c,
-            stop_on_exception=True,
-            skip_retry=True,
-            raise_validation_errors=True,
-            disable_remote_inference=True,
-        )
+        results = job.run(cache=c, debug=debug)
         return results
 
     def rich_print(self):

edsl/results/ResultsDBMixin.py

@@ -136,9 +136,9 @@ class ResultsDBMixin:
 
         >>> from edsl.results import Results
         >>> r = Results.example()
-        >>> d = r.sql("select data_type, key, value from self where data_type = 'answer' order by value limit 3", shape="long")
-        >>> sorted(list(d['value']))
-        ['Good', 'Great', 'Great']
+        >>> d = r.sql("select data_type, key, value from self where data_type = 'answer' limit 3", shape="long")
+        >>> list(d['value'])
+        ['OK', 'This is a real survey response from a human.', 'Great']
 
         We can also return the data in wide format.
         Note the use of single quotes to escape the column names, as required by sql.

edsl/scenarios/FileStore.py

@@ -120,22 +120,14 @@ class FileStore(Scenario):
         return info
 
     @classmethod
-    def pull(cls, uuid, expected_parrot_url: Optional[str] = None):
-        scenario_version = Scenario.pull(uuid, expected_parrot_url=expected_parrot_url)
+    def pull(cls, uuid):
+        scenario_version = Scenario.pull(uuid)
         return cls.from_dict(scenario_version.to_dict())
 
 
 class CSVFileStore(FileStore):
-    def __init__(
-        self,
-        filename,
-        binary: Optional[bool] = None,
-        suffix: Optional[str] = None,
-        base64_string: Optional[str] = None,
-    ):
-        super().__init__(
-            filename, binary=binary, base64_string=base64_string, suffix=".csv"
-        )
+    def __init__(self, filename):
+        super().__init__(filename, suffix=".csv")
 
     @classmethod
     def example(cls):
@@ -155,16 +147,8 @@ class CSVFileStore(FileStore):
 
 
 class PDFFileStore(FileStore):
-    def __init__(
-        self,
-        filename,
-        binary: Optional[bool] = None,
-        suffix: Optional[str] = None,
-        base64_string: Optional[str] = None,
-    ):
-        super().__init__(
-            filename, binary=binary, base64_string=base64_string, suffix=".pdf"
-        )
+    def __init__(self, filename):
+        super().__init__(filename, suffix=".pdf")
 
     def view(self):
         pdf_path = self.to_tempfile()
@@ -241,16 +225,8 @@ class PDFFileStore(FileStore):
 
 
 class PNGFileStore(FileStore):
-    def __init__(
-        self,
-        filename,
-        binary: Optional[bool] = None,
-        suffix: Optional[str] = None,
-        base64_string: Optional[str] = None,
-    ):
-        super().__init__(
-            filename, binary=binary, base64_string=base64_string, suffix=".png"
-        )
+    def __init__(self, filename):
+        super().__init__(filename, suffix=".png")
 
     @classmethod
     def example(cls):
@@ -275,16 +251,8 @@ class PNGFileStore(FileStore):
 
 
 class SQLiteFileStore(FileStore):
-    def __init__(
-        self,
-        filename,
-        binary: Optional[bool] = None,
-        suffix: Optional[str] = None,
-        base64_string: Optional[str] = None,
-    ):
-        super().__init__(
-            filename, binary=binary, base64_string=base64_string, suffix=".sqlite"
-        )
+    def __init__(self, filename):
+        super().__init__(filename, suffix=".sqlite")
 
     @classmethod
     def example(cls):
@@ -297,8 +265,6 @@ class SQLiteFileStore(FileStore):
             c.execute("""CREATE TABLE stocks (date text)""")
             conn.commit()
 
-            return cls(f.name)
-
     def view(self):
         import subprocess
         import os
@@ -307,33 +273,6 @@ class SQLiteFileStore(FileStore):
         os.system(f"sqlite3 {sqlite_path}")
 
 
-class HTMLFileStore(FileStore):
-    def __init__(
-        self,
-        filename,
-        binary: Optional[bool] = None,
-        suffix: Optional[str] = None,
-        base64_string: Optional[str] = None,
-    ):
-        super().__init__(
-            filename, binary=binary, base64_string=base64_string, suffix=".html"
-        )
-
-    @classmethod
-    def example(cls):
-        import tempfile
-
-        with tempfile.NamedTemporaryFile(suffix=".html", delete=False) as f:
-            f.write("<html><body><h1>Test</h1></body></html>".encode())
-            return cls(f.name)
-
-    def view(self):
-        import webbrowser
-
-        html_path = self.to_tempfile()
-        webbrowser.open("file://" + html_path)
-
-
 if __name__ == "__main__":
     # file_path = "../conjure/examples/Ex11-2.sav"
     # fs = FileStore(file_path)

edsl/scenarios/Scenario.py

@@ -5,10 +5,6 @@ import copy
 import base64
 import hashlib
 import os
-import reprlib
-import imghdr
-
-
 from collections import UserDict
 from typing import Union, List, Optional, Generator
 from uuid import uuid4
@@ -17,8 +13,6 @@ from edsl.scenarios.ScenarioImageMixin import ScenarioImageMixin
 from edsl.scenarios.ScenarioHtmlMixin import ScenarioHtmlMixin
 from edsl.utilities.decorators import add_edsl_version, remove_edsl_version
 
-from edsl.data_transfer_models import ImageInfo
-
 
 class Scenario(Base, UserDict, ScenarioImageMixin, ScenarioHtmlMixin):
     """A Scenario is a dictionary of keys/values.
@@ -55,39 +49,6 @@ class Scenario(Base, UserDict, ScenarioImageMixin, ScenarioHtmlMixin):
             self._has_image = False
         return self._has_image
 
-    @property
-    def has_jinja_braces(self) -> bool:
-        """Return whether the scenario has jinja braces. This matters for rendering.
-
-        >>> s = Scenario({"food": "I love {{wood chips}}"})
-        >>> s.has_jinja_braces
-        True
-        """
-        for key, value in self.items():
-            if "{{" in str(value) and "}}" in value:
-                return True
-        return False
-
-    def convert_jinja_braces(
-        self, replacement_left="<<", replacement_right=">>"
-    ) -> Scenario:
-        """Convert Jinja braces to some other character.
-
-        >>> s = Scenario({"food": "I love {{wood chips}}"})
-        >>> s.convert_jinja_braces()
-        Scenario({'food': 'I love <<wood chips>>'})
-
-        """
-        new_scenario = Scenario()
-        for key, value in self.items():
-            if isinstance(value, str):
-                new_scenario[key] = value.replace("{{", replacement_left).replace(
-                    "}}", replacement_right
-                )
-            else:
-                new_scenario[key] = value
-        return new_scenario
-
     @has_image.setter
     def has_image(self, value):
         self._has_image = value
@@ -181,7 +142,6 @@ class Scenario(Base, UserDict, ScenarioImageMixin, ScenarioHtmlMixin):
         print_json(json.dumps(self.to_dict()))
 
     def __repr__(self):
-        # return "Scenario(" + reprlib.repr(self.data) + ")"
         return "Scenario(" + repr(self.data) + ")"
 
     def _repr_html_(self):
@@ -236,54 +196,33 @@ class Scenario(Base, UserDict, ScenarioImageMixin, ScenarioHtmlMixin):
         return cls({"url": url, field_name: text})
 
     @classmethod
-    def from_image(
-        cls, image_path: str, image_name: Optional[str] = None
-    ) -> "Scenario":
-        """
-        Creates a scenario with a base64 encoding of an image.
-
-        Args:
-            image_path (str): Path to the image file.
-
-        Returns:
-            Scenario: A new Scenario instance with image information.
+    def from_image(cls, image_path: str) -> str:
+        """Creates a scenario with a base64 encoding of an image.
 
         Example:
+
         >>> s = Scenario.from_image(Scenario.example_image())
         >>> s
-        Scenario({'logo': ...})
+        Scenario({'file_path': '...', 'encoded_image': '...'})
         """
-        if not os.path.exists(image_path):
-            raise FileNotFoundError(f"Image file not found: {image_path}")
-
         with open(image_path, "rb") as image_file:
-            file_content = image_file.read()
-
-        file_name = os.path.basename(image_path)
-        file_size = os.path.getsize(image_path)
-        image_format = imghdr.what(image_path) or "unknown"
-
-        if image_name is None:
-            image_name = file_name.split(".")[0]
-
-        image_info = ImageInfo(
-            file_path=image_path,
-            file_name=file_name,
-            image_format=image_format,
-            file_size=file_size,
-            encoded_image=base64.b64encode(file_content).decode("utf-8"),
-        )
-
-        scenario_data = {image_name: image_info}
-        s = cls(scenario_data)
-        s.has_image = True
-        return s
+            s = cls(
+                {
+                    "file_path": image_path,
+                    "encoded_image": base64.b64encode(image_file.read()).decode(
+                        "utf-8"
+                    ),
+                }
+            )
+            s.has_image = True
+            return s
 
     @classmethod
     def from_pdf(cls, pdf_path):
-        # Ensure the file exists
-        import fitz
+        import fitz  # PyMuPDF
+        from edsl import Scenario
 
+        # Ensure the file exists
         if not os.path.exists(pdf_path):
             raise FileNotFoundError(f"The file {pdf_path} does not exist.")
 
@@ -297,14 +236,7 @@ class Scenario(Base, UserDict, ScenarioImageMixin, ScenarioHtmlMixin):
         text = ""
         for page_num in range(len(document)):
             page = document.load_page(page_num)
-            blocks = page.get_text("blocks")  # Extract text blocks
-
-            # Sort blocks by their vertical position (y0) to maintain reading order
-            blocks.sort(key=lambda b: (b[1], b[0]))  # Sort by y0 first, then x0
-
-            # Combine the text blocks in order
-            for block in blocks:
-                text += block[4] + "\n"
+            text = text + page.get_text()
 
         # Create a dictionary for the combined text
         page_info = {"filename": filename, "text": text}
@@ -491,21 +423,18 @@ class Scenario(Base, UserDict, ScenarioImageMixin, ScenarioHtmlMixin):
         return table
 
     @classmethod
-    def example(cls, randomize: bool = False, has_image=False) -> Scenario:
+    def example(cls, randomize: bool = False) -> Scenario:
         """
         Returns an example Scenario instance.
 
         :param randomize: If True, adds a random string to the value of the example key.
         """
-        if not has_image:
-            addition = "" if not randomize else str(uuid4())
-            return cls(
-                {
-                    "persona": f"A reseacher studying whether LLMs can be used to generate surveys.{addition}",
-                }
-            )
-        else:
-            return cls.from_image(cls.example_image())
+        addition = "" if not randomize else str(uuid4())
+        return cls(
+            {
+                "persona": f"A reseacher studying whether LLMs can be used to generate surveys.{addition}",
+            }
+        )
 
     def code(self) -> List[str]:
         """Return the code for the scenario."""

edsl/scenarios/ScenarioImageMixin.py

@@ -13,7 +13,7 @@ class ScenarioImageMixin:
         >>> from edsl.scenarios.Scenario import Scenario
         >>> s = Scenario({"food": "wood chips"})
         >>> s.add_image(Scenario.example_image())
-        Scenario({'food': 'wood chips', 'logo': ...})
+        Scenario({'food': 'wood chips', 'file_path': '...', 'encoded_image': '...'})
         """
         new_scenario = self.from_image(image_path)
         return self + new_scenario
@@ -33,7 +33,7 @@ class ScenarioImageMixin:
         >>> from edsl.scenarios.Scenario import Scenario
         >>> s = Scenario.from_image(Scenario.example_image())
         >>> s
-        Scenario({'logo': ...})
+        Scenario({'file_path': '...', 'encoded_image': '...'})
         """
 
         if image_path.startswith("http://") or image_path.startswith("https://"):