edsl 0.1.29.dev3__py3-none-any.whl → 0.1.30__py3-none-any.whl

edsl/results/Result.py

@@ -3,16 +3,7 @@ from __future__ import annotations
 from collections import UserDict
 from typing import Any, Type, Callable, Optional
 from collections import UserDict
-
-from rich.table import Table
-
-from IPython.display import display
-
-from edsl.agents import Agent
-from edsl.language_models import LanguageModel
-from edsl.scenarios import Scenario
 from edsl.Base import Base
-from edsl.prompts import Prompt
 from edsl.utilities.decorators import add_edsl_version, remove_edsl_version
 
 
@@ -21,6 +12,8 @@ class PromptDict(UserDict):
 
     def rich_print(self):
         """Display an object as a table."""
+        from rich.table import Table
+
         table = Table(title="")
         table.add_column("Attribute", style="bold")
         table.add_column("Value")
@@ -71,9 +64,9 @@ class Result(Base, UserDict):
 
     def __init__(
         self,
-        agent: Agent,
-        scenario: Scenario,
-        model: Type[LanguageModel],
+        agent: "Agent",
+        scenario: "Scenario",
+        model: Type["LanguageModel"],
         iteration: int,
         answer: str,
         prompt: dict[str, str] = None,
@@ -133,6 +126,9 @@ class Result(Base, UserDict):
         self.survey = survey
         self.question_to_attributes = question_to_attributes
 
+        self._combined_dict = None
+        self._problem_keys = None
+
     ###############
     # Used in Results
     ###############
@@ -171,25 +167,64 @@ class Result(Base, UserDict):
             "answer": self.answer,
             "prompt": self.prompt,
             "raw_model_response": self.raw_model_response,
-            "iteration": {"iteration": self.iteration},
+            #            "iteration": {"iteration": self.iteration},
             "question_text": question_text_dict,
             "question_options": question_options_dict,
             "question_type": question_type_dict,
             "comment": comments_dict,
         }
 
+    def check_expression(self, expression) -> None:
+        for key in self.problem_keys:
+            if key in expression and not key + "." in expression:
+                raise ValueError(
+                    f"Key by iself {key} is problematic. Use the full key {key + '.' + key} name instead."
+                )
+        return None
+
     def code(self):
         """Return a string of code that can be used to recreate the Result object."""
         raise NotImplementedError
 
     @property
-    def combined_dict(self) -> dict[str, Any]:
-        """Return a dictionary that includes all sub_dicts, but also puts the key-value pairs in each sub_dict as a key_value pair in the combined dictionary."""
+    def problem_keys(self):
+        """Return a list of keys that are problematic."""
+        return self._problem_keys
+
+    def _compute_combined_dict_and_problem_keys(self) -> None:
         combined = {}
+        problem_keys = []
         for key, sub_dict in self.sub_dicts.items():
             combined.update(sub_dict)
+            # in some cases, the sub_dict might have keys that conflict with the main dict
+            if key in combined:
+                # The key is already in the combined dict
+                problem_keys = problem_keys + [key]
+
             combined.update({key: sub_dict})
-        return combined
+            # I *think* this allows us to do do things like "answer.how_feelling" i.e., that the evaluator can use
+            # dot notation to access the subdicts.
+        self._combined_dict = combined
+        self._problem_keys = problem_keys
+
+    @property
+    def combined_dict(self) -> dict[str, Any]:
+        """Return a dictionary that includes all sub_dicts, but also puts the key-value pairs in each sub_dict as a key_value pair in the combined dictionary.
+
+        >>> r = Result.example()
+        >>> r.combined_dict['how_feeling']
+        'OK'
+        """
+        if self._combined_dict is None or self._problem_keys is None:
+            self._compute_combined_dict_and_problem_keys()
+        return self._combined_dict
+
+    @property
+    def problem_keys(self):
+        """Return a list of keys that are problematic."""
+        if self._combined_dict is None or self._problem_keys is None:
+            self._compute_combined_dict_and_problem_keys()
+        return self._problem_keys
 
     def get_value(self, data_type: str, key: str) -> Any:
         """Return the value for a given data type and key.
@@ -233,7 +268,13 @@ class Result(Base, UserDict):
         return Result.from_dict(self.to_dict())
 
     def __eq__(self, other) -> bool:
-        """Return True if the Result object is equal to another Result object."""
+        """Return True if the Result object is equal to another Result object.
+
+        >>> r = Result.example()
+        >>> r == r
+        True
+
+        """
         return self.to_dict() == other.to_dict()
 
     ###############
@@ -278,6 +319,12 @@ class Result(Base, UserDict):
     @remove_edsl_version
     def from_dict(self, json_dict: dict) -> Result:
         """Return a Result object from a dictionary representation."""
+
+        from edsl import Agent
+        from edsl import Scenario
+        from edsl.language_models.LanguageModel import LanguageModel
+        from edsl.prompts.Prompt import Prompt
+
         prompt_data = json_dict.get("prompt", {})
         prompt_d = {}
         for prompt_name, prompt_obj in prompt_data.items():
@@ -301,6 +348,7 @@ class Result(Base, UserDict):
         """Display an object as a table."""
         # from edsl.utilities import print_dict_with_rich
         from rich import print
+        from rich.table import Table
 
         table = Table(title="Result")
         table.add_column("Attribute", style="bold")
@@ -325,7 +373,7 @@ class Result(Base, UserDict):
     @classmethod
     def example(cls):
         """Return an example Result object."""
-        from edsl.results import Results
+        from edsl.results.Results import Results
 
         return Results.example()[0]
 
@@ -350,59 +398,7 @@ class Result(Base, UserDict):
         return scoring_function(**params)
 
 
-def main():
-    """Run the main function."""
-    from edsl.results.Result import Result
-    import json
-
-    print("Being imported")
-    json_string = """
-    {
-        "agent": {
-            "traits": {
-                "status": "Unhappy"
-            }
-        },
-        "scenario": {
-            "period": "morning"
-        },
-        "model": {
-            "model": "gpt-3.5-turbo",
-            "parameters": {
-                "temperature": 0.5,
-                "max_tokens": 1000,
-                "top_p": 1,
-                "frequency_penalty": 0,
-                "presence_penalty": 0,
-                "use_cache": true
-            }
-        },
-        "iteration": 0,
-        "answer": {
-            "how_feeling": "Bad"
-        }, 
-        "prompt": {"how_feeling_user_prompt": "How are you feeling today?", "how_feeling_system_prompt": "Answer the question"}
-    }
-    """
-
-    result = Result.from_dict(json.loads(json_string))
-
-    result.sub_dicts
-    assert result.combined_dict["how_feeling"] == "Bad"
-
-    result.combined_dict
-    assert result.get_value("answer", "how_feeling") == "Bad"
-
-    result.key_to_data_type
-    print(result)
-
-    assert result == result.copy()
-
-    result.to_dict()
-
-
 if __name__ == "__main__":
-    # print(Result.example())
     import doctest
 
     doctest.testmod(optionflags=doctest.ELLIPSIS)

edsl/results/Results.py

@@ -5,16 +5,10 @@ It is not typically instantiated directly, but is returned by the run method of
 
 from __future__ import annotations
 import json
-import hashlib
 import random
 from collections import UserList, defaultdict
 from typing import Optional, Callable, Any, Type, Union, List
 
-from pygments import highlight
-from pygments.lexers import JsonLexer
-from pygments.formatters import HtmlFormatter
-from IPython.display import HTML
-
 from simpleeval import EvalWithCompoundTypes
 
 from edsl.exceptions.results import (
@@ -24,29 +18,17 @@ from edsl.exceptions.results import (
     ResultsMutateError,
     ResultsFilterError,
 )
-from edsl.agents import Agent, AgentList
-from edsl.language_models.LanguageModel import LanguageModel
-from edsl.results.Dataset import Dataset
-from edsl.results.Result import Result
+
 from edsl.results.ResultsExportMixin import ResultsExportMixin
-from edsl.scenarios import Scenario
-
-# from edsl.scenarios.ScenarioList import ScenarioList
-from edsl.surveys import Survey
-from edsl.data.Cache import Cache
-from edsl.utilities import (
-    is_valid_variable_name,
-    shorten_string,
-)
-from edsl.utilities.decorators import add_edsl_version, remove_edsl_version
-from edsl.utilities.utilities import dict_hash
 from edsl.results.ResultsToolsMixin import ResultsToolsMixin
-
 from edsl.results.ResultsDBMixin import ResultsDBMixin
 from edsl.results.ResultsGGMixin import ResultsGGMixin
+from edsl.results.ResultsFetchMixin import ResultsFetchMixin
+
+from edsl.utilities.decorators import add_edsl_version, remove_edsl_version
+from edsl.utilities.utilities import dict_hash
 
 from edsl.Base import Base
-from edsl.results.ResultsFetchMixin import ResultsFetchMixin
 
 
 class Mixins(
@@ -56,7 +38,22 @@ class Mixins(
     ResultsGGMixin,
     ResultsToolsMixin,
 ):
-    pass
+    def print_long(self, max_rows=None) -> None:
+        """Print the results in long format.
+
+        >>> from edsl.results import Results
+        >>> r = Results.example()
+        >>> r.select('how_feeling').print_long(max_rows = 2)
+        ┏━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━┓
+        ┃ Result index ┃ Key         ┃ Value ┃
+        ┡━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━┩
+        │ 0            │ how_feeling │ OK    │
+        │ 1            │ how_feeling │ Great │
+        └──────────────┴─────────────┴───────┘
+        """
+        from edsl.utilities.interface import print_results_long
+
+        print_results_long(self, max_rows=max_rows)
 
 
 class Results(UserList, Mixins, Base):
@@ -84,10 +81,10 @@ class Results(UserList, Mixins, Base):
 
     def __init__(
         self,
-        survey: Optional[Survey] = None,
-        data: Optional[list[Result]] = None,
+        survey: Optional["Survey"] = None,
+        data: Optional[list["Result"]] = None,
         created_columns: Optional[list[str]] = None,
-        cache: Optional[Cache] = None,
+        cache: Optional["Cache"] = None,
         job_uuid: Optional[str] = None,
         total_results: Optional[int] = None,
     ):
@@ -100,6 +97,8 @@ class Results(UserList, Mixins, Base):
         :param total_results: An integer representing the total number of results.
         """
         super().__init__(data)
+        from edsl.data.Cache import Cache
+
         self.survey = survey
         self.created_columns = created_columns or []
         self._job_uuid = job_uuid
@@ -125,6 +124,10 @@ class Results(UserList, Mixins, Base):
         raise TypeError("Invalid argument type")
 
     def _update_results(self) -> None:
+        from edsl import Agent, Scenario
+        from edsl.language_models import LanguageModel
+        from edsl.results import Result
+
         if self._job_uuid and len(self.data) < self._total_results:
             results = [
                 Result(
@@ -168,7 +171,13 @@ class Results(UserList, Mixins, Base):
         return f"Results(data = {self.data}, survey = {repr(self.survey)}, created_columns = {self.created_columns})"
 
     def _repr_html_(self) -> str:
+        from IPython.display import HTML
+
         json_str = json.dumps(self.to_dict()["data"], indent=4)
+        from pygments import highlight
+        from pygments.lexers import JsonLexer
+        from pygments.formatters import HtmlFormatter
+
         formatted_json = highlight(
             json_str,
             JsonLexer(),
@@ -177,6 +186,8 @@ class Results(UserList, Mixins, Base):
         return HTML(formatted_json).data
 
     def _to_dict(self, sort=False):
+        from edsl.data.Cache import Cache
+
         if sort:
             data = sorted([result for result in self.data], key=lambda x: hash(x))
         else:
@@ -226,6 +237,31 @@ class Results(UserList, Mixins, Base):
     def hashes(self) -> set:
         return set(hash(result) for result in self.data)
 
+    def sample(self, n: int) -> "Results":
+        """Return a random sample of the results.
+
+        :param n: The number of samples to return.
+
+        >>> from edsl.results import Results
+        >>> r = Results.example()
+        >>> len(r.sample(2))
+        2
+        """
+        indices = None
+
+        for entry in self:
+            key, values = list(entry.items())[0]
+            if indices is None:  # gets the indices for the first time
+                indices = list(range(len(values)))
+                sampled_indices = random.sample(indices, n)
+                if n > len(indices):
+                    raise ValueError(
+                        f"Cannot sample {n} items from a list of length {len(indices)}."
+                    )
+            entry[key] = [values[i] for i in sampled_indices]
+
+        return self
+
     @classmethod
     @remove_edsl_version
     def from_dict(cls, data: dict[str, Any]) -> Results:
@@ -241,12 +277,21 @@ class Results(UserList, Mixins, Base):
         >>> r == r2
         True
         """
-        results = cls(
-            survey=Survey.from_dict(data["survey"]),
-            data=[Result.from_dict(r) for r in data["data"]],
-            created_columns=data.get("created_columns", None),
-            cache=Cache.from_dict(data.get("cache")) if "cache" in data else Cache(),
-        )
+        from edsl import Survey, Cache
+        from edsl.results.Result import Result
+
+        try:
+            results = cls(
+                survey=Survey.from_dict(data["survey"]),
+                data=[Result.from_dict(r) for r in data["data"]],
+                created_columns=data.get("created_columns", None),
+                cache=(
+                    Cache.from_dict(data.get("cache")) if "cache" in data else Cache()
+                ),
+            )
+        except Exception as e:
+            print(e)
+            # breakpoint()
         return results
 
     ######################
@@ -313,6 +358,8 @@ class Results(UserList, Mixins, Base):
         >>> r.answer_keys
         {'how_feeling': 'How are you this {{ period }}?', 'how_feeling_yesterday': 'How were you feeling yesterday {{ period }}?'}
         """
+        from edsl.utilities.utilities import shorten_string
+
         if not self.survey:
             raise Exception("Survey is not defined so no answer keys are available.")
 
@@ -327,7 +374,7 @@ class Results(UserList, Mixins, Base):
         return sorted_dict
 
     @property
-    def agents(self) -> AgentList:
+    def agents(self) -> "AgentList":
         """Return a list of all of the agents in the Results.
 
         Example:
@@ -336,10 +383,12 @@ class Results(UserList, Mixins, Base):
         >>> r.agents
         AgentList([Agent(traits = {'status': 'Joyful'}), Agent(traits = {'status': 'Joyful'}), Agent(traits = {'status': 'Sad'}), Agent(traits = {'status': 'Sad'})])
         """
+        from edsl import AgentList
+
         return AgentList([r.agent for r in self.data])
 
     @property
-    def models(self) -> list[Type[LanguageModel]]:
+    def models(self) -> list[Type["LanguageModel"]]:
         """Return a list of all of the models in the Results.
 
         Example:
@@ -461,7 +510,7 @@ class Results(UserList, Mixins, Base):
                     )
         return data_type, key
 
-    def first(self) -> Result:
+    def first(self) -> "Result":
         """Return the first observation in the results.
 
         Example:
@@ -555,6 +604,38 @@ class Results(UserList, Mixins, Base):
             self = self.add_column(key, values)
         return self
 
+    @staticmethod
+    def _create_evaluator(
+        result: Result, functions_dict: Optional[dict] = None
+    ) -> EvalWithCompoundTypes:
+        """Create an evaluator for the expression.
+
+        >>> from unittest.mock import Mock
+        >>> result = Mock()
+        >>> result.combined_dict = {'how_feeling': 'OK'}
+
+        >>> evaluator = Results._create_evaluator(result = result, functions_dict = {})
+        >>> evaluator.eval("how_feeling == 'OK'")
+        True
+
+        >>> result.combined_dict = {'answer': {'how_feeling': 'OK'}}
+        >>> evaluator = Results._create_evaluator(result = result, functions_dict = {})
+        >>> evaluator.eval("answer.how_feeling== 'OK'")
+        True
+
+        Note that you need to refer to the answer dictionary in the expression.
+
+        >>> evaluator.eval("how_feeling== 'OK'")
+        Traceback (most recent call last):
+        ...
+        simpleeval.NameNotDefined: 'how_feeling' is not defined for expression 'how_feeling== 'OK''
+        """
+        if functions_dict is None:
+            functions_dict = {}
+        return EvalWithCompoundTypes(
+            names=result.combined_dict, functions=functions_dict
+        )
+
     def mutate(
         self, new_var_string: str, functions_dict: Optional[dict] = None
     ) -> Results:
@@ -579,19 +660,16 @@ class Results(UserList, Mixins, Base):
             )
         raw_var_name, expression = new_var_string.split("=", 1)
         var_name = raw_var_name.strip()
+        from edsl.utilities.utilities import is_valid_variable_name
+
         if not is_valid_variable_name(var_name):
             raise ResultsInvalidNameError(f"{var_name} is not a valid variable name.")
 
         # create the evaluator
         functions_dict = functions_dict or {}
 
-        def create_evaluator(result: Result) -> EvalWithCompoundTypes:
-            return EvalWithCompoundTypes(
-                names=result.combined_dict, functions=functions_dict
-            )
-
-        def new_result(old_result: Result, var_name: str) -> Result:
-            evaluator = create_evaluator(old_result)
+        def new_result(old_result: "Result", var_name: str) -> "Result":
+            evaluator = self._create_evaluator(old_result, functions_dict)
             value = evaluator.eval(expression)
             new_result = old_result.copy()
             new_result["answer"][var_name] = value
@@ -680,7 +758,7 @@ class Results(UserList, Mixins, Base):
 
         return Results(survey=self.survey, data=new_data, created_columns=None)
 
-    def select(self, *columns: Union[str, list[str]]) -> Dataset:
+    def select(self, *columns: Union[str, list[str]]) -> "Dataset":
         """
         Select data from the results and format it.
 
@@ -691,7 +769,11 @@ class Results(UserList, Mixins, Base):
         >>> results = Results.example()
         >>> results.select('how_feeling')
         Dataset([{'answer.how_feeling': ['OK', 'Great', 'Terrible', 'OK']}])
+
+        >>> results.select('how_feeling', 'model', 'how_feeling')
+        Dataset([{'answer.how_feeling': ['OK', 'Great', 'Terrible', 'OK']}, {'model.model': ['gpt-4-1106-preview', 'gpt-4-1106-preview', 'gpt-4-1106-preview', 'gpt-4-1106-preview']}, {'answer.how_feeling': ['OK', 'Great', 'Terrible', 'OK']}])
         """
+
         if len(self) == 0:
             raise Exception("No data to select from---the Results object is empty.")
 
@@ -747,9 +829,19 @@ class Results(UserList, Mixins, Base):
             # Return the index of this key in the list_of_keys
             return items_in_order.index(single_key)
 
-        sorted(new_data, key=sort_by_key_order)
+        # sorted(new_data, key=sort_by_key_order)
+        from edsl.results.Dataset import Dataset
+
+        sorted_new_data = []
 
-        return Dataset(new_data)
+        # WORKS but slow
+        for key in items_in_order:
+            for d in new_data:
+                if key in d:
+                    sorted_new_data.append(d)
+                    break
+
+        return Dataset(sorted_new_data)
 
     def sort_by(self, *columns: str, reverse: bool = False) -> Results:
         import warnings
@@ -864,29 +956,29 @@ class Results(UserList, Mixins, Base):
                 "You must use '==' instead of '=' in the filter expression."
             )
 
-        def create_evaluator(result):
-            """Create an evaluator for the given result.
-            The 'combined_dict' is a mapping of all values for that Result object.
-            """
-            return EvalWithCompoundTypes(names=result.combined_dict)
-
         try:
             # iterates through all the results and evaluates the expression
-            new_data = [
-                result
-                for result in self.data
-                if create_evaluator(result).eval(expression)
-            ]
+            new_data = []
+            for result in self.data:
+                evaluator = self._create_evaluator(result)
+                result.check_expression(expression)  # check expression
+                if evaluator.eval(expression):
+                    new_data.append(result)
+
+        except ValueError as e:
+            raise ResultsFilterError(
+                f"Error in filter. Exception:{e}",
+                f"The expression you provided was: {expression}",
+                "See https://docs.expectedparrot.com/en/latest/results.html#filtering-results for more details.",
+            )
         except Exception as e:
             raise ResultsFilterError(
-                f"""Error in filter. Exception:{e}.
-            The expression you provided was: {expression}. 
-            Please make sure that the expression is a valid Python expression that evaluates to a boolean.
-            For example, 'how_feeling == "Great"' is a valid expression, as is 'how_feeling in ["Great", "Terrible"]'.
-            However, 'how_feeling = "Great"' is not a valid expression.
-
-            See https://docs.expectedparrot.com/en/latest/results.html#filtering-results for more details.
-            """
+                f"""Error in filter. Exception:{e}.""",
+                f"""The expression you provided was: {expression}.""",
+                """Please make sure that the expression is a valid Python expression that evaluates to a boolean.""",
+                """For example, 'how_feeling == "Great"' is a valid expression, as is 'how_feeling in ["Great", "Terrible"]'., """,
+                """However, 'how_feeling = "Great"' is not a valid expression.""",
+                """See https://docs.expectedparrot.com/en/latest/results.html#filtering-results for more details.""",
             )
 
         if len(new_data) == 0:
@@ -897,7 +989,7 @@ class Results(UserList, Mixins, Base):
         return Results(survey=self.survey, data=new_data, created_columns=None)
 
     @classmethod
-    def example(cls, debug: bool = False) -> Results:
+    def example(cls, debug: bool = False, randomize: bool = False) -> Results:
         """Return an example `Results` object.
 
         Example usage:
@@ -906,11 +998,11 @@ class Results(UserList, Mixins, Base):
 
         :param debug: if False, uses actual API calls
         """
-        from edsl.jobs import Jobs
+        from edsl.jobs.Jobs import Jobs
         from edsl.data.Cache import Cache
 
         c = Cache()
-        job = Jobs.example()
+        job = Jobs.example(randomize=randomize)
         results = job.run(cache=c, debug=debug)
         return results
 

edsl/results/ResultsDBMixin.py

@@ -1,8 +1,6 @@
 """Mixin for working with SQLite respresentation of a 'Results' object."""
 
-import pandas as pd
 import sqlite3
-from sqlalchemy import create_engine
 from enum import Enum
 from typing import Literal, Union, Optional
 
@@ -92,6 +90,8 @@ class ResultsDBMixin:
             conn.commit()
             return conn
         elif shape == SQLDataShape.WIDE:
+            from sqlalchemy import create_engine
+
             engine = create_engine("sqlite:///:memory:")
             df = self.to_pandas(remove_prefix=remove_prefix)
             df.to_sql("self", engine, index=False, if_exists="replace")
@@ -121,7 +121,7 @@ class ResultsDBMixin:
         to_list=False,
         to_latex=False,
         filename: Optional[str] = None,
-    ) -> Union[pd.DataFrame, str]:
+    ) -> Union["pd.DataFrame", str]:
         """Execute a SQL query and return the results as a DataFrame.
 
         :param query: The SQL query to execute
@@ -151,6 +151,8 @@ class ResultsDBMixin:
         2    Terrible
         3          OK
         """
+        import pandas as pd
+
         shape_enum = self._get_shape_enum(shape)
 
         conn = self._db(shape=shape_enum, remove_prefix=remove_prefix)
@@ -205,6 +207,8 @@ class ResultsDBMixin:
         ...
         <BLANKLINE>
         """
+        import pandas as pd
+
         shape_enum = self._get_shape_enum(shape)
         conn = self._db(shape=shape_enum, remove_prefix=remove_prefix)