edsl 0.1.27.dev2__py3-none-any.whl → 0.1.28__py3-none-any.whl

edsl/language_models/repair.py

@@ -1,14 +1,19 @@
 import json
 import asyncio
+import warnings
+from rich import print
+from rich.console import Console
+from rich.syntax import Syntax
 
 from edsl.utilities.utilities import clean_json
 
+from edsl.utilities.repair_functions import extract_json_from_string
 
-async def async_repair(bad_json, error_message=""):
-    s = clean_json(bad_json)
-    from edsl import Model
 
-    m = Model()
+async def async_repair(
+    bad_json, error_message="", user_prompt=None, system_prompt=None, cache=None
+):
+    s = clean_json(bad_json)
 
     try:
         # this is the OpenAI version, but that's fine
@@ -17,56 +22,122 @@ async def async_repair(bad_json, error_message=""):
     except json.JSONDecodeError:
         valid_dict = {}
         success = False
-        # print("Replacing control characters didn't work. Trying with the model.")
+        # print("Replacing control characters didn't work. Trying extracting the sub-string.")
     else:
         return valid_dict, success
 
-    prompt = f"""This is the output from a less capable language model.  
-    It was supposed to respond with just a JSON object with an answer to a question and some commentary, 
-    in a field called "comment" next to "answer".
-    Please repair this bad JSON: {bad_json}."""
+    try:
+        valid_dict = extract_json_from_string(s)
+        success = True
+    except ValueError:
+        valid_dict = {}
+        success = False
+    else:
+        return valid_dict, success
 
-    if error_message:
-        prompt += f" Parsing error message: {error_message}"
+    from edsl import Model
 
-    try:
-        results = await m.async_execute_model_call(
-            prompt,
-            system_prompt="You are a helpful agent. Only return the repaired JSON, nothing else.",
+    m = Model()
+
+    from edsl import QuestionExtract
+
+    with warnings.catch_warnings():
+        warnings.simplefilter("ignore", UserWarning)
+
+        q = QuestionExtract(
+            question_text="""
+        A language model was supposed to respond to a question. 
+        The response should have been JSON object with an answer to a question and some commentary.
+        
+        It should have retured a string like this: 
+        
+        '{'answer': 'The answer to the question.', 'comment': 'Some commentary.'}'
+        
+        or:
+
+        '{'answer': 'The answer to the question.'}'
+
+        The answer field is very like an integer number. The comment field is always string.
+
+        You job is to return just the repaired JSON object that the model should have returned, properly formatted.
+
+            - It might have included some preliminary comments.
+            - It might have included some control characters.
+            - It might have included some extraneous text.
+
+        DO NOT include any extraneous text in your response. Just return the repaired JSON object.
+        Do not preface the JSON object with any text. Just return the JSON object.
+
+        Bad answer: """
+            + str(bad_json)
+            + "The model received a user prompt of: '"
+            + str(user_prompt)
+            + """'
+        The model received a system prompt of: ' """
+            + str(system_prompt)
+            + """
+        '
+        Please return the repaired JSON object, following the instructions the original model should have followed, though 
+        using 'new_answer' a nd 'new_comment' as the keys.""",
+            answer_template={
+                "new_answer": "<number, string, list, etc.>",
+                "new_comment": "Model's comments",
+            },
+            question_name="model_repair",
         )
-    except Exception as e:
-        return {}, False
+
+    results = await q.run_async(cache=cache)
 
     try:
         # this is the OpenAI version, but that's fine
-        valid_dict = json.loads(results["choices"][0]["message"]["content"])
+        valid_dict = json.loads(json.dumps(results))
         success = True
+        # this is to deal with the fact that the model returns the answer and comment as new_answer and new_comment
+        valid_dict["answer"] = valid_dict.pop("new_answer")
+        valid_dict["comment"] = valid_dict.pop("new_comment")
     except json.JSONDecodeError:
         valid_dict = {}
         success = False
+        console = Console()
+        error_message = (
+            f"All repairs. failed. LLM Model given [red]{str(bad_json)}[/red]"
+        )
+        console.print("    " + error_message)
+        model_returned = results["choices"][0]["message"]["content"]
+        console.print(f"LLM Model returned: [blue]{model_returned}[/blue]")
 
     return valid_dict, success
 
 
-def repair_wrapper(bad_json, error_message=""):
+def repair_wrapper(
+    bad_json, error_message="", user_prompt=None, system_prompt=None, cache=None
+):
     try:
         loop = asyncio.get_event_loop()
         if loop.is_running():
             # Add repair as a task to the running loop
-            task = loop.create_task(async_repair(bad_json, error_message))
+            task = loop.create_task(
+                async_repair(bad_json, error_message, user_prompt, system_prompt, cache)
+            )
             return task
         else:
             # Run a new event loop for repair
-            return loop.run_until_complete(async_repair(bad_json, error_message))
+            return loop.run_until_complete(
+                async_repair(bad_json, error_message, user_prompt, system_prompt, cache)
+            )
     except RuntimeError:
         # Create a new event loop if one is not already available
         loop = asyncio.new_event_loop()
         asyncio.set_event_loop(loop)
-        return loop.run_until_complete(async_repair(bad_json, error_message))
+        return loop.run_until_complete(
+            async_repair(bad_json, error_message, user_prompt, system_prompt, cache)
+        )
 
 
-def repair(bad_json, error_message=""):
-    return repair_wrapper(bad_json, error_message)
+def repair(
+    bad_json, error_message="", user_prompt=None, system_prompt=None, cache=None
+):
+    return repair_wrapper(bad_json, error_message, user_prompt, system_prompt, cache)
 
 
 # Example usage:

edsl/notebooks/Notebook.py

@@ -1,5 +1,8 @@
-"""A Notebook is ...."""
+"""A Notebook is a utility class that allows you to easily share/pull ipynbs from Coop."""
 
+import json
+import nbformat
+from nbconvert import HTMLExporter
 from typing import Dict, List, Optional
 from rich.table import Table
 from edsl.Base import Base
@@ -14,43 +17,58 @@ class Notebook(Base):
     A Notebook is a utility class that allows you to easily share/pull ipynbs from Coop.
     """
 
-    def __init__(self, data: Optional[Dict] = None, path: Optional[str] = None):
+    default_name = "notebook"
+
+    def __init__(
+        self,
+        data: Optional[Dict] = None,
+        path: Optional[str] = None,
+        name: Optional[str] = None,
+    ):
         """
         Initialize a new Notebook.
-        - if a path is provided, try to load the notebook from that path.
-        - if no path is provided, assume this code is run in a notebook and try to load the current notebook.
+
+        :param data: A dictionary representing the notebook data.
+        This dictionary must conform to the official Jupyter Notebook format, as defined by nbformat.
+        :param path: A filepath from which to load the notebook.
+        If no path is provided, assume this code is run in a notebook and try to load the current notebook from file.
+        :param name: A name for the Notebook.
         """
+        # Load current notebook path as fallback (VS Code only)
+        path = path or globals().get("__vsc_ipynb_file__")
         if data is not None:
+            nbformat.validate(data)
             self.data = data
         elif path is not None:
-            # TO BE IMPLEMENTED
-            # store in this var the data from the notebook
-            self.data = {"some": "data"}
+            with open(path, mode="r", encoding="utf-8") as f:
+                data = nbformat.read(f, as_version=4)
+            self.data = json.loads(json.dumps(data))
         else:
-            # TO BE IMPLEMENTED
-            # 1. Check you're in a notebook ...
-            # 2. get its info and store it in self.data
-            self.data = {"some": "data"}
+            # TODO: Support for IDEs other than VSCode
+            raise NotImplementedError(
+                "Cannot create a notebook from within itself in this development environment"
+            )
 
-        # deprioritize - perhaps add sanity check function
+        # TODO: perhaps add sanity check function
         # 1. could check if the notebook is a valid notebook
         # 2. could check notebook uses EDSL
         # ....
 
+        self.name = name or self.default_name
+
     def __eq__(self, other):
         """
         Check if two Notebooks are equal.
+        This only checks the notebook data.
         """
         return self.data == other.data
 
     @add_edsl_version
     def to_dict(self) -> dict:
         """
-        Convert to a dictionary.
-        AF: here you will create a dict from which self.from_dict can recreate the object.
-        AF: the decorator will add the edsl_version to the dict.
+        Convert a Notebook to a dictionary.
         """
-        return {"data": self.data}
+        return {"name": self.name, "data": self.data}
 
     @classmethod
     @remove_edsl_version
@@ -58,12 +76,17 @@ class Notebook(Base):
         """
         Convert a dictionary representation of a Notebook to a Notebook object.
         """
-        return cls(data=d["data"])
+        return cls(data=d["data"], name=d["name"])
+
+    def to_file(self, path: str):
+        """
+        Save the notebook at the specified filepath.
+        """
+        nbformat.write(nbformat.from_dict(self.data), fp=path)
 
     def print(self):
         """
         Print the notebook.
-        AF: not sure how this should behave for a notebook
         """
         from rich import print_json
         import json
@@ -72,45 +95,110 @@ class Notebook(Base):
 
     def __repr__(self):
         """
-        AF: not sure how this should behave for a notebook
+        Return representation of Notebook.
         """
-        return f"Notebook({self.to_dict()})"
+        return f'Notebook(data={self.data}, name="""{self.name}""")'
 
     def _repr_html_(self):
         """
-        AF: not sure how this should behave for a notebook
+        Return HTML representation of Notebook.
         """
-        from edsl.utilities.utilities import data_to_html
+        notebook = nbformat.from_dict(self.data)
+        html_exporter = HTMLExporter(template_name="basic")
+        (body, _) = html_exporter.from_notebook_node(notebook)
+        return body
 
-        return data_to_html(self.to_dict())
+    def _table(self) -> tuple[dict, list]:
+        """
+        Prepare generic table data.
+        """
+        table_data = []
+
+        notebook_preview = ""
+        for cell in self.data["cells"]:
+            if "source" in cell:
+                notebook_preview += f"{cell['source']}\n"
+            if len(notebook_preview) > 1000:
+                notebook_preview = f"{notebook_preview[:1000]} [...]"
+                break
+        notebook_preview = notebook_preview.rstrip()
+
+        table_data.append(
+            {
+                "Attribute": "name",
+                "Value": repr(self.name),
+            }
+        )
+        table_data.append(
+            {
+                "Attribute": "notebook_preview",
+                "Value": notebook_preview,
+            }
+        )
+
+        column_names = ["Attribute", "Value"]
+        return table_data, column_names
 
     def rich_print(self) -> "Table":
         """
-        AF: not sure how we should implement this for a notebook
+        Display a Notebook as a rich table.
         """
-        pass
+        table_data, column_names = self._table()
+        table = Table(title=f"{self.__class__.__name__} Attributes")
+        for column in column_names:
+            table.add_column(column, style="bold")
+
+        for row in table_data:
+            row_data = [row[column] for column in column_names]
+            table.add_row(*row_data)
+
+        return table
 
     @classmethod
     def example(cls) -> "Notebook":
         """
         Return an example Notebook.
-        AF: add a simple custom example here
         """
-        return cls(data={"some": "data"})
+        cells = [
+            {
+                "cell_type": "markdown",
+                "metadata": dict(),
+                "source": "# Test notebook",
+            },
+            {
+                "cell_type": "code",
+                "execution_count": 1,
+                "metadata": dict(),
+                "outputs": [
+                    {
+                        "name": "stdout",
+                        "output_type": "stream",
+                        "text": "Hello world!\n",
+                    }
+                ],
+                "source": 'print("Hello world!")',
+            },
+        ]
+        data = {
+            "metadata": dict(),
+            "nbformat": 4,
+            "nbformat_minor": 4,
+            "cells": cells,
+        }
+        return cls(data=data)
 
     def code(self) -> List[str]:
         """
         Return the code that could be used to create this Notebook.
-        AF: Again, not sure
         """
         lines = []
-        lines.append("from edsl.notebooks import Notebook")
-        lines.append(f"s = Notebook({self.data})")
+        lines.append("from edsl import Notebook")
+        lines.append(f'nb = Notebook(data={self.data}, name="""{self.name}""")')
         return lines
 
 
 if __name__ == "__main__":
-    from edsl.notebooks import Notebook
+    from edsl import Notebook
 
     notebook = Notebook.example()
     assert notebook == notebook.from_dict(notebook.to_dict())

edsl/questions/QuestionBase.py

@@ -3,7 +3,8 @@
 from __future__ import annotations
 from abc import ABC, abstractmethod
 from rich.table import Table
-from typing import Any, Type, Optional
+from typing import Any, Type, Optional, List, Callable
+import copy
 
 from edsl.exceptions import (
     QuestionResponseValidationError,
@@ -15,6 +16,7 @@ from edsl.prompts.registry import get_classes as prompt_lookup
 from edsl.questions.AnswerValidatorMixin import AnswerValidatorMixin
 from edsl.questions.RegisterQuestionsMeta import RegisterQuestionsMeta
 from edsl.Base import PersistenceMixin, RichPrintingMixin
+from edsl.BaseDiff import BaseDiff, BaseDiffCollection
 
 from edsl.questions.SimpleAskMixin import SimpleAskMixin
 from edsl.utilities.decorators import add_edsl_version, remove_edsl_version
@@ -37,6 +39,12 @@ class QuestionBase(
         """Get an attribute of the question."""
         return getattr(self, key)
 
+    def __hash__(self) -> int:
+        """Return a hash of the question."""
+        from edsl.utilities.utilities import dict_hash
+
+        return dict_hash(self._to_dict())
+
     def _repr_html_(self):
         from edsl.utilities.utilities import data_to_html
 
@@ -49,6 +57,35 @@ class QuestionBase(
 
         return data_to_html(data)
 
+    def apply_function(self, func: Callable, exclude_components=None) -> QuestionBase:
+        """Apply a function to the question parts
+
+        >>> from edsl.questions import QuestionFreeText
+        >>> q = QuestionFreeText(question_name = "color", question_text = "What is your favorite color?")
+        >>> shouting = lambda x: x.upper()
+        >>> q.apply_function(shouting)
+        Question('free_text', question_name = \"""color\""", question_text = \"""WHAT IS YOUR FAVORITE COLOR?\""")
+
+        """
+        if exclude_components is None:
+            exclude_components = ["question_name", "question_type"]
+
+        d = copy.deepcopy(self._to_dict())
+        for key, value in d.items():
+            if key in exclude_components:
+                continue
+            if isinstance(value, dict):
+                for k, v in value.items():
+                    value[k] = func(v)
+                d[key] = value
+                continue
+            if isinstance(value, list):
+                value = [func(v) for v in value]
+                d[key] = value
+                continue
+            d[key] = func(value)
+        return QuestionBase.from_dict(d)
+
     @property
     def data(self) -> dict:
         """Return a dictionary of question attributes **except** for question_type."""
@@ -101,6 +138,33 @@ class QuestionBase(
             self._model_instructions = {}
         return self._model_instructions
 
+    def _all_text(self) -> str:
+        """Return the question text."""
+        txt = ""
+        for key, value in self.data.items():
+            if isinstance(value, str):
+                txt += value
+            elif isinstance(value, list):
+                txt += "".join(str(value))
+        return txt
+
+    @property
+    def parameters(self) -> set[str]:
+        """Return the parameters of the question."""
+        from jinja2 import Environment, meta
+
+        env = Environment()
+        # Parse the template
+        txt = self._all_text()
+        # txt = self.question_text
+        # if hasattr(self, "question_options"):
+        #    txt += " ".join(self.question_options)
+        parsed_content = env.parse(txt)
+        # Extract undeclared variables
+        variables = meta.find_undeclared_variables(parsed_content)
+        # Return as a list
+        return set(variables)
+
     @model_instructions.setter
     def model_instructions(self, data: dict):
         """Set the model-specific instructions for the question."""
@@ -167,13 +231,17 @@ class QuestionBase(
     ############################
     # Serialization methods
     ############################
-    @add_edsl_version
-    def to_dict(self) -> dict[str, Any]:
+    def _to_dict(self):
         """Convert the question to a dictionary that includes the question type (used in deserialization)."""
         candidate_data = self.data.copy()
         candidate_data["question_type"] = self.question_type
         return candidate_data
 
+    @add_edsl_version
+    def to_dict(self) -> dict[str, Any]:
+        """Convert the question to a dictionary that includes the question type (used in deserialization)."""
+        return self._to_dict()
+
     @classmethod
     @remove_edsl_version
     def from_dict(cls, data: dict) -> Type[QuestionBase]:
@@ -211,6 +279,10 @@ class QuestionBase(
 
         return question_class(**local_data)
 
+    def copy(self) -> Type[QuestionBase]:
+        """Return a deep copy of the question."""
+        return copy.deepcopy(self)
+
     ############################
     # Dunder methods
     ############################
@@ -220,6 +292,25 @@ class QuestionBase(
 
         print_json(json.dumps(self.to_dict()))
 
+    def __call__(self, just_answer=True, model=None, agent=None, **kwargs):
+        """Call the question."""
+        survey = self.to_survey()
+        results = survey(model=model, agent=agent, **kwargs)
+        if just_answer:
+            return results.select(f"answer.{self.question_name}").first()
+        else:
+            return results
+
+    async def run_async(self, just_answer=True, model=None, agent=None, **kwargs):
+        """Call the question."""
+        survey = self.to_survey()
+        ## asyncio.run(survey.async_call());
+        results = await survey.run_async(model=model, agent=agent, **kwargs)
+        if just_answer:
+            return results.select(f"answer.{self.question_name}").first()
+        else:
+            return results
+
     def __repr__(self) -> str:
         """Return a string representation of the question. Should be able to be used to reconstruct the question."""
         class_name = self.__class__.__name__
@@ -237,21 +328,27 @@ class QuestionBase(
             return False
         return self.to_dict() == other.to_dict()
 
+    def __sub__(self, other) -> BaseDiff:
+        """Return the difference between two objects."""
+
+        return BaseDiff(other, self)
+
     # TODO: Throws an error that should be addressed at QuestionFunctional
-    def __add__(self, other_question):
+    def __add__(self, other_question_or_diff):
         """
         Compose two questions into a single question.
 
-        >>> from edsl.scenarios.Scenario import Scenario
-        >>> from edsl.questions.QuestionFreeText import QuestionFreeText
-        >>> from edsl.questions.QuestionNumerical import QuestionNumerical
-        >>> q1 = QuestionFreeText(question_text = "What is the capital of {{country}}", question_name = "capital")
-        >>> q2 = QuestionNumerical(question_text = "What is the population of {{capital}}, in millions. Please round", question_name = "population")
-        >>> q3 = q1 + q2
+        TODO: Probably getting deprecated.
+
         """
+        if isinstance(other_question_or_diff, BaseDiff) or isinstance(
+            other_question_or_diff, BaseDiffCollection
+        ):
+            return other_question_or_diff.apply(self)
+
         from edsl.questions import compose_questions
 
-        return compose_questions(self, other_question)
+        return compose_questions(self, other_question_or_diff)
 
     @abstractmethod
     def _validate_answer(self, answer: dict[str, str]):
@@ -279,7 +376,7 @@ class QuestionBase(
     ############################
     # Forward methods
     ############################
-    def add_question(self, other: Question) -> "Survey":
+    def add_question(self, other: QuestionBase) -> "Survey":
         """Add a question to this question by turning them into a survey with two questions."""
         from edsl.surveys.Survey import Survey
 

edsl/questions/descriptors.py

@@ -331,6 +331,9 @@ class QuestionTextDescriptor(BaseDescriptor):
         if not isinstance(value, str):
             raise Exception("Question must be a string!")
         if contains_single_braced_substring(value):
-            print(
-                f"WARNING: Question text contains a single-braced substring: {value}.\nIf you intended to parameterize the question with a Scenario this should be changed to a double-braced substring, e.g. {{variable}}.\nSee details on constructing Scenarios in the docs: https://docs.expectedparrot.com/en/latest/scenarios.html"
+            import warnings
+
+            warnings.warn(
+                f"WARNING: Question text contains a single-braced substring: If you intended to parameterize the question with a Scenario this should be changed to a double-braced substring, e.g. {{variable}}.\nSee details on constructing Scenarios in the docs: https://docs.expectedparrot.com/en/latest/scenarios.html",
+                UserWarning,
             )

edsl/questions/question_registry.py

@@ -52,6 +52,13 @@ class Question(metaclass=Meta):
         instance.__init__(*args, **kwargs)
         return instance
 
+    @classmethod
+    def example(cls, question_type: str):
+        """Return an example question of the given type."""
+        get_question_classes = RegisterQuestionsMeta.question_types_to_classes()
+        q = get_question_classes.get(question_type, None)
+        return q.example()
+
     @classmethod
     def pull(cls, id_or_url: str):
         """Pull the object from coop."""

edsl/results/Result.py

@@ -58,6 +58,8 @@ class Result(Base, UserDict):
 
     The answer dictionary has the structure:
 
+    >>> import warnings
+    >>> warnings.simplefilter("ignore", UserWarning)
     >>> Result.example().answer
     {'how_feeling': 'OK', 'how_feeling_comment': 'This is a real survey response from a human.', 'how_feeling_yesterday': 'Great', 'how_feeling_yesterday_comment': 'This is a real survey response from a human.'}
 
@@ -237,14 +239,8 @@ class Result(Base, UserDict):
     ###############
     # Serialization
     ###############
-    @add_edsl_version
-    def to_dict(self) -> dict[str, Any]:
-        """Return a dictionary representation of the Result object.
-
-        >>> r = Result.example()
-        >>> r.to_dict()['scenario']
-        {'period': 'morning', 'edsl_version': '...', 'edsl_class_name': 'Scenario'}
-        """
+    def _to_dict(self) -> dict[str, Any]:
+        """Return a dictionary representation of the Result object."""
         d = {}
         for key, value in self.items():
             if hasattr(value, "to_dict"):
@@ -262,6 +258,22 @@ class Result(Base, UserDict):
                 d[key] = new_prompt_dict
         return d
 
+    @add_edsl_version
+    def to_dict(self) -> dict[str, Any]:
+        """Return a dictionary representation of the Result object.
+
+        >>> r = Result.example()
+        >>> r.to_dict()['scenario']
+        {'period': 'morning', 'edsl_version': '...', 'edsl_class_name': 'Scenario'}
+        """
+        return self._to_dict()
+
+    def __hash__(self):
+        """Return a hash of the Result object."""
+        from edsl.utilities.utilities import dict_hash
+
+        return dict_hash(self._to_dict())
+
     @classmethod
     @remove_edsl_version
     def from_dict(self, json_dict: dict) -> Result: