edsl 0.1.38.dev3__py3-none-any.whl → 0.1.39__py3-none-any.whl

edsl/language_models/repair.py

@@ -1,156 +1,156 @@
-import json
-import asyncio
-import warnings
-
-
-async def async_repair(
-    bad_json, error_message="", user_prompt=None, system_prompt=None, cache=None
-):
-    from edsl.utilities.utilities import clean_json
-
-    s = clean_json(bad_json)
-
-    try:
-        # this is the OpenAI version, but that's fine
-        valid_dict = json.loads(s)
-        success = True
-    except json.JSONDecodeError:
-        valid_dict = {}
-        success = False
-        # print("Replacing control characters didn't work. Trying extracting the sub-string.")
-    else:
-        return valid_dict, success
-
-    try:
-        from edsl.utilities.repair_functions import extract_json_from_string
-
-        valid_dict = extract_json_from_string(s)
-        success = True
-    except ValueError:
-        valid_dict = {}
-        success = False
-    else:
-        return valid_dict, success
-
-    from edsl import Model
-
-    m = Model()
-
-    from edsl import QuestionExtract
-
-    with warnings.catch_warnings():
-        warnings.simplefilter("ignore", UserWarning)
-
-        q = QuestionExtract(
-            question_text="""
-        A language model was supposed to respond to a question. 
-        The response should have been JSON object with an answer to a question and some commentary.
-        
-        It should have retured a string like this: 
-        
-        '{'answer': 'The answer to the question.', 'comment': 'Some commentary.'}'
-        
-        or:
-
-        '{'answer': 'The answer to the question.'}'
-
-        The answer field is very like an integer number. The comment field is always string.
-
-        You job is to return just the repaired JSON object that the model should have returned, properly formatted.
-
-            - It might have included some preliminary comments.
-            - It might have included some control characters.
-            - It might have included some extraneous text.
-
-        DO NOT include any extraneous text in your response. Just return the repaired JSON object.
-        Do not preface the JSON object with any text. Just return the JSON object.
-
-        Bad answer: """
-            + str(bad_json)
-            + "The model received a user prompt of: '"
-            + str(user_prompt)
-            + """'
-        The model received a system prompt of: ' """
-            + str(system_prompt)
-            + """
-        '
-        Please return the repaired JSON object, following the instructions the original model should have followed, though 
-        using 'new_answer' a nd 'new_comment' as the keys.""",
-            answer_template={
-                "new_answer": "<number, string, list, etc.>",
-                "new_comment": "Model's comments",
-            },
-            question_name="model_repair",
-        )
-
-    results = await q.run_async(cache=cache)
-
-    try:
-        # this is the OpenAI version, but that's fine
-        valid_dict = json.loads(json.dumps(results))
-        success = True
-        # this is to deal with the fact that the model returns the answer and comment as new_answer and new_comment
-        valid_dict["answer"] = valid_dict.pop("new_answer")
-        valid_dict["comment"] = valid_dict.pop("new_comment")
-    except json.JSONDecodeError:
-        valid_dict = {}
-        success = False
-        from rich import print
-        from rich.console import Console
-        from rich.syntax import Syntax
-
-        console = Console()
-        error_message = (
-            f"All repairs. failed. LLM Model given [red]{str(bad_json)}[/red]"
-        )
-        console.print("    " + error_message)
-        model_returned = results["choices"][0]["message"]["content"]
-        console.print(f"LLM Model returned: [blue]{model_returned}[/blue]")
-
-    return valid_dict, success
-
-
-def repair_wrapper(
-    bad_json, error_message="", user_prompt=None, system_prompt=None, cache=None
-):
-    try:
-        loop = asyncio.get_event_loop()
-        if loop.is_running():
-            # Add repair as a task to the running loop
-            task = loop.create_task(
-                async_repair(bad_json, error_message, user_prompt, system_prompt, cache)
-            )
-            return task
-        else:
-            # Run a new event loop for repair
-            return loop.run_until_complete(
-                async_repair(bad_json, error_message, user_prompt, system_prompt, cache)
-            )
-    except RuntimeError:
-        # Create a new event loop if one is not already available
-        loop = asyncio.new_event_loop()
-        asyncio.set_event_loop(loop)
-        return loop.run_until_complete(
-            async_repair(bad_json, error_message, user_prompt, system_prompt, cache)
-        )
-
-
-def repair(
-    bad_json, error_message="", user_prompt=None, system_prompt=None, cache=None
-):
-    return repair_wrapper(bad_json, error_message, user_prompt, system_prompt, cache)
-
-
-if __name__ == "__main__":
-    bad_json = """
-    {
-      'answer': "The problematic phrase in the excerpt is \'typically\'. This word is vague and can lead to different interpretations. An alternative phrasing that would be less problematic is: 
-      'On average, how long do you cook scrambled eggs?}
-    """
-    try:
-        json.loads(bad_json)
-        print("Loaded")
-    except json.JSONDecodeError as e:
-        error_message = str(e)
-        repaired, success = repair(bad_json, error_message)
-        print(f"Repaired: {repaired}")
+import json
+import asyncio
+import warnings
+
+
+async def async_repair(
+    bad_json, error_message="", user_prompt=None, system_prompt=None, cache=None
+):
+    from edsl.utilities.utilities import clean_json
+
+    s = clean_json(bad_json)
+
+    try:
+        # this is the OpenAI version, but that's fine
+        valid_dict = json.loads(s)
+        success = True
+    except json.JSONDecodeError:
+        valid_dict = {}
+        success = False
+        # print("Replacing control characters didn't work. Trying extracting the sub-string.")
+    else:
+        return valid_dict, success
+
+    try:
+        from edsl.utilities.repair_functions import extract_json_from_string
+
+        valid_dict = extract_json_from_string(s)
+        success = True
+    except ValueError:
+        valid_dict = {}
+        success = False
+    else:
+        return valid_dict, success
+
+    from edsl.language_models.model import Model
+
+    m = Model()
+
+    from edsl.questions.QuestionExtract import QuestionExtract
+
+    with warnings.catch_warnings():
+        warnings.simplefilter("ignore", UserWarning)
+
+        q = QuestionExtract(
+            question_text="""
+        A language model was supposed to respond to a question. 
+        The response should have been JSON object with an answer to a question and some commentary.
+        
+        It should have retured a string like this: 
+        
+        '{'answer': 'The answer to the question.', 'comment': 'Some commentary.'}'
+        
+        or:
+
+        '{'answer': 'The answer to the question.'}'
+
+        The answer field is very like an integer number. The comment field is always string.
+
+        You job is to return just the repaired JSON object that the model should have returned, properly formatted.
+
+            - It might have included some preliminary comments.
+            - It might have included some control characters.
+            - It might have included some extraneous text.
+
+        DO NOT include any extraneous text in your response. Just return the repaired JSON object.
+        Do not preface the JSON object with any text. Just return the JSON object.
+
+        Bad answer: """
+            + str(bad_json)
+            + "The model received a user prompt of: '"
+            + str(user_prompt)
+            + """'
+        The model received a system prompt of: ' """
+            + str(system_prompt)
+            + """
+        '
+        Please return the repaired JSON object, following the instructions the original model should have followed, though 
+        using 'new_answer' a nd 'new_comment' as the keys.""",
+            answer_template={
+                "new_answer": "<number, string, list, etc.>",
+                "new_comment": "Model's comments",
+            },
+            question_name="model_repair",
+        )
+
+    results = await q.run_async(cache=cache)
+
+    try:
+        # this is the OpenAI version, but that's fine
+        valid_dict = json.loads(json.dumps(results))
+        success = True
+        # this is to deal with the fact that the model returns the answer and comment as new_answer and new_comment
+        valid_dict["answer"] = valid_dict.pop("new_answer")
+        valid_dict["comment"] = valid_dict.pop("new_comment")
+    except json.JSONDecodeError:
+        valid_dict = {}
+        success = False
+        from rich import print
+        from rich.console import Console
+        from rich.syntax import Syntax
+
+        console = Console()
+        error_message = (
+            f"All repairs. failed. LLM Model given [red]{str(bad_json)}[/red]"
+        )
+        console.print("    " + error_message)
+        model_returned = results["choices"][0]["message"]["content"]
+        console.print(f"LLM Model returned: [blue]{model_returned}[/blue]")
+
+    return valid_dict, success
+
+
+def repair_wrapper(
+    bad_json, error_message="", user_prompt=None, system_prompt=None, cache=None
+):
+    try:
+        loop = asyncio.get_event_loop()
+        if loop.is_running():
+            # Add repair as a task to the running loop
+            task = loop.create_task(
+                async_repair(bad_json, error_message, user_prompt, system_prompt, cache)
+            )
+            return task
+        else:
+            # Run a new event loop for repair
+            return loop.run_until_complete(
+                async_repair(bad_json, error_message, user_prompt, system_prompt, cache)
+            )
+    except RuntimeError:
+        # Create a new event loop if one is not already available
+        loop = asyncio.new_event_loop()
+        asyncio.set_event_loop(loop)
+        return loop.run_until_complete(
+            async_repair(bad_json, error_message, user_prompt, system_prompt, cache)
+        )
+
+
+def repair(
+    bad_json, error_message="", user_prompt=None, system_prompt=None, cache=None
+):
+    return repair_wrapper(bad_json, error_message, user_prompt, system_prompt, cache)
+
+
+if __name__ == "__main__":
+    bad_json = """
+    {
+      'answer': "The problematic phrase in the excerpt is \'typically\'. This word is vague and can lead to different interpretations. An alternative phrasing that would be less problematic is: 
+      'On average, how long do you cook scrambled eggs?}
+    """
+    try:
+        json.loads(bad_json)
+        print("Loaded")
+    except json.JSONDecodeError as e:
+        error_message = str(e)
+        repaired, success = repair(bad_json, error_message)
+        print(f"Repaired: {repaired}")

edsl/language_models/utilities.py

@@ -1,64 +1,65 @@
-import asyncio
-from typing import Any, Optional, List
-from edsl import Survey
-from edsl.config import CONFIG
-from edsl.enums import InferenceServiceType
-from edsl.language_models.LanguageModel import LanguageModel
-from edsl.questions import QuestionFreeText
-
-
-def create_survey(num_questions: int, chained: bool = True, take_scenario=False):
-    survey = Survey()
-    for i in range(num_questions):
-        if take_scenario:
-            q = QuestionFreeText(
-                question_text=f"XX{i}XX and {{scenario_value }}",
-                question_name=f"question_{i}",
-            )
-        else:
-            q = QuestionFreeText(
-                question_text=f"XX{i}XX", question_name=f"question_{i}"
-            )
-        survey.add_question(q)
-        if i > 0 and chained:
-            survey.add_targeted_memory(f"question_{i}", f"question_{i-1}")
-    return survey
-
-
-def create_language_model(
-    exception: Exception, fail_at_number: int, never_ending=False
-):
-    class LanguageModelFromUtilities(LanguageModel):
-        _model_ = "test"
-        _parameters_ = {"temperature": 0.5}
-        _inference_service_ = InferenceServiceType.TEST.value
-        key_sequence = ["message", 0, "text"]
-        usage_sequence = ["usage"]
-        input_token_name = "prompt_tokens"
-        output_token_name = "completion_tokens"
-        _rpm = 1000000000000
-        _tpm = 1000000000000
-
-        async def async_execute_model_call(
-            self,
-            user_prompt: str,
-            system_prompt: str,
-            files_list: Optional[List[Any]] = None,
-        ) -> dict[str, Any]:
-            question_number = int(
-                user_prompt.split("XX")[1]
-            )  ## grabs the question number from the prompt
-            await asyncio.sleep(0.1)
-            if never_ending:  ## you're not going anywhere buddy
-                await asyncio.sleep(float("inf"))
-            if question_number == fail_at_number:
-                if asyncio.iscoroutinefunction(exception):
-                    await exception()
-                else:
-                    raise exception
-            return {
-                "message": [{"text": "SPAM!"}],
-                "usage": {"prompt_tokens": 1, "completion_tokens": 1},
-            }
-
-    return LanguageModelFromUtilities
+import asyncio
+from typing import Any, Optional, List
+from edsl.enums import InferenceServiceType
+
+
+def create_survey(num_questions: int, chained: bool = True, take_scenario=False):
+    from edsl.surveys.Survey import Survey
+    from edsl.questions.QuestionFreeText import QuestionFreeText
+
+    survey = Survey()
+    for i in range(num_questions):
+        if take_scenario:
+            q = QuestionFreeText(
+                question_text=f"XX{i}XX and {{scenario_value }}",
+                question_name=f"question_{i}",
+            )
+        else:
+            q = QuestionFreeText(
+                question_text=f"XX{i}XX", question_name=f"question_{i}"
+            )
+        survey.add_question(q)
+        if i > 0 and chained:
+            survey.add_targeted_memory(f"question_{i}", f"question_{i-1}")
+    return survey
+
+
+def create_language_model(
+    exception: Exception, fail_at_number: int, never_ending=False
+):
+    from edsl.language_models.LanguageModel import LanguageModel
+
+    class LanguageModelFromUtilities(LanguageModel):
+        _model_ = "test"
+        _parameters_ = {"temperature": 0.5}
+        _inference_service_ = InferenceServiceType.TEST.value
+        key_sequence = ["message", 0, "text"]
+        usage_sequence = ["usage"]
+        input_token_name = "prompt_tokens"
+        output_token_name = "completion_tokens"
+        _rpm = 1000000000000
+        _tpm = 1000000000000
+
+        async def async_execute_model_call(
+            self,
+            user_prompt: str,
+            system_prompt: str,
+            files_list: Optional[List[Any]] = None,
+        ) -> dict[str, Any]:
+            question_number = int(
+                user_prompt.split("XX")[1]
+            )  ## grabs the question number from the prompt
+            await asyncio.sleep(0.1)
+            if never_ending:  ## you're not going anywhere buddy
+                await asyncio.sleep(float("inf"))
+            if question_number == fail_at_number:
+                if asyncio.iscoroutinefunction(exception):
+                    await exception()
+                else:
+                    raise exception
+            return {
+                "message": [{"text": "SPAM!"}],
+                "usage": {"prompt_tokens": 1, "completion_tokens": 1},
+            }
+
+    return LanguageModelFromUtilities