edsl 0.1.44__py3-none-any.whl → 0.1.46__py3-none-any.whl

edsl/Base.py

@@ -65,10 +65,10 @@ class PersistenceMixin:
     def pull(
         cls,
         url_or_uuid: Optional[Union[str, UUID]] = None,
-        #expected_parrot_url: Optional[str] = None,
+        # expected_parrot_url: Optional[str] = None,
     ):
         """Pull the object from coop.
-        
+
         Args:
             url_or_uuid: Either a UUID string or a URL pointing to the object
             expected_parrot_url: Optional URL for the Parrot server
@@ -80,7 +80,11 @@ class PersistenceMixin:
         coop = Coop()
 
         # Determine if input is URL or UUID
-        if url_or_uuid and ("http://" in str(url_or_uuid) or "https://" in str(url_or_uuid)):
+        if "www" in url_or_uuid:
+            url_or_uuid = url_or_uuid.replace("www", "api")
+        if url_or_uuid and (
+            "http://" in str(url_or_uuid) or "https://" in str(url_or_uuid)
+        ):
             return coop.get(url=url_or_uuid, expected_object_type=object_type)
         else:
             return coop.get(uuid=url_or_uuid, expected_object_type=object_type)

edsl/__version__.py

@@ -1 +1 @@
-__version__ = "0.1.44"
+__version__ = "0.1.46"

edsl/agents/InvigilatorBase.py

@@ -74,7 +74,7 @@ class InvigilatorBase(ABC):
     @property
     def prompt_constructor(self) -> PromptConstructor:
         """Return the prompt constructor."""
-        return PromptConstructor(self, prompt_plan=self.prompt_plan)
+        return PromptConstructor.from_invigilator(self, prompt_plan=self.prompt_plan)
 
     def to_dict(self, include_cache=False) -> Dict[str, Any]:
         attributes = [
@@ -87,6 +87,7 @@ class InvigilatorBase(ABC):
             "iteration",
             "additional_prompt_data",
             "survey",
+            "raw_model_response",
         ]
         if include_cache:
             attributes.append("cache")
@@ -135,6 +136,7 @@ class InvigilatorBase(ABC):
         d["additional_prompt_data"] = data["additional_prompt_data"]
 
         d = cls(**d)
+        d.raw_model_response = data.get("raw_model_response")
         return d
 
     def __repr__(self) -> str:

edsl/agents/PromptConstructor.py

@@ -1,20 +1,16 @@
 from __future__ import annotations
 from typing import Dict, Any, Optional, Set, Union, TYPE_CHECKING, Literal
 from functools import cached_property
-from multiprocessing import Pool, freeze_support, get_context
-from concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor
 import time
 import logging
 
 from edsl.prompts.Prompt import Prompt
 
-from dataclasses import dataclass
-
-from .prompt_helpers import PromptPlan
-from .QuestionTemplateReplacementsBuilder import (
+from edsl.agents.prompt_helpers import PromptPlan
+from edsl.agents.QuestionTemplateReplacementsBuilder import (
     QuestionTemplateReplacementsBuilder,
 )
-from .question_option_processor import QuestionOptionProcessor
+from edsl.agents.question_option_processor import QuestionOptionProcessor
 
 if TYPE_CHECKING:
     from edsl.agents.InvigilatorBase import InvigilatorBase
@@ -71,24 +67,49 @@ class PromptConstructor:
     - The question instructions - "You are being asked the following question: Do you like school? The options are 0: yes 1: no Return a valid JSON formatted like this, selecting only the number of the option: {"answer": <put answer code here>, "comment": "<put explanation here>"} Only 1 option may be selected."
     - The memory prompt - "Before the question you are now answering, you already answered the following question(s): Question: Do you like school? Answer: Prior answer"
     """
+    @classmethod
+    def from_invigilator(
+        cls,
+        invigilator: "InvigilatorBase",
+        prompt_plan: Optional["PromptPlan"] = None
+    ) -> "PromptConstructor":
+        return cls(
+            agent=invigilator.agent,
+            question=invigilator.question,
+            scenario=invigilator.scenario,
+            survey=invigilator.survey,
+            model=invigilator.model,
+            current_answers=invigilator.current_answers,
+            memory_plan=invigilator.memory_plan,
+            prompt_plan=prompt_plan
+        )
 
     def __init__(
-        self, invigilator: "InvigilatorBase", prompt_plan: Optional["PromptPlan"] = None
+        self,
+        agent: "Agent",
+        question: "QuestionBase",
+        scenario: "Scenario",
+        survey: "Survey",
+        model: "LanguageModel",
+        current_answers: dict,
+        memory_plan: "MemoryPlan",
+        prompt_plan: Optional["PromptPlan"] = None
     ):
-        self.invigilator = invigilator
+        self.agent = agent
+        self.question = question
+        self.scenario = scenario
+        self.survey = survey
+        self.model = model
+        self.current_answers = current_answers
+        self.memory_plan = memory_plan
         self.prompt_plan = prompt_plan or PromptPlan()
 
-        self.agent = invigilator.agent
-        self.question = invigilator.question
-        self.scenario = invigilator.scenario
-        self.survey = invigilator.survey
-        self.model = invigilator.model
-        self.current_answers = invigilator.current_answers
-        self.memory_plan = invigilator.memory_plan
-
-    def get_question_options(self, question_data):
+    def get_question_options(self, question_data: dict) -> list[str]:
         """Get the question options."""
-        return QuestionOptionProcessor(self).get_question_options(question_data)
+        return (QuestionOptionProcessor
+                .from_prompt_constructor(self)
+                .get_question_options(question_data)
+        )
 
     @cached_property
     def agent_instructions_prompt(self) -> Prompt:
@@ -121,23 +142,29 @@ class PromptConstructor:
         return self.agent.prompt()
 
     def prior_answers_dict(self) -> dict[str, "QuestionBase"]:
-        """This is a dictionary of prior answers, if they exist."""
+        """This is a dictionary of prior answers, if they exist.
+        
+        >>> from edsl.agents.InvigilatorBase import InvigilatorBase
+        >>> i = InvigilatorBase.example()
+        >>> i.prompt_constructor.prior_answers_dict()
+        {'q0': ..., 'q1': ...}
+        """
         return self._add_answers(
             self.survey.question_names_to_questions(), self.current_answers
         )
 
     @staticmethod
-    def _extract_quetion_and_entry_type(key_entry) -> tuple[str, str]:
+    def _extract_question_and_entry_type(key_entry) -> tuple[str, str]:
         """
         Extracts the question name and type for the current answer dictionary key entry.
 
-        >>> PromptConstructor._extract_quetion_and_entry_type("q0")
+        >>> PromptConstructor._extract_question_and_entry_type("q0")
         ('q0', 'answer')
-        >>> PromptConstructor._extract_quetion_and_entry_type("q0_comment")
+        >>> PromptConstructor._extract_question_and_entry_type("q0_comment")
         ('q0', 'comment')
-        >>> PromptConstructor._extract_quetion_and_entry_type("q0_alternate_generated_tokens")
+        >>> PromptConstructor._extract_question_and_entry_type("q0_alternate_generated_tokens")
         ('q0_alternate', 'generated_tokens')
-        >>> PromptConstructor._extract_quetion_and_entry_type("q0_alt_comment")
+        >>> PromptConstructor._extract_question_and_entry_type("q0_alt_comment")
         ('q0_alt', 'comment')
         """
         split_list = key_entry.rsplit("_", maxsplit=1)
@@ -167,7 +194,7 @@ class PromptConstructor:
         d = defaultdict(dict)
         for key, value in current_answers.items():
             question_name, entry_type = (
-                PromptConstructor._extract_quetion_and_entry_type(key)
+                PromptConstructor._extract_question_and_entry_type(key)
             )
             d[question_name][entry_type] = value
         return dict(d)
@@ -198,9 +225,10 @@ class PromptConstructor:
     @cached_property
     def question_file_keys(self) -> list:
         """Extracts the file keys from the question text.
+        
         It checks if the variables in the question text are in the scenario file keys.
         """
-        return QuestionTemplateReplacementsBuilder(self).question_file_keys()
+        return QuestionTemplateReplacementsBuilder.from_prompt_constructor(self).question_file_keys()
 
     @cached_property
     def question_instructions_prompt(self) -> Prompt:
@@ -219,7 +247,7 @@ class PromptConstructor:
             QuestionInstructionPromptBuilder,
         )
 
-        return QuestionInstructionPromptBuilder(self).build()
+        return QuestionInstructionPromptBuilder.from_prompt_constructor(self).build()
 
     @cached_property
     def prior_question_memory_prompt(self) -> Prompt:
@@ -247,97 +275,44 @@ class PromptConstructor:
             question_name, self.current_answers
         )
 
-    def get_prompts(self, parallel: Literal["thread", "process", None] = None) -> Dict[str, Any]:
+    def get_prompts(self) -> Dict[str, Any]:
         """Get the prompts for the question."""
         start = time.time()
         
         # Build all the components
-        instr_start = time.time()
         agent_instructions = self.agent_instructions_prompt
-        instr_end = time.time()
-        logger.debug(f"Time taken for agent instructions: {instr_end - instr_start:.4f}s")
-        
-        persona_start = time.time()
         agent_persona = self.agent_persona_prompt
-        persona_end = time.time()
-        logger.debug(f"Time taken for agent persona: {persona_end - persona_start:.4f}s")
-        
-        q_instr_start = time.time()
         question_instructions = self.question_instructions_prompt
-        q_instr_end = time.time()
-        logger.debug(f"Time taken for question instructions: {q_instr_end - q_instr_start:.4f}s")
-        
-        memory_start = time.time()
         prior_question_memory = self.prior_question_memory_prompt
-        memory_end = time.time()
-        logger.debug(f"Time taken for prior question memory: {memory_end - memory_start:.4f}s")
-
+        
         # Get components dict
         components = {
             "agent_instructions": agent_instructions.text,
             "agent_persona": agent_persona.text,
             "question_instructions": question_instructions.text,
             "prior_question_memory": prior_question_memory.text,
-        }
-
-        # Use PromptPlan's get_prompts method
-        plan_start = time.time()
-        
+        }        
         # Get arranged components first
         arranged = self.prompt_plan.arrange_components(**components)
         
-        if parallel == "process":
-            ctx = get_context('fork')
-            with ctx.Pool() as pool:
-                results = pool.map(_process_prompt, [
-                    (arranged["user_prompt"], {}),
-                    (arranged["system_prompt"], {})
-                ])
-                prompts = {
-                    "user_prompt": results[0],
-                    "system_prompt": results[1]
-                }
-            
-        elif parallel == "thread":
-            with ThreadPoolExecutor() as executor:
-                user_prompt_list = arranged["user_prompt"]
-                system_prompt_list = arranged["system_prompt"]
-                
-                # Process both prompt lists in parallel
-                rendered_user = executor.submit(_process_prompt, (user_prompt_list, {}))
-                rendered_system = executor.submit(_process_prompt, (system_prompt_list, {}))
-                
-                prompts = {
-                    "user_prompt": rendered_user.result(),
-                    "system_prompt": rendered_system.result()
-                }
-                
-        else:  # sequential processing
-            prompts = self.prompt_plan.get_prompts(**components)
-
-        plan_end = time.time()
-        logger.debug(f"Time taken for prompt processing: {plan_end - plan_start:.4f}s")
+        prompts = self.prompt_plan.get_prompts(**components)
         
         # Handle file keys if present
         if hasattr(self, 'question_file_keys') and self.question_file_keys:
-            files_start = time.time()
             files_list = []
             for key in self.question_file_keys:
                 files_list.append(self.scenario[key])
             prompts["files_list"] = files_list
-            files_end = time.time()
-            logger.debug(f"Time taken for file key processing: {files_end - files_start:.4f}s")
         
-        end = time.time()
-        logger.debug(f"Total time in get_prompts: {end - start:.4f}s")
         return prompts
 
 
-def _process_prompt(args):
-    """Helper function to process a single prompt list with its replacements."""
-    prompt_list, replacements = args
-    return prompt_list.reduce()
+# def _process_prompt(args):
+#     """Helper function to process a single prompt list with its replacements."""
+#     prompt_list, replacements = args
+#     return prompt_list.reduce()
 
 
 if __name__ == '__main__':
-    freeze_support()
+    import doctest
+    doctest.testmod(optionflags=doctest.ELLIPSIS)

edsl/agents/QuestionInstructionPromptBuilder.py

@@ -1,8 +1,16 @@
-from typing import Dict, List, Set
+from typing import Dict, List, Set, Any, Union, TYPE_CHECKING
 from warnings import warn
 import logging
 from edsl.prompts.Prompt import Prompt
 
+if TYPE_CHECKING:
+    from edsl.agents.PromptConstructor import PromptConstructor
+    from edsl import Model
+    from edsl import Survey
+    from edsl.questions.QuestionBase import QuestionBase
+    from edsl import Scenario
+    from edsl import Agent
+
 from edsl.agents.QuestionTemplateReplacementsBuilder import (
     QuestionTemplateReplacementsBuilder as QTRB,
 )
@@ -11,97 +19,179 @@ from edsl.agents.QuestionTemplateReplacementsBuilder import (
 class QuestionInstructionPromptBuilder:
     """Handles the construction and rendering of question instructions."""
 
-    def __init__(self, prompt_constructor: "PromptConstructor"):
-        self.prompt_constructor = prompt_constructor
+    @classmethod
+    def from_prompt_constructor(cls, prompt_constructor: "PromptConstructor"):
+
+        model = prompt_constructor.model
+        survey = prompt_constructor.survey
+        question = prompt_constructor.question
+        scenario = prompt_constructor.scenario
+        prior_answers_dict = prompt_constructor.prior_answers_dict()
+        agent = prompt_constructor.agent
+        return cls(
+            prompt_constructor,
+            model,
+            survey,
+            question,
+            scenario,
+            prior_answers_dict,
+            agent,
+        )
+
+    def __init__(
+        self,
+        prompt_constructor: "PromptConstructor",
+        model: "Model",
+        survey: "Survey",
+        question: "QuestionBase",
+        scenario: "Scenario",
+        prior_answers_dict: Dict[str, Any],
+        agent: "Agent",
+    ):
+
+        self.qtrb = QTRB(scenario, question, prior_answers_dict, agent)
 
-        self.model = self.prompt_constructor.model
-        self.survey = self.prompt_constructor.survey
-        self.question = self.prompt_constructor.question
+        self.model = model
+        self.survey = survey
+        self.question = question
+        self.agent = agent
+        self.scenario = scenario
+        self.prior_answers_dict = prior_answers_dict
 
     def build(self) -> Prompt:
         """Builds the complete question instructions prompt with all necessary components.
 
         Returns:
-            Prompt: The fully rendered question instructions
+            Prompt: The fully rendered question instructions to be send to the Language Model
+
+        >>> from edsl import QuestionMultipleChoice
+        >>> from edsl import Survey
+        >>> q = Survey.example().questions[0]
+        >>> from edsl import Model
+        >>> class FakePromptConstructor:
+        ...     def __init__(self, scenario, question, agent):
+        ...         self.scenario = scenario
+        ...         self.question = question
+        ...         self.agent = agent
+        ...         self.model = Model('test')
+        ...         self.survey = Survey.example()
+        ...     scenario = {"file1": "file1"}
+        ...     question = q
+        ...     agent = "agent"
+        ...     def prior_answers_dict(self):
+        ...         return {'q0': 'q0'}
+        >>> mpc = FakePromptConstructor(
+        ...     scenario={"file1": "file1"},
+        ...     question=q,
+        ...     agent="agent"
+        ... )
+        >>> qipb = QuestionInstructionPromptBuilder.from_prompt_constructor(mpc)
+        >>> qipb.build()
+        Prompt(text=\"""
+        Do you like school?
+        <BLANKLINE>
+        <BLANKLINE>
+        yes
+        <BLANKLINE>
+        no
+        <BLANKLINE>
+        <BLANKLINE>
+        Only 1 option may be selected.
+        <BLANKLINE>
+        Respond only with a string corresponding to one of the options.
+        <BLANKLINE>
+        <BLANKLINE>
+        After the answer, you can put a comment explaining why you chose that option on the next line.\""")
         """
-        import time
-        
-        start = time.time()
-        
         # Create base prompt
-        base_start = time.time()
         base_prompt = self._create_base_prompt()
-        base_end = time.time()
-        logging.debug(f"Time for base prompt: {base_end - base_start}")
-        
+
         # Enrich with options
-        enrich_start = time.time()
-        enriched_prompt = self._enrich_with_question_options(base_prompt)
-        enrich_end = time.time()
-        logging.debug(f"Time for enriching with options: {enrich_end - enrich_start}")
-        
+        enriched_prompt = self._enrich_with_question_options(
+            prompt_data=base_prompt,
+            scenario=self.scenario,
+            prior_answers_dict=self.prior_answers_dict,
+        )
+
         # Render prompt
-        render_start = time.time()
         rendered_prompt = self._render_prompt(enriched_prompt)
-        render_end = time.time()
-        logging.debug(f"Time for rendering prompt: {render_end - render_start}")
-        
+
         # Validate template variables
-        validate_start = time.time()
         self._validate_template_variables(rendered_prompt)
-        validate_end = time.time()
-        logging.debug(f"Time for template validation: {validate_end - validate_start}")
-        
+
         # Append survey instructions
-        append_start = time.time()
         final_prompt = self._append_survey_instructions(rendered_prompt)
-        append_end = time.time()
-        logging.debug(f"Time for appending survey instructions: {append_end - append_start}")
-        
-        end = time.time()
-        logging.debug(f"Total time in build_question_instructions: {end - start}")
-        
+
         return final_prompt
 
-    def _create_base_prompt(self) -> Dict:
+    def _create_base_prompt(self) -> Dict[str, Union[Prompt, Dict[str, Any]]]:
         """Creates the initial prompt with basic question data.
 
+        The data are, e.g., the question name, question text, question options, etc.
+
+        >>> from edsl import QuestionMultipleChoice
+        >>> QuestionMultipleChoice.example().data.copy()
+        {'question_name': 'how_feeling', 'question_text': 'How are you?', 'question_options': ['Good', 'Great', 'OK', 'Bad'], 'include_comment': False}
+
         Returns:
-            Dict: Base question data
+            Dict[str, Union[Prompt, Dict[str, Any]]]: Base question data with prompt and data fields
         """
         return {
             "prompt": Prompt(self.question.get_instructions(model=self.model.model)),
             "data": self.question.data.copy(),
         }
 
-    def _enrich_with_question_options(self, prompt_data: Dict) -> Dict:
-        """Enriches the prompt data with question options if they exist.
+    @staticmethod
+    def _process_question_options(
+        question_data: Dict, scenario: "Scenario", prior_answers_dict: Dict
+    ) -> Dict:
+        """Processes and replaces question options in the question data if they exist.
+
+        The question_options could be intended to be replaced with data from a scenario or prior answers.
+
+        >>> question_data = {'question_name': 'q0', 'question_text': 'Do you like school?', 'question_options': '{{ options }}'}
+        >>> scenario = {"options": ["yes", "no"]}
+        >>> prior_answers_dict = {}
+        >>> QuestionInstructionPromptBuilder._process_question_options(question_data, scenario, prior_answers_dict)
+        {'question_name': 'q0', 'question_text': 'Do you like school?', 'question_options': ['yes', 'no']}
 
         Args:
-            prompt_data: Dictionary containing prompt and question data
+            question_data: Dictionary containing question data
+            scenario: Scenario object
+            prior_answers_dict: Dictionary of prior answers
 
         Returns:
-            Dict: Enriched prompt data
+            Dict: Question data with processed question options
         """
-        import time
-        
-        start = time.time()
-        
-        if "question_options" in prompt_data["data"]:
+        if "question_options" in question_data:
             from edsl.agents.question_option_processor import QuestionOptionProcessor
-            
-            processor_start = time.time()
+
             question_options = QuestionOptionProcessor(
-                self.prompt_constructor
-            ).get_question_options(question_data=prompt_data["data"])
-            processor_end = time.time()
-            logging.debug(f"Time to process question options: {processor_end - processor_start}")
-            
-            prompt_data["data"]["question_options"] = question_options
-            
-        end = time.time()
-        logging.debug(f"Total time in _enrich_with_question_options: {end - start}")
-        
+                scenario, prior_answers_dict
+            ).get_question_options(question_data=question_data)
+            question_data["question_options"] = question_options
+
+        return question_data
+
+    @staticmethod
+    def _enrich_with_question_options(
+        prompt_data: Dict, scenario: "Scenario", prior_answers_dict: Dict
+    ) -> Dict:
+        """Enriches the prompt data with processed question options if they exist.
+
+        Args:
+            prompt_data: Dictionary containing prompt and question data
+            scenario: Scenario object
+            prior_answers_dict: Dictionary of prior answers
+
+        Returns:
+            Dict: Enriched prompt data
+        """
+        prompt_data["data"] = (
+            QuestionInstructionPromptBuilder._process_question_options(
+                prompt_data["data"], scenario, prior_answers_dict
+            )
+        )
         return prompt_data
 
     def _render_prompt(self, prompt_data: Dict) -> Prompt:
@@ -113,28 +203,11 @@ class QuestionInstructionPromptBuilder:
         Returns:
             Prompt: Rendered instructions
         """
-        import time
-        
-        start = time.time()
-        
         # Build replacement dict
-        dict_start = time.time()
-        replacement_dict = QTRB(self.prompt_constructor).build_replacement_dict(
-            prompt_data["data"]
-        )
-        dict_end = time.time()
-        logging.debug(f"Time to build replacement dict: {dict_end - dict_start}")
-        
+        replacement_dict = self.qtrb.build_replacement_dict(prompt_data["data"])
+
         # Render with dict
-        render_start = time.time()
-        result = prompt_data["prompt"].render(replacement_dict)
-        render_end = time.time()
-        logging.debug(f"Time to render with dict: {render_end - render_start}")
-        
-        end = time.time()
-        logging.debug(f"Total time in _render_prompt: {end - start}")
-        
-        return result
+        return prompt_data["prompt"].render(replacement_dict)
 
     def _validate_template_variables(self, rendered_prompt: Prompt) -> None:
         """Validates that all template variables have been properly replaced.
@@ -162,7 +235,9 @@ class QuestionInstructionPromptBuilder:
         """
         for question_name in self.survey.question_names:
             if question_name in undefined_vars:
-                logging.warning(f"Question name found in undefined_template_variables: {question_name}")
+                logging.warning(
+                    f"Question name found in undefined_template_variables: {question_name}"
+                )
 
     def _append_survey_instructions(self, rendered_prompt: Prompt) -> Prompt:
         """Appends any relevant survey instructions to the rendered prompt.
@@ -185,3 +260,9 @@ class QuestionInstructionPromptBuilder:
             preamble += instruction.text
 
         return preamble + rendered_prompt
+
+
+if __name__ == "__main__":
+    import doctest
+
+    doctest.testmod()