edsl 0.1.53__py3-none-any.whl → 0.1.55__py3-none-any.whl

edsl/inference_services/services/test_service.py

@@ -4,11 +4,14 @@ import random
 
 from ..inference_service_abc import InferenceServiceABC
 
-from ...language_models import LanguageModel
 from ...enums import InferenceServiceType
 
+# Use TYPE_CHECKING to avoid circular imports at runtime
 if TYPE_CHECKING:
-    from ....scenarios.file_store import FileStore as File
+    from ...language_models import LanguageModel
+
+if TYPE_CHECKING:
+    from ...scenarios.file_store import FileStore as File
 
 
 class TestService(InferenceServiceABC):
@@ -36,9 +39,12 @@ class TestService(InferenceServiceABC):
         return ["test"]
 
     @classmethod
-    def create_model(cls, model_name, model_class_name=None) -> LanguageModel:
+    def create_model(cls, model_name, model_class_name=None) -> "LanguageModel":
         # Removed unused variable
 
+        # Import LanguageModel only when actually creating a model
+        from ...language_models import LanguageModel
+
         class TestServiceLanguageModel(LanguageModel):
             _model_ = "test"
             _parameters_ = {"temperature": 0.5}
@@ -53,7 +59,6 @@ class TestService(InferenceServiceABC):
             @property
             def _canned_response(self):
                 if hasattr(self, "canned_response"):
-
                     return self.canned_response
                 else:
                     return "Hello, world X"
@@ -64,6 +69,7 @@ class TestService(InferenceServiceABC):
                 system_prompt: str,
                 # func: Optional[callable] = None,
                 files_list: Optional[List["File"]] = None,
+                question_name: Optional[str] = None,
             ) -> dict[str, Any]:
                 await asyncio.sleep(0.1)
 
@@ -74,8 +80,9 @@ class TestService(InferenceServiceABC):
                         p = 1
 
                     if random.random() < p:
-                        from ..exceptions import InferenceServiceError
-                        raise InferenceServiceError("This is a test error")
+                        from ..exceptions import InferenceServiceIntendedError
+
+                        raise InferenceServiceIntendedError("This is a test error")
 
                 if hasattr(self, "func"):
                     return {
@@ -85,9 +92,100 @@ class TestService(InferenceServiceABC):
                         "usage": {"prompt_tokens": 1, "completion_tokens": 1},
                     }
 
+                response = self._canned_response
+                if isinstance(response, dict) and question_name:
+                    canned_text = response.get(
+                        question_name, f"No canned response for '{question_name}'"
+                    )
+                else:
+                    canned_text = response
+
                 return {
-                    "message": [{"text": f"{self._canned_response}"}],
+                    "message": [{"text": f"{canned_text}"}],
                     "usage": {"prompt_tokens": 1, "completion_tokens": 1},
                 }
 
+            def set_canned_response(self, survey: "Survey") -> None:
+                from edsl import Model
+                from edsl.questions import (
+                    QuestionMultipleChoice,
+                    QuestionCheckBox,
+                    QuestionLinearScale,
+                    QuestionList,
+                    QuestionDict,
+                    QuestionNumerical,
+                    QuestionFreeText,
+                )
+
+                canned_response = {}
+
+                for q in survey.questions:
+                    name = q.question_name
+
+                    if isinstance(q, QuestionMultipleChoice):
+                        # Return first option
+                        canned_response[name] = q.question_options[0]
+
+                    elif isinstance(q, QuestionCheckBox):
+                        # Return first two options as a list
+                        canned_response[name] = q.question_options[:2]
+
+                    elif isinstance(q, QuestionLinearScale):
+                        # Return middle of the scale
+                        values = q.question_options
+                        if isinstance(values, list) and all(
+                            isinstance(i, int) for i in values
+                        ):
+                            mid = values[len(values) // 2]
+                            canned_response[name] = mid
+                        else:
+                            canned_response[name] = 5  # default fallback
+
+                    elif isinstance(q, QuestionNumerical):
+                        # Return a fixed float value
+                        canned_response[name] = 42.0
+
+                    elif isinstance(q, QuestionList):
+                        # Return a list of simple strings
+                        canned_response[name] = [f"{name} item 1", f"{name} item 2"]
+
+                    elif isinstance(q, QuestionDict):
+                        # Handle response types for each key
+                        keys = getattr(q, "answer_keys", ["field1", "field2"])
+                        value_types = getattr(q, "value_types", [])
+                        canned_response[name] = {}
+
+                        for i, key in enumerate(keys):
+                            # Check the type for each key and generate the appropriate response
+                            response_type = (
+                                value_types[i] if i < len(value_types) else "string"
+                            )  # Default to "string" if not provided
+
+                            if "str" in response_type:
+                                canned_response[name][key] = f"{key} value"
+                            elif "int" in response_type:
+                                canned_response[name][
+                                    key
+                                ] = 42  # Example integer response
+                            elif "float" in response_type:
+                                canned_response[name][
+                                    key
+                                ] = 42.0  # Example float response
+                            elif "bool" in response_type:
+                                canned_response[name][
+                                    key
+                                ] = True  # Example boolean response
+                            else:
+                                canned_response[name][key] = f"{key} unknown type"
+
+                    elif isinstance(q, QuestionFreeText):
+                        # Return a string
+                        canned_response[name] = f"This is a canned answer for {name}."
+
+                    else:
+                        # Fallback: simple string
+                        canned_response[name] = f"Canned fallback for {name}"
+
+                self.canned_response = canned_response
+
         return TestServiceLanguageModel

edsl/interviews/answering_function.py

@@ -1,5 +1,6 @@
 import asyncio
 import copy
+import weakref
 from typing import TYPE_CHECKING, Any, Callable, Union
 
 if TYPE_CHECKING:
@@ -27,21 +28,73 @@ class RetryConfig:
 
 class SkipHandler:
     def __init__(self, interview: "Interview"):
-        self.interview = interview
-        self.question_index = self.interview.to_index
+        # Store a weak reference to the interview
+        self._interview_ref = weakref.ref(interview)
 
-        self.skip_function: Callable = (
-            self.interview.survey.rule_collection.skip_question_before_running
-        )
+        # Cache only the skip function which doesn't maintain a reference to the interview
+        try:
+            self.skip_function: Callable = (
+                interview.survey.rule_collection.skip_question_before_running
+            )
+        except (AttributeError, KeyError):
+            # Fallback for test environments
+            self.skip_function = lambda *args: False
+
+    @property
+    def interview(self):
+        """Access the interview via weak reference if it still exists."""
+        interview = self._interview_ref()
+        if interview is None:
+            raise RuntimeError("Interview has been garbage collected")
+        return interview
+
+    @property
+    def _to_index(self):
+        return self.interview.to_index
+
+    @property
+    def _survey(self):
+        return self.interview.survey
+
+    @property
+    def _answers(self):
+        return self.interview.answers
+
+    @property
+    def _scenario(self):
+        return self.interview.scenario
+
+    @property
+    def _agent_traits(self):
+        try:
+            return self.interview.agent["traits"]
+        except (AttributeError, KeyError):
+            return {}
+
+    @property
+    def _skip_flags(self):
+        return self.interview.skip_flags
 
     def should_skip(self, current_question: "QuestionBase") -> bool:
         """Determine if the current question should be skipped."""
-        current_question_index = self.question_index[current_question.question_name]
-        combined_answers = (
-            self.interview.answers
-            | self.interview.scenario
-            | self.interview.agent["traits"]
-        )
+        current_question_index = self._to_index[current_question.question_name]
+
+        # Handle ScenarioList case - convert to dict first
+        scenario_dict = {}
+        if hasattr(self._scenario, "items"):
+            # Handle standard dict scenario
+            scenario_dict = self._scenario
+        else:
+            # Handle ScenarioList or other scenario object
+            # Access as a dict if possible, otherwise try to convert
+            scenario_dict = (
+                dict(self._scenario) if hasattr(self._scenario, "__iter__") else {}
+            )
+
+        combined_answers = dict(self._answers)
+        combined_answers.update(scenario_dict)
+        combined_answers.update(self._agent_traits)
+
         return self.skip_function(current_question_index, combined_answers)
 
     def _current_info_env(self) -> dict[str, Any]:
@@ -52,7 +105,7 @@ class SkipHandler:
         """
         # Process answers dictionary
         processed_answers = {}
-        for key, value in self.interview.answers.items():
+        for key, value in self._answers.items():
             if key.endswith("_generated_tokens"):
                 base_name = key.replace("_generated_tokens", "")
                 processed_answers[f"{base_name}.generated_tokens"] = value
@@ -64,33 +117,22 @@ class SkipHandler:
                 processed_answers[f"{key}.answer"] = value
 
         # Process scenario dictionary
-        processed_scenario = {
-            f"scenario.{k}": v for k, v in self.interview.scenario.items()
-        }
+        processed_scenario = {f"scenario.{k}": v for k, v in self._scenario.items()}
 
         # Process agent traits
-        processed_agent = {
-            f"agent.{k}": v for k, v in self.interview.agent["traits"].items()
-        }
+        processed_agent = {f"agent.{k}": v for k, v in self._agent_traits.items()}
 
         return processed_answers | processed_scenario | processed_agent
 
     def cancel_skipped_questions(self, current_question: "QuestionBase") -> None:
         """Cancel the tasks for questions that should be skipped."""
-        current_question_index: int = self.interview.to_index[
-            current_question.question_name
-        ]
+        current_question_index: int = self._to_index[current_question.question_name]
         answers = self._current_info_env()
-        # answers = (
-        #     self.interview.answers
-        #     | self.interview.scenario
-        #     | self.interview.agent["traits"]
-        # )
 
         # Get the index of the next question, which could also be the end of the survey
         next_question: Union[
             int, EndOfSurvey
-        ] = self.interview.survey.rule_collection.next_question(
+        ] = self._survey.rule_collection.next_question(
             q_now=current_question_index,
             answers=answers,
         )
@@ -101,14 +143,15 @@ class SkipHandler:
                 # print(f"Cancelling task {i}")
                 # self.interview.tasks[i].cancel()
                 # self.interview.tasks[i].set_result("skipped")
-                self.interview.skip_flags[
-                    self.interview.survey.questions[i].question_name
-                ] = True
+                interview = self._interview_ref()
+                if interview is not None:
+                    interview.skip_flags[self._survey.questions[i].question_name] = True
+                else:
+                    # If interview is gone, there's nothing to skip anymore
+                    return
 
         if (next_question_index := next_question.next_q) == EndOfSurvey:
-            cancel_between(
-                current_question_index + 1, len(self.interview.survey.questions)
-            )
+            cancel_between(current_question_index + 1, len(self._survey.questions))
             return
 
         if next_question_index > (current_question_index + 1):
@@ -119,48 +162,90 @@ class AnswerQuestionFunctionConstructor:
     """Constructs a function that answers a question and records the answer."""
 
     def __init__(self, interview: "Interview", key_lookup: "KeyLookup"):
-        self.interview = interview
+        # Store a weak reference to the interview
+        self._interview_ref = weakref.ref(interview)
         self.key_lookup = key_lookup
 
-        self.had_language_model_no_response_error: bool = False
-        self.question_index = self.interview.to_index
-
-        self.skip_function: Callable = (
-            self.interview.survey.rule_collection.skip_question_before_running
+        # Store configuration settings that won't change during lifecycle
+        self._raise_validation_errors = getattr(
+            interview, "raise_validation_errors", False
         )
+        self._stop_on_exception = getattr(interview, "stop_on_exception", False)
 
+        self.had_language_model_no_response_error: bool = False
+
+        # Initialize fetch invigilator with the interview - this should use weakref internally
         self.invigilator_fetcher = FetchInvigilator(
-            self.interview, key_lookup=self.key_lookup
+            interview, key_lookup=self.key_lookup
         )
-        self.skip_handler = SkipHandler(self.interview)
+
+        # In our test environment, we might not be able to create the SkipHandler
+        # because example Interview might not have all required attributes
+        # So we'll initialize it conditionally
+        if hasattr(interview, "skip_flags"):
+            self.skip_handler = SkipHandler(interview)
+        else:
+            self.skip_handler = None
+
+    @property
+    def interview(self):
+        """Access the interview via weak reference if it still exists."""
+        interview = self._interview_ref()
+        if interview is None:
+            raise RuntimeError("Interview has been garbage collected")
+        return interview
+
+    @property
+    def _answers(self):
+        return self.interview.answers
+
+    @property
+    def _exceptions(self):
+        return self.interview.exceptions
+
+    @property
+    def _to_index(self):
+        return self.interview.to_index
+
+    @property
+    def _skip_flags(self):
+        if hasattr(self.interview, "skip_flags"):
+            return self.interview.skip_flags
+        return {}
 
     def _handle_exception(
         self, e: Exception, invigilator: "InvigilatorBase", task=None
     ):
         """Handle an exception that occurred while answering a question."""
+        interview = self._interview_ref()
+        if interview is None:
+            # If interview is gone, we can't really handle the exception properly
+            # Just raise it to the caller
+            raise e
+
+        # Copy to freeze the answers here for logging
+        answers = copy.copy(self._answers)
 
-        answers = copy.copy(
-            self.interview.answers
-        )  # copy to freeze the answers here for logging
         exception_entry = InterviewExceptionEntry(
             exception=e,
             invigilator=invigilator,
             answers=answers,
         )
+
         if task:
             task.task_status = TaskStatus.FAILED
 
-        self.interview.exceptions.add(
-            invigilator.question.question_name, exception_entry
-        )
+        # Add to exceptions - need to use the interview reference here
+        interview.exceptions.add(invigilator.question.question_name, exception_entry)
 
-        if self.interview.raise_validation_errors and isinstance(
+        # Check if we should raise validation errors
+        if self._raise_validation_errors and isinstance(
             e, QuestionAnswerValidationError
         ):
             raise e
 
-        stop_on_exception = getattr(self.interview, "stop_on_exception", False)
-        if stop_on_exception:
+        # Check if we should stop on exception
+        if self._stop_on_exception:
             raise e
 
     def __call__(self):
@@ -190,27 +275,46 @@ class AnswerQuestionFunctionConstructor:
             reraise=True,
         )
         async def attempt_answer():
+            # Get a reference to the interview (may be None if it's been garbage collected)
+            interview = self._interview_ref()
+
+            # Get the invigilator for this question
             invigilator = self.invigilator_fetcher(question)
 
-            if self.interview.skip_flags.get(question.question_name, False):
+            # Check if interview still exists
+            if interview is None:
+                # If interview is gone, we can't really process this question
+                # Return a failure result
+                return invigilator.get_failed_task_result(
+                    failure_reason="Interview has been garbage collected."
+                )
+
+            # Check if question should be skipped - use cached skip_flags if available
+            skip_flags = getattr(self, "_skip_flags", None) or interview.skip_flags
+            if skip_flags.get(question.question_name, False):
                 return invigilator.get_failed_task_result(
                     failure_reason="Question skipped."
                 )
-            if self.skip_handler.should_skip(question):
+
+            if self.skip_handler and self.skip_handler.should_skip(question):
                 return invigilator.get_failed_task_result(
                     failure_reason="Question skipped."
                 )
 
+            had_language_model_no_response_error = False
             try:
                 response: EDSLResultObjectInput = (
                     await invigilator.async_answer_question()
                 )
                 if response.validated:
-                    self.interview.answers.add_answer(
-                        response=response, question=question
-                    )
-
-                    self.skip_handler.cancel_skipped_questions(question)
+                    # Re-check if interview exists before updating it
+                    interview = self._interview_ref()
+                    if interview is not None:
+                        interview.answers.add_answer(
+                            response=response, question=question
+                        )
+                        if self.skip_handler:
+                            self.skip_handler.cancel_skipped_questions(question)
                 else:
                     if (
                         hasattr(response, "exception_occurred")
@@ -239,11 +343,15 @@ class AnswerQuestionFunctionConstructor:
                 raise LanguageModelNoResponseError(
                     f"Language model did not return a response for question '{question.question_name}.'"
                 )
+
+            # Re-check if interview exists before accessing exceptions
+            interview = self._interview_ref()
             if (
-                question.question_name in self.interview.exceptions
+                interview is not None
+                and question.question_name in interview.exceptions
                 and had_language_model_no_response_error
             ):
-                self.interview.exceptions.record_fixed_question(question.question_name)
+                interview.exceptions.record_fixed_question(question.question_name)
 
             return response