edsl 0.1.59__tar.gz → 0.1.60__tar.gz

{edsl-0.1.59 → edsl-0.1.60}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: edsl
-Version: 0.1.59
+Version: 0.1.60
 Summary: Create and analyze LLM-based surveys
 Home-page: https://www.expectedparrot.com/
 License: MIT
@@ -43,7 +43,7 @@ Requires-Dist: pydot (>=2.0.0,<3.0.0)
 Requires-Dist: pygments (>=2.17.2,<3.0.0)
 Requires-Dist: pymupdf (>=1.25.5,<2.0.0)
 Requires-Dist: pypdf2 (>=3.0.1,<4.0.0)
-Requires-Dist: pyreadstat (>=1.2.7,<2.0.0)
+Requires-Dist: pyreadstat (==1.2.8)
 Requires-Dist: python-docx (>=1.1.0,<2.0.0)
 Requires-Dist: python-dotenv (>=1.0.0,<2.0.0)
 Requires-Dist: python-pptx (>=1.0.2,<2.0.0)

edsl-0.1.60/edsl/__version__.py

@@ -0,0 +1 @@
+__version__ = "0.1.60"

{edsl-0.1.59 → edsl-0.1.60}/edsl/base/data_transfer_models.py

@@ -17,6 +17,7 @@ class EDSLOutput(NamedTuple):
     answer: Any
     generated_tokens: str
     comment: Optional[str] = None
+    reasoning_summary: Optional[Any] = None
 
 
 class ModelResponse(NamedTuple):
@@ -49,6 +50,7 @@ class EDSLResultObjectInput(NamedTuple):
     cache_key: str
     answer: Any
     comment: str
+    reasoning_summary: Optional[Any] = None
     validated: bool = False
     exception_occurred: Exception = None
     input_tokens: Optional[int] = None
@@ -96,12 +98,15 @@ class Answers(UserDict):
         answer = response.answer
         comment = response.comment
         generated_tokens = response.generated_tokens
+        reasoning_summary = response.reasoning_summary
         # record the answer
         if generated_tokens:
             self[question.question_name + "_generated_tokens"] = generated_tokens
         self[question.question_name] = answer
         if comment:
             self[question.question_name + "_comment"] = comment
+        if reasoning_summary:
+            self[question.question_name + "_reasoning_summary"] = reasoning_summary
 
     def replace_missing_answers_with_none(self, survey: "Survey") -> None:
         """Replace missing answers with None. Answers can be missing if the agent skips a question."""

{edsl-0.1.59 → edsl-0.1.60}/edsl/base/enums.py

@@ -57,6 +57,7 @@ class InferenceServiceType(EnumWithChecks):
     DEEP_INFRA = "deep_infra"
     REPLICATE = "replicate"
     OPENAI = "openai"
+    OPENAI_V2 = "openai_v2"
     GOOGLE = "google"
     TEST = "test"
     ANTHROPIC = "anthropic"
@@ -77,6 +78,7 @@ InferenceServiceLiteral = Literal[
     "deep_infra",
     "replicate",
     "openai",
+    "openai_v2",
     "google",
     "test",
     "anthropic",
@@ -93,6 +95,7 @@ InferenceServiceLiteral = Literal[
 available_models_urls = {
     "anthropic": "https://docs.anthropic.com/en/docs/about-claude/models",
     "openai": "https://platform.openai.com/docs/models/gp",
+    "openai_v2": "https://platform.openai.com/docs/models/gp",
     "groq": "https://console.groq.com/docs/models",
     "google": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models",
 }
@@ -102,6 +105,7 @@ service_to_api_keyname = {
     InferenceServiceType.DEEP_INFRA.value: "DEEP_INFRA_API_KEY",
     InferenceServiceType.REPLICATE.value: "TBD",
     InferenceServiceType.OPENAI.value: "OPENAI_API_KEY",
+    InferenceServiceType.OPENAI_V2.value: "OPENAI_API_KEY",
     InferenceServiceType.GOOGLE.value: "GOOGLE_API_KEY",
     InferenceServiceType.TEST.value: "TBD",
     InferenceServiceType.ANTHROPIC.value: "ANTHROPIC_API_KEY",
@@ -135,7 +139,7 @@ class TokenPricing:
             and self.prompt_token_price == other.prompt_token_price
             and self.completion_token_price == other.completion_token_price
         )
-    
+
     @classmethod
     def example(cls) -> "TokenPricing":
         """Return an example TokenPricing object."""
@@ -145,6 +149,7 @@ class TokenPricing:
             completion_token_price_per_k=0.03,
         )
 
+
 pricing = {
     "dbrx-instruct": TokenPricing(
         model_name="dbrx-instruct",
@@ -212,4 +217,4 @@ def get_token_pricing(model_name):
             model_name=model_name,
             prompt_token_price_per_k=0.0,
             completion_token_price_per_k=0.0,
-        )
+        )

{edsl-0.1.59 → edsl-0.1.60}/edsl/dataset/dataset_operations_mixin.py

@@ -357,7 +357,7 @@ class DataOperationsBase:
             4
             >>> engine = Results.example()._db(shape = "long")
             >>> len(engine.execute(text("SELECT * FROM self")).fetchall())
-            204
+            212
         """
         # Import needed for database connection
         from sqlalchemy import create_engine
@@ -442,7 +442,7 @@ class DataOperationsBase:
 
             # Using long format
             >>> len(r.sql("SELECT * FROM self", shape="long"))
-            204
+            212
         """
         import pandas as pd
 

{edsl-0.1.59 → edsl-0.1.60}/edsl/inference_services/services/__init__.py

@@ -8,6 +8,7 @@ from .groq_service import GroqService
 from .mistral_ai_service import MistralAIService
 from .ollama_service import OllamaService
 from .open_ai_service import OpenAIService
+from .open_ai_service_v2 import OpenAIServiceV2
 from .perplexity_service import PerplexityService
 from .test_service import TestService
 from .together_ai_service import TogetherAIService
@@ -24,8 +25,9 @@ __all__ = [
     "MistralAIService",
     "OllamaService",
     "OpenAIService",
+    "OpenAIServiceV2",
     "PerplexityService",
     "TestService",
     "TogetherAIService",
     "XAIService",
-] 
+]

edsl-0.1.60/edsl/inference_services/services/open_ai_service_v2.py

@@ -0,0 +1,243 @@
+from __future__ import annotations
+from typing import Any, List, Optional, Dict, NewType, TYPE_CHECKING
+import os
+
+import openai
+
+from ..inference_service_abc import InferenceServiceABC
+
+# Use TYPE_CHECKING to avoid circular imports at runtime
+if TYPE_CHECKING:
+    from ...language_models import LanguageModel
+from ..rate_limits_cache import rate_limits
+
+# Default to completions API but can use responses API with parameter
+
+if TYPE_CHECKING:
+    from ....scenarios.file_store import FileStore as Files
+    from ....invigilators.invigilator_base import InvigilatorBase as InvigilatorAI
+
+
+APIToken = NewType("APIToken", str)
+
+
+class OpenAIServiceV2(InferenceServiceABC):
+    """OpenAI service class using the Responses API."""
+
+    _inference_service_ = "openai_v2"
+    _env_key_name_ = "OPENAI_API_KEY"
+    _base_url_ = None
+
+    _sync_client_ = openai.OpenAI
+    _async_client_ = openai.AsyncOpenAI
+
+    _sync_client_instances: Dict[APIToken, openai.OpenAI] = {}
+    _async_client_instances: Dict[APIToken, openai.AsyncOpenAI] = {}
+
+    # sequence to extract text from response.output
+    key_sequence = ["output", 1, "content", 0, "text"]
+    usage_sequence = ["usage"]
+    # sequence to extract reasoning summary from response.output
+    reasoning_sequence = ["output", 0, "summary"]
+    input_token_name = "prompt_tokens"
+    output_token_name = "completion_tokens"
+
+    available_models_url = "https://platform.openai.com/docs/models/gp"
+
+    def __init_subclass__(cls, **kwargs):
+        super().__init_subclass__(**kwargs)
+        cls._sync_client_instances = {}
+        cls._async_client_instances = {}
+
+    @classmethod
+    def sync_client(cls, api_key: str) -> openai.OpenAI:
+        if api_key not in cls._sync_client_instances:
+            client = cls._sync_client_(
+                api_key=api_key,
+                base_url=cls._base_url_,
+            )
+            cls._sync_client_instances[api_key] = client
+        return cls._sync_client_instances[api_key]
+
+    @classmethod
+    def async_client(cls, api_key: str) -> openai.AsyncOpenAI:
+        if api_key not in cls._async_client_instances:
+            client = cls._async_client_(
+                api_key=api_key,
+                base_url=cls._base_url_,
+            )
+            cls._async_client_instances[api_key] = client
+        return cls._async_client_instances[api_key]
+
+    model_exclude_list = [
+        "whisper-1",
+        "davinci-002",
+        "dall-e-2",
+        "tts-1-hd-1106",
+        "tts-1-hd",
+        "dall-e-3",
+        "tts-1",
+        "babbage-002",
+        "tts-1-1106",
+        "text-embedding-3-large",
+        "text-embedding-3-small",
+        "text-embedding-ada-002",
+        "ft:davinci-002:mit-horton-lab::8OfuHgoo",
+        "gpt-3.5-turbo-instruct-0914",
+        "gpt-3.5-turbo-instruct",
+    ]
+    _models_list_cache: List[str] = []
+
+    @classmethod
+    def get_model_list(cls, api_key: str | None = None) -> List[str]:
+        if api_key is None:
+            api_key = os.getenv(cls._env_key_name_)
+        raw = cls.sync_client(api_key).models.list()
+        return raw.data if hasattr(raw, "data") else raw
+
+    @classmethod
+    def available(cls, api_token: str | None = None) -> List[str]:
+        if api_token is None:
+            api_token = os.getenv(cls._env_key_name_)
+        if not cls._models_list_cache:
+            data = cls.get_model_list(api_key=api_token)
+            cls._models_list_cache = [
+                m.id for m in data if m.id not in cls.model_exclude_list
+            ]
+        return cls._models_list_cache
+
+    @classmethod
+    def create_model(
+        cls,
+        model_name: str,
+        model_class_name: str | None = None,
+    ) -> LanguageModel:
+        if model_class_name is None:
+            model_class_name = cls.to_class_name(model_name)
+
+        from ...language_models import LanguageModel
+
+        class LLM(LanguageModel):
+            """Child class for OpenAI Responses API"""
+
+            key_sequence = cls.key_sequence
+            usage_sequence = cls.usage_sequence
+            reasoning_sequence = cls.reasoning_sequence
+            input_token_name = cls.input_token_name
+            output_token_name = cls.output_token_name
+            _inference_service_ = cls._inference_service_
+            _model_ = model_name
+            _parameters_ = {
+                "temperature": 0.5,
+                "max_tokens": 2000,
+                "top_p": 1,
+                "frequency_penalty": 0,
+                "presence_penalty": 0,
+                "logprobs": False,
+                "top_logprobs": 3,
+            }
+
+            def sync_client(self) -> openai.OpenAI:
+                return cls.sync_client(api_key=self.api_token)
+
+            def async_client(self) -> openai.AsyncOpenAI:
+                return cls.async_client(api_key=self.api_token)
+
+            @classmethod
+            def available(cls) -> list[str]:
+                return cls.sync_client().models.list().data
+
+            def get_headers(self) -> dict[str, Any]:
+                client = self.sync_client()
+                response = client.responses.with_raw_response.create(
+                    model=self.model,
+                    input=[{"role": "user", "content": "Say this is a test"}],
+                    store=False,
+                )
+                return dict(response.headers)
+
+            def get_rate_limits(self) -> dict[str, Any]:
+                try:
+                    headers = rate_limits.get("openai", self.get_headers())
+                except Exception:
+                    return {"rpm": 10000, "tpm": 2000000}
+                return {
+                    "rpm": int(headers["x-ratelimit-limit-requests"]),
+                    "tpm": int(headers["x-ratelimit-limit-tokens"]),
+                }
+
+            async def async_execute_model_call(
+                self,
+                user_prompt: str,
+                system_prompt: str = "",
+                files_list: Optional[List[Files]] = None,
+                invigilator: Optional[InvigilatorAI] = None,
+            ) -> dict[str, Any]:
+                content = user_prompt
+                if files_list:
+                    # embed files as separate inputs
+                    content = [{"type": "text", "text": user_prompt}]
+                    for f in files_list:
+                        content.append(
+                            {
+                                "type": "image_url",
+                                "image_url": {
+                                    "url": f"data:{f.mime_type};base64,{f.base64_string}"
+                                },
+                            }
+                        )
+                # build input sequence
+                messages: Any
+                if system_prompt and not self.omit_system_prompt_if_empty:
+                    messages = [
+                        {"role": "system", "content": system_prompt},
+                        {"role": "user", "content": content},
+                    ]
+                else:
+                    messages = [{"role": "user", "content": content}]
+
+                # All OpenAI models with the responses API use these base parameters
+                params = {
+                    "model": self.model,
+                    "input": messages,
+                    "temperature": self.temperature,
+                    "top_p": self.top_p,
+                    "store": False,
+                }
+                
+                # Check if this is a reasoning model (o-series models)
+                is_reasoning_model = any(tag in self.model for tag in ["o1", "o1-mini", "o3", "o3-mini", "o1-pro", "o4-mini"])
+                
+                # Only add reasoning parameter for reasoning models
+                if is_reasoning_model:
+                    params["reasoning"] = {"summary": "auto"}
+                
+                # For all models using the responses API, use max_output_tokens
+                # instead of max_tokens (which is for the completions API)
+                params["max_output_tokens"] = self.max_tokens
+                
+                # Specifically for o-series, we also set temperature to 1
+                if is_reasoning_model:
+                    params["temperature"] = 1
+
+                client = self.async_client()
+                try:
+                    response = await client.responses.create(**params)
+
+                except Exception as e:
+                    return {"message": str(e)}
+
+                # convert to dict
+                response_dict = response.model_dump()
+                return response_dict
+
+        LLM.__name__ = model_class_name
+        return LLM
+
+    @staticmethod
+    def _create_reasoning_sequence():
+        """Create the reasoning sequence for extracting reasoning summaries from model responses."""
+        # For OpenAI responses, the reasoning summary is typically found at:
+        # ["output", 0, "summary"]
+        # This is the path to the 'summary' field in the first item of the 'output' array
+        return ["output", 0, "summary"]

{edsl-0.1.59 → edsl-0.1.60}/edsl/jobs/data_structures.py

@@ -213,6 +213,9 @@ class Answers(UserDict):
         if comment:
             self[question.question_name + "_comment"] = comment
 
+        if getattr(response, "reasoning_summary", None):
+            self[question.question_name + "_reasoning_summary"] = response.reasoning_summary
+
     def replace_missing_answers_with_none(self, survey: "Survey") -> None:
         """
         Replace missing answers with None for all questions in the survey.

{edsl-0.1.59 → edsl-0.1.60}/edsl/key_management/key_lookup_builder.py

@@ -363,13 +363,35 @@ class KeyLookupBuilder:
         >>> builder._add_api_key("OPENAI_API_KEY", "sk-1234", "env")
         >>> 'sk-1234' == builder.key_data["openai"][-1].value
         True
+        >>> 'sk-1234' == builder.key_data["openai_v2"][-1].value
+        True
         """
         service = api_keyname_to_service[key]
         new_entry = APIKeyEntry(service=service, name=key, value=value, source=source)
-        if service not in self.key_data:
-            self.key_data[service] = [new_entry]
+        
+        # Special case for OPENAI_API_KEY - add to both openai and openai_v2
+        if key == "OPENAI_API_KEY":
+            # Add to openai service
+            openai_service = "openai"
+            openai_entry = APIKeyEntry(service=openai_service, name=key, value=value, source=source)
+            if openai_service not in self.key_data:
+                self.key_data[openai_service] = [openai_entry]
+            else:
+                self.key_data[openai_service].append(openai_entry)
+                
+            # Add to openai_v2 service
+            openai_v2_service = "openai_v2"
+            openai_v2_entry = APIKeyEntry(service=openai_v2_service, name=key, value=value, source=source)
+            if openai_v2_service not in self.key_data:
+                self.key_data[openai_v2_service] = [openai_v2_entry]
+            else:
+                self.key_data[openai_v2_service].append(openai_v2_entry)
         else:
-            self.key_data[service].append(new_entry)
+            # Normal case for all other API keys
+            if service not in self.key_data:
+                self.key_data[service] = [new_entry]
+            else:
+                self.key_data[service].append(new_entry)
 
     def update_from_dict(self, d: dict) -> None:
         """

{edsl-0.1.59 → edsl-0.1.60}/edsl/language_models/language_model.py

@@ -174,7 +174,8 @@ class LanguageModel(
         """
         key_sequence = cls.key_sequence
         usage_sequence = cls.usage_sequence if hasattr(cls, "usage_sequence") else None
-        return RawResponseHandler(key_sequence, usage_sequence)
+        reasoning_sequence = cls.reasoning_sequence if hasattr(cls, "reasoning_sequence") else None
+        return RawResponseHandler(key_sequence, usage_sequence, reasoning_sequence)
 
     def __init__(
         self,

edsl-0.1.60/edsl/language_models/raw_response_handler.py

@@ -0,0 +1,230 @@
+import json
+from typing import Optional, Any, List
+from .exceptions import (
+    LanguageModelBadResponseError,
+    LanguageModelTypeError,
+    LanguageModelIndexError,
+    LanguageModelKeyError
+)
+
+from json_repair import repair_json
+
+
+def _extract_item_from_raw_response(data, sequence):
+    if isinstance(data, str):
+        try:
+            data = json.loads(data)
+        except json.JSONDecodeError:
+            return data
+    current_data = data
+    for i, key in enumerate(sequence):
+        try:
+            if isinstance(current_data, (list, tuple)):
+                if not isinstance(key, int):
+                    raise LanguageModelTypeError(
+                        f"Expected integer index for sequence at position {i}, got {type(key).__name__}"
+                    )
+                if key < 0 or key >= len(current_data):
+                    raise LanguageModelIndexError(
+                        f"Index {key} out of range for sequence of length {len(current_data)} at position {i}"
+                    )
+            elif isinstance(current_data, dict):
+                if key not in current_data:
+                    raise LanguageModelKeyError(
+                        f"Key '{key}' not found in dictionary at position {i}"
+                    )
+            else:
+                raise LanguageModelTypeError(
+                    f"Cannot index into {type(current_data).__name__} at position {i}. Full response is: {data} of type {type(data)}. Key sequence is: {sequence}"
+                )
+
+            current_data = current_data[key]
+        except Exception as e:
+            path = " -> ".join(map(str, sequence[: i + 1]))
+            
+            # Create a safe error message that won't be None
+            if "error" in data and data["error"] is not None:
+                msg = str(data["error"])
+            else:
+                msg = f"Error accessing path: {path}. {str(e)}. Full response is: '{data}'"
+                
+            raise LanguageModelBadResponseError(message=msg, response_json=data)
+    if isinstance(current_data, str):
+        return current_data.strip()
+    else:
+        return current_data
+
+
+class RawResponseHandler:
+    """Class to handle raw responses from language models."""
+
+    def __init__(self, key_sequence: list, usage_sequence: Optional[list] = None, reasoning_sequence: Optional[list] = None):
+        self.key_sequence = key_sequence
+        self.usage_sequence = usage_sequence
+        self.reasoning_sequence = reasoning_sequence
+
+    def get_generated_token_string(self, raw_response):
+        try:
+            return _extract_item_from_raw_response(raw_response, self.key_sequence)
+        except (LanguageModelKeyError, LanguageModelIndexError, LanguageModelTypeError, LanguageModelBadResponseError) as e:
+            # For non-reasoning models or reasoning models with different response formats,
+            # try to extract text directly from common response formats
+            if isinstance(raw_response, dict):
+                # Responses API format for non-reasoning models
+                if 'output' in raw_response and isinstance(raw_response['output'], list):
+                    # Try to get first message content
+                    if len(raw_response['output']) > 0:
+                        item = raw_response['output'][0]
+                        if isinstance(item, dict) and 'content' in item:
+                            if isinstance(item['content'], list) and len(item['content']) > 0:
+                                first_content = item['content'][0]
+                                if isinstance(first_content, dict) and 'text' in first_content:
+                                    return first_content['text']
+                            elif isinstance(item['content'], str):
+                                return item['content']
+                
+                # OpenAI completions format
+                if 'choices' in raw_response and isinstance(raw_response['choices'], list) and len(raw_response['choices']) > 0:
+                    choice = raw_response['choices'][0]
+                    if isinstance(choice, dict):
+                        if 'text' in choice:
+                            return choice['text']
+                        elif 'message' in choice and isinstance(choice['message'], dict) and 'content' in choice['message']:
+                            return choice['message']['content']
+                
+                # Text directly in response
+                if 'text' in raw_response:
+                    return raw_response['text']
+                elif 'content' in raw_response:
+                    return raw_response['content']
+                
+                # Error message - try to return a coherent error for debugging
+                if 'message' in raw_response:
+                    return f"[ERROR: {raw_response['message']}]"
+            
+            # If we get a string directly, return it
+            if isinstance(raw_response, str):
+                return raw_response
+            
+            # As a last resort, convert the whole response to string
+            try:
+                return f"[ERROR: Could not extract text. Raw response: {str(raw_response)}]"
+            except:
+                return "[ERROR: Could not extract text from response]"
+
+    def get_usage_dict(self, raw_response):
+        if self.usage_sequence is None:
+            return {}
+        try:
+            return _extract_item_from_raw_response(raw_response, self.usage_sequence)
+        except (LanguageModelKeyError, LanguageModelIndexError, LanguageModelTypeError, LanguageModelBadResponseError):
+            # For non-reasoning models, try to extract usage from common response formats
+            if isinstance(raw_response, dict):
+                # Standard OpenAI usage format
+                if 'usage' in raw_response:
+                    return raw_response['usage']
+                
+                # Look for nested usage info
+                if 'choices' in raw_response and len(raw_response['choices']) > 0:
+                    choice = raw_response['choices'][0]
+                    if isinstance(choice, dict) and 'usage' in choice:
+                        return choice['usage']
+            
+            # If no usage info found, return empty dict
+            return {}
+
+    def get_reasoning_summary(self, raw_response):
+        """
+        Extract reasoning summary from the model response.
+        
+        Handles various response structures:
+        1. Standard path extraction using self.reasoning_sequence
+        2. Direct access to output[0]['summary'] for OpenAI responses
+        3. List responses where the first item contains the output structure
+        """
+        if self.reasoning_sequence is None:
+            return None
+            
+        try:
+            # First try the standard extraction path
+            summary_data = _extract_item_from_raw_response(raw_response, self.reasoning_sequence)
+            
+            # If summary_data is a list of dictionaries with 'text' and 'type' fields
+            # (as in OpenAI's response format), combine them into a single string
+            if isinstance(summary_data, list) and all(isinstance(item, dict) and 'text' in item for item in summary_data):
+                return '\n\n'.join(item['text'] for item in summary_data)
+            
+            return summary_data
+        except Exception:
+            # Fallback approaches for different response structures
+            try:
+                # Case 1: Direct dict with 'output' field (common OpenAI format)
+                if isinstance(raw_response, dict) and 'output' in raw_response:
+                    output = raw_response['output']
+                    if isinstance(output, list) and len(output) > 0 and 'summary' in output[0]:
+                        summary_data = output[0]['summary']
+                        if isinstance(summary_data, list) and all(isinstance(item, dict) and 'text' in item for item in summary_data):
+                            return '\n\n'.join(item['text'] for item in summary_data)
+                
+                # Case 2: List where the first item is a dict with 'output' field
+                if isinstance(raw_response, list) and len(raw_response) > 0:
+                    first_item = raw_response[0]
+                    if isinstance(first_item, dict) and 'output' in first_item:
+                        output = first_item['output']
+                        if isinstance(output, list) and len(output) > 0 and 'summary' in output[0]:
+                            summary_data = output[0]['summary']
+                            if isinstance(summary_data, list) and all(isinstance(item, dict) and 'text' in item for item in summary_data):
+                                return '\n\n'.join(item['text'] for item in summary_data)
+            except Exception:
+                pass
+            
+            return None
+
+    def parse_response(self, raw_response: dict[str, Any]) -> Any:
+        """Parses the API response and returns the response text."""
+
+        from edsl.data_transfer_models import EDSLOutput
+
+        generated_token_string = self.get_generated_token_string(raw_response)
+        # Ensure generated_token_string is a string before using string methods
+        if not isinstance(generated_token_string, str):
+            generated_token_string = str(generated_token_string)
+        last_newline = generated_token_string.rfind("\n")
+        reasoning_summary = self.get_reasoning_summary(raw_response)
+
+        if last_newline == -1:
+            # There is no comment
+            edsl_dict = {
+                "answer": self.convert_answer(generated_token_string),
+                "generated_tokens": generated_token_string,
+                "comment": None,
+                "reasoning_summary": reasoning_summary,
+            }
+        else:
+            edsl_dict = {
+                "answer": self.convert_answer(generated_token_string[:last_newline]),
+                "comment": generated_token_string[last_newline + 1:].strip(),
+                "generated_tokens": generated_token_string,
+                "reasoning_summary": reasoning_summary,
+            }
+        return EDSLOutput(**edsl_dict)
+
+    @staticmethod
+    def convert_answer(response_part):
+        import json
+
+        response_part = response_part.strip()
+
+        if response_part == "None":
+            return None
+
+        repaired = repair_json(response_part)
+        if repaired == '""':
+            # it was a literal string
+            return response_part
+
+        try:
+            return json.loads(repaired)
+        except json.JSONDecodeError:
+            # last resort
+            return response_part