nvidia-nat 1.1.0a20251020__py3-none-any.whl

nat/experimental/test_time_compute/functions/ttc_tool_orchestration_function.py

@@ -0,0 +1,205 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import asyncio
+import logging
+
+from pydantic import Field
+
+from nat.builder.builder import Builder
+from nat.builder.framework_enum import LLMFrameworkEnum
+from nat.builder.function_info import FunctionInfo
+from nat.cli.register_workflow import register_function
+from nat.data_models.component_ref import FunctionRef
+from nat.data_models.component_ref import TTCStrategyRef
+from nat.data_models.function import FunctionBaseConfig
+from nat.experimental.test_time_compute.models.stage_enums import PipelineTypeEnum
+from nat.experimental.test_time_compute.models.stage_enums import StageTypeEnum
+from nat.experimental.test_time_compute.models.tool_use_config import ToolUseInputSchema
+from nat.experimental.test_time_compute.models.tool_use_config import ToolUselist
+from nat.experimental.test_time_compute.models.ttc_item import TTCItem
+
+logger = logging.getLogger(__name__)
+
+
+class TTCToolOrchestrationFunctionConfig(FunctionBaseConfig, name="ttc_tool_orchestration"):
+    """
+    Configuration for the TTCToolOrchestrationFunction, which is used to orchestrate multiple functions.
+    """
+
+    augmented_fns: list[FunctionRef] = Field(
+        description="list of FunctionRefs for the functions to be orchestrated. Must be wrapped in `ttc_tool_wrapper`.")
+
+    search_strategy: TTCStrategyRef | None = Field(
+        description="The TTC search strategy to use for orchestrating invocation of the functions."
+        " If None, no search will be performed.",
+        default=None,
+    )
+
+    editing_strategy: TTCStrategyRef | None = Field(
+        default=None,
+        description="The TTC editing strategy to use for orchestrating invocation of the functions. "
+        "If None, no editing will be performed.",
+    )
+
+    scoring_strategy: TTCStrategyRef | None = Field(
+        default=None,
+        description="The TTC scoring strategy to use for orchestrating invocation of the functions. "
+        "If None, no scoring will be performed.",
+    )
+
+    selection_strategy: TTCStrategyRef = Field(
+        description="The TTC selection strategy to use for orchestrating invocation of the functions.")
+
+
+@register_function(config_type=TTCToolOrchestrationFunctionConfig, framework_wrappers=[LLMFrameworkEnum.LANGCHAIN])
+async def register_ttc_tool_orchestration_function(
+    config: TTCToolOrchestrationFunctionConfig,
+    builder: Builder,
+):
+    """
+    Registers an TTC-based orchestration function that:
+     1. Instantiates all relevant strategies (search, editing, scoring, selection).
+     2. Accepts a ToolUselist, converts each item to an TTCItem, optionally runs search/editing.
+     3. Calls the correct augmented_fn per item using name=tool name.
+     4. If configured, runs scoring and selection on the result.
+     5. Returns a new ToolUselist with each output set.
+    """
+
+    # 1) Gather references to all augmented (wrapped) functions
+    function_map = {}
+    for fn_ref in config.augmented_fns:
+        # Retrieve the actual function from the builder
+        fn_obj = await builder.get_function(fn_ref)
+        function_map[fn_ref] = fn_obj
+
+    # 2) Instantiate search, editing, scoring, selection strategies (if any)
+    search = None
+    if config.search_strategy is not None:
+        search = await builder.get_ttc_strategy(
+            strategy_name=config.search_strategy,
+            pipeline_type=PipelineTypeEnum.TOOL_USE,
+            stage_type=StageTypeEnum.SEARCH,
+        )
+
+    editing = None
+    if config.editing_strategy is not None:
+        editing = await builder.get_ttc_strategy(
+            strategy_name=config.editing_strategy,
+            pipeline_type=PipelineTypeEnum.TOOL_USE,
+            stage_type=StageTypeEnum.EDITING,
+        )
+
+    scoring = None
+    if config.scoring_strategy is not None:
+        scoring = await builder.get_ttc_strategy(
+            strategy_name=config.scoring_strategy,
+            pipeline_type=PipelineTypeEnum.TOOL_USE,
+            stage_type=StageTypeEnum.SCORING,
+        )
+
+    selection = await builder.get_ttc_strategy(
+        strategy_name=config.selection_strategy,
+        pipeline_type=PipelineTypeEnum.TOOL_USE,
+        stage_type=StageTypeEnum.SELECTION,
+    )
+
+    fn_description = ("\n".join(f"- **{fn_ref}**: {function_map[fn_ref].description or 'No description provided.'}"
+                                for fn_ref in config.augmented_fns))
+
+    # 3) Create the inner function to handle single (non-streaming) calls.
+    async def single_inner(tool_list: ToolUselist) -> ToolUselist:
+        """
+        Orchestrates multiple tool usages, optionally using search/editing/scoring/selection steps.
+        """
+        # Convert each ToolUseInputSchema to TTCItem
+        ttc_items = []
+        for t in tool_list.tools:
+            item = TTCItem(
+                input=t.task_description,  # The user "task"
+                output=None,
+                name=t.tool_name,  # The "tool name"
+                metadata=t.motivation,  # The "justification"
+            )
+            ttc_items.append(item)
+
+        # Run search strategy if present
+        if search is not None:
+            ttc_items = await search.ainvoke(ttc_items)
+
+        logger.info("TTC orchestration function: %d items after search", len(ttc_items))
+
+        # Invoke the correct augmented function for each item concurrently
+        # Helper coroutine to invoke a tool function and capture result or error
+        async def _invoke_tool(item: TTCItem, fn):
+            try:
+                result = await fn.acall_invoke(item.output)
+                return item, result, None
+            except Exception as e:
+                logger.exception(f"Error invoking function '{item.name}': {e}")
+                return item, None, str(e)
+
+        tasks = []
+        for item in ttc_items:
+            if item.name not in function_map:
+                logger.error(f"Function '{item.name}' not found in function map.", exc_info=True)
+                item.output = f"Error: Function '{item.name}' not found in function map. Check your input"
+            else:
+                fn = function_map[item.name]
+                tasks.append(_invoke_tool(item, fn))
+
+        # Await all tasks and assign outputs
+        if tasks:
+            results = await asyncio.gather(*tasks)
+            for item, result, error in results:
+                if error:
+                    item.output = f"Error invoking function '{item.name}': {error}"
+                else:
+                    item.output = result
+
+        if editing:
+            ttc_items = await editing.ainvoke(ttc_items)
+
+        # Run scoring strategy if present
+        if scoring is not None:
+            ttc_items = await scoring.ainvoke(ttc_items)
+
+        # Run selection strategy
+        if selection is not None:
+            ttc_items = await selection.ainvoke(ttc_items)
+
+        logger.info("TTC orchestration function: %d items after selection", len(ttc_items))
+
+        # Convert final results from TTCItems back to a ToolUselist
+        final_list = ToolUselist(tools=[])
+        for item in ttc_items:
+            # Compose a new ToolUseInputSchema with final output
+            new_tool = ToolUseInputSchema(
+                tool_name=item.name,
+                task_description=str(item.input),
+                motivation=item.metadata if item.metadata else None,
+                output=str(item.output) if item.output is not None else None,
+            )
+            final_list.tools.append(new_tool)
+
+        return final_list
+
+    # 4) Return the function info (only a single_fn is needed; no streaming)
+    yield FunctionInfo.create(
+        single_fn=single_inner,
+        stream_fn=None,  # No streaming required
+        input_schema=ToolUselist,
+        single_output_schema=ToolUselist,
+        description=fn_description)

nat/experimental/test_time_compute/functions/ttc_tool_wrapper_function.py

@@ -0,0 +1,146 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+
+from pydantic import BaseModel
+from pydantic import Field
+
+from nat.builder.builder import Builder
+from nat.builder.framework_enum import LLMFrameworkEnum
+from nat.builder.function import Function
+from nat.builder.function_info import FunctionInfo
+from nat.cli.register_workflow import register_function
+from nat.data_models.component_ref import FunctionRef
+from nat.data_models.component_ref import LLMRef
+from nat.data_models.function import FunctionBaseConfig
+from nat.utils.string_utils import convert_to_str
+
+logger = logging.getLogger(__name__)
+
+
+class TTCToolWrapperFunctionConfig(FunctionBaseConfig, name="ttc_tool_wrapper"):
+    """
+    Configuration for the TTCToolWrapperFunction, which is used to wrap a function that will be executed
+    in the inference time scaling pipeline.
+
+    This function is responsible for turning an 'objective' or description for the tool into tool input.
+
+    NOTE: Only supports LLMs with structured output.
+    """
+
+    augmented_fn: FunctionRef = Field(description="The name of the function to reason on.")
+
+    input_llm: LLMRef = Field(description="The LLM that will generate input to the function.")
+    verbose: bool = Field(default=False, description="Whether to log detailed information.")
+
+    downstream_template: str = Field(
+        description="The template for the input LLM to generate structured input to the function.",
+        default=("You are highly sophisticated generalist AI assistant. Your objective is to act as a"
+                 " conduit between a user's task for a function and the function itself. You will be given a general "
+                 "description of the task, or pseudo input for a function. You will also be provided with description "
+                 "of the function, its input schema, and the output schema. Your task is to generate structured input "
+                 "to the function based on the description of the task and the function's input schema. If you do not "
+                 "have enough information to generate structured input, you should respond with 'NOT ENOUGH "
+                 "INFORMATION'. \n\n The description of the function is: {function_description}\n\n"
+                 "The input schema of the function is: {input_schema}\n\n"
+                 "The output schema of the function is: {output_schema}\n\n"
+                 "The description of the task is: {task_description}\n\n"
+                 "The structured input to the function is: "))
+
+    tool_description: str | None = Field(description="The description of the tool to be used for the function.",
+                                         default=None)
+
+
+@register_function(config_type=TTCToolWrapperFunctionConfig, framework_wrappers=[LLMFrameworkEnum.LANGCHAIN])
+async def register_ttc_tool_wrapper_function(
+    config: TTCToolWrapperFunctionConfig,
+    builder: Builder,
+):
+    """
+    Register the TTCToolWrapperFunction with the provided builder and configuration.
+    """
+
+    try:
+        from langchain_core.language_models import BaseChatModel
+        from langchain_core.prompts import PromptTemplate
+    except ImportError:
+        raise ImportError("langchain-core is not installed. Please install it to use SingleShotMultiPlanPlanner.\n"
+                          "This error can be resolved by installing nvidia-nat-langchain.")
+
+    augmented_function: Function = await builder.get_function(config.augmented_fn)
+    input_llm: BaseChatModel = await builder.get_llm(config.input_llm, wrapper_type=LLMFrameworkEnum.LANGCHAIN)
+
+    if not augmented_function.has_single_output:
+        raise ValueError("TTCToolWrapperFunction only supports functions with a single output.")
+
+    if not augmented_function.has_single_output:
+        raise ValueError("TTCToolWrapperFunction only supports functions with a single output.")
+
+    if augmented_function.description and augmented_function.description != "":
+        augmented_function_desc = augmented_function.description
+    else:
+        if not config.tool_description:
+            raise ValueError(f"Function {config.augmented_fn} does not have a description. Cannot augment "
+                             f"function without a description and without a tool description.")
+
+        augmented_function_desc = config.tool_description
+
+    fn_input_schema: type[BaseModel] = augmented_function.input_schema
+    fn_output_schema: type[BaseModel] | type[None] = augmented_function.single_output_schema
+
+    runnable_llm = input_llm.with_structured_output(schema=fn_input_schema)
+
+    template = PromptTemplate(
+        template=config.downstream_template,
+        input_variables=["function_description", "input_schema", "output_schema", "task_description"],
+        validate_template=True)
+
+    function_description = (f"\nDescription: {augmented_function_desc}\n" +
+                            "\n Input should be a thorough description with all relevant information on what "
+                            f"the tool should do.  The tool requires information about "
+                            f"{fn_input_schema.model_fields}")
+
+    async def single_inner(input_message: str) -> fn_output_schema:
+        """
+        Inner function to handle the streaming output of the TTCToolWrapperFunction.
+        It generates structured input for the augmented function based on the input message.
+        """
+
+        prompt = await template.ainvoke(
+            input={
+                "function_description": augmented_function_desc,
+                "input_schema": fn_input_schema,
+                "output_schema": fn_output_schema,
+                "task_description": input_message
+            })
+
+        prompt = prompt.to_string()
+
+        if config.verbose:
+            logger.info("TTCToolWrapperFunction: Generated prompt: %s", prompt)
+
+        llm_parsed = await runnable_llm.ainvoke(prompt)
+
+        if not llm_parsed:
+            logger.warning("TTCToolWrapperFunction: LLM parsing error")
+            return "Not enough information"
+
+        # Call the augmented function with the structured input
+        result = await augmented_function.acall_invoke(llm_parsed)
+
+        return result
+
+    yield FunctionInfo.from_fn(fn=single_inner, description=function_description, converters=[convert_to_str])

nat/experimental/test_time_compute/models/editor_config.py

@@ -0,0 +1,132 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import typing
+
+from pydantic import Field
+from pydantic import model_validator
+
+from nat.data_models.component_ref import LLMRef
+from nat.data_models.ttc_strategy import TTCStrategyBaseConfig
+
+
+class LLMAsAJudgeEditorConfig(TTCStrategyBaseConfig, name="llm_as_a_judge_editor"):
+    """
+    Configuration for the LLMAsAJudgeEditor.
+    """
+    num_feedback: int = Field(default=10,
+                              description="Number of feedback items to generate for each plan during editing. "
+                              "This can help in refining the plans based on feedback.")
+
+    # If strategy is provided, LLM must be
+    editing_llm: LLMRef | typing.Any | None = Field(
+        default=None,
+        description="The LLM to use for editing the plans. This can be a callable or an instance of an LLM client.")
+
+    # If strategy is LLM_AS_A_JUDGE, ensure that the feedback_llm is provided.
+    feedback_llm: LLMRef | typing.Any | None = Field(default=None,
+                                                     description="The LLM to use for generating feedback on the plans."
+                                                     " This can be a callable or an instance of an LLM client.")
+
+    editor_template: str = Field(default=(
+        "You are an expert at improving execution plans. You will be given a plan and feedback on that plan."
+        " Your task is to create an improved version of the plan that addresses the feedback "
+        "while maintaining its strengths.\n\n"
+        "Here is the context:\n\n"
+        "{context}\n\n"
+        "**Input:** \n{original_prompt}\n\n"
+        "**Original Plan:**\n{plan}\n\n"
+        "**Feedback on the Plan:**\n{feedback}\n\n"
+        "Please provide an improved version of the plan that addresses"
+        " the feedback points. Maintain the same structure and "
+        "step-by-step format, but enhance the content. Do not include explanations of your changes, just provide the "
+        "improved plan directly:\n\n"
+        "Begin the final improve plan with 'EDITED PLAN:'"),
+                                 description="The template to use for editing the planning items based on feedback.")
+
+    feedback_template: str = Field(
+        default=("You are an expert at evaluating execution plans. You will be given a plan and "
+                 "need to provide {num_feedback} "
+                 "specific points of feedback about its strengths and weaknesses.\n\n"
+                 "Your feedback should cover aspects like:\n"
+                 "- Comprehensiveness of the plan\n"
+                 "- Logical flow and sequencing\n"
+                 "- Appropriate use of available tools\n"
+                 "- Potential edge cases or failure points\n"
+                 "- Efficiency and optimization opportunities\n\n"
+                 "Here is the context and plan to evaluate:\n\n"
+                 "{context}\n\n"
+                 "**Objective:** \n{original_prompt}\n\n"
+                 "**Plan to Evaluate:**\n{plan}\n\n"
+                 "Please provide exactly {num_feedback} numbered points of feedback, including "
+                 "both strengths and areas for improvement. Begin the feedback with 'FEEDBACK:' and provide"
+                 "{num_feedback} specific feedback points."),
+        description="The template to use for generating feedback for each planning item.")
+
+    @model_validator(mode="before")
+    def validate_strategies(cls, values: dict[str, typing.Any]) -> dict[str, typing.Any]:
+
+        if values.get('editing_llm') is None:
+            raise ValueError('editing_llm must be provided when editing_strategy is set.')
+        # If editing strategy is LLM_AS_A_JUDGE, feedback_llm must also be provided
+        if (values.get('feedback_llm') is None):
+            raise ValueError('feedback_llm must be provided when editing_strategy is LLM_AS_A_JUDGE.')
+
+        return values
+
+
+class IterativePlanRefinementConfig(TTCStrategyBaseConfig, name="iterative_plan_refinement"):
+    """Configuration for an 'iterative plan refinement' strategy."""
+    editor_llm: LLMRef | typing.Any | None = Field(
+        default=None, description="The LLM to use for generating and refining the plan across multiple iterations.")
+    num_iterations: int = Field(default=3, description="How many refinement steps to perform.")
+    refinement_template: str = Field(
+        default=("You have the current plan:\n{current_plan}\n\n"
+                 "The plan was generated to achieve the following objective:\n{original_prompt}\n\n"
+                 "Using an agent system with the following description:\n{context}\n\n"
+                 "Refine or improve it to achieve the objective better."
+                 "Output the updated plan, beginning with:\nEDITED PLAN:\n"),
+        description="Prompt used in each iteration to refine the plan.")
+
+    @model_validator(mode="before")
+    def validate_iterative_strategies(cls, values: dict) -> dict:
+        if not values.get('editor_llm'):
+            raise ValueError('planning_llm must be provided for iterative plan refinement.')
+        if values.get('num_iterations', 0) < 1:
+            raise ValueError('num_iterations must be >= 1 for iterative plan refinement.')
+        return values
+
+
+class MotivationAwareSummarizationConfig(TTCStrategyBaseConfig, name="motivation_aware_editing"):
+    """
+    Configuration for the MotivationAwareSummarization strategy.
+    """
+    editor_llm: LLMRef | typing.Any | None = Field(
+        default=None,
+        description="The LLM to use for editing the plans. This can be a callable or an instance of an LLM client.")
+
+    editor_template: str = Field(
+        default=("You are an expert at summarizing key information from relevant documents based on an input task"
+                 "and motivation. Given a task and motivation, and documents, your task is to create a concise "
+                 "a summarized response to the task and motivation grounded in the documents .\n\n"
+                 "Here is the task:\n\n"
+                 "{task}\n\n"
+                 "Here is the motivation:\n\n"
+                 "{motivation}\n\n"
+                 "and here are the documents:\n\n"
+                 "{output}\n\n"
+                 "Please respond with a concise summary that addresses the task and motivation, in at most one"
+                 "or two sentences. Do not include any other output except the summary. "),
+        description="The template to use for summarizing documents.")

nat/experimental/test_time_compute/models/scoring_config.py

@@ -0,0 +1,112 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import typing
+
+from pydantic import Field
+from pydantic import model_validator
+
+from nat.data_models.component_ref import LLMRef
+from nat.data_models.ttc_strategy import TTCStrategyBaseConfig
+
+
+class LLMBasedPlanScoringConfig(TTCStrategyBaseConfig, name="llm_based_plan_scoring"):
+    """
+    Configuration for LLMBasedScoring.
+    """
+    scoring_llm: LLMRef | typing.Any | None = Field(
+        default=None,
+        description="The LLM to use for scoring the plans. This can be a callable or an instance of an LLM client.")
+
+    scoring_template: str = Field(
+        default=("You are an expert reasoning model tasked with scoring the following execution plan based on its"
+                 "quality and relevance to the provided input to an agent system.\n\n"
+                 "The agent system's role is:\n{context}\n\n"
+                 "It has been tasked with achieving the following goal: \n{original_prompt}\n\n"
+                 "The following plan has been generated to achieve this goal:\n\n{plan}\n\n"
+                 "Score the plan on a scale from 1 to 10, where 10 is the best. "
+                 "Return the final score as a floating point number preceded by `FINAL SCORE:` without any "
+                 "other text before or after it\n"),
+        description="The template to use for scoring the plans.")
+
+    @model_validator(mode="before")
+    def validate_strategies(cls, values: dict[str, typing.Any]) -> dict[str, typing.Any]:
+        """
+        Ensure that the scoring_llm is provided when using LLMBasedScoring.
+        """
+        if values.get('scoring_llm') is None:
+            raise ValueError('scoring_llm must be provided when scorer_type is set to LLM_BASED_SCORING.')
+
+        return values
+
+
+class LLMBasedAgentScoringConfig(TTCStrategyBaseConfig, name="llm_based_agent_scoring"):
+    """
+    Configuration for LLMBasedScoring.
+    """
+    scoring_llm: LLMRef | typing.Any | None = Field(
+        default=None,
+        description="The LLM to use for scoring the plans. This can be a callable or an instance of an LLM client.")
+
+    scoring_template: str = Field(
+        description="Prompt template to use for scoring the function output",
+        default=("You are an expert reasoning model tasked with scoring the following "
+                 "result of an agent system based on its input and objective. Judge"
+                 " the quality and relevance of the answer to score it.\n\n"
+                 "The agent system's objective is:\n{objective}\n\n"
+                 "It has been tasked with achieving the following goal: \n{input}\n\n"
+                 "The following output has been generated by the agent:\n\n{output}\n\n"
+                 "Score the result on a scale from 1 to 10, where 10 is the best. "
+                 "Return the final score as a floating point number preceded by `FINAL SCORE:` without any "
+                 "other text before or after it\n"),
+    )
+
+    @model_validator(mode="before")
+    def validate_strategies(cls, values: dict[str, typing.Any]) -> dict[str, typing.Any]:
+        """
+        Ensure that the scoring_llm is provided when using LLMBasedScoring.
+        """
+        if values.get('scoring_llm') is None:
+            raise ValueError('scoring_llm must be provided when scorer_type is set to LLM_BASED_SCORING.')
+
+        return values
+
+
+class MotivationAwareScoringConfig(TTCStrategyBaseConfig, name="motivation_aware_scoring"):
+    """
+    Configuration for a scoring strategy that considers both the original input (task)
+    and the motivation (from metadata) along with the current output.
+    """
+
+    scoring_llm: LLMRef | None = Field(
+        default=None, description="The LLM used to evaluate how well the output addresses the task plus motivation.")
+
+    scoring_template: str = Field(
+        default=("You are an expert at assessing the quality of an output in relation to its task and motivation.\n"
+                 "Task: {task}\n"
+                 "Motivation: {motivation}\n"
+                 "Output: {output}\n"
+                 "On a scale from 1 to 10 (10 being the best), how well does this output fulfill "
+                 "the original task in the context "
+                 "of the provided motivation? Note that the task might answer one part of a bigger question "
+                 "which should count as a satisfactory response and should not receive a lower score.\n"
+                 "Return the final score as a floating point number preceded by 'FINAL SCORE:'."),
+        description="The prompt template used to evaluate and score the output.")
+
+    @model_validator(mode="before")
+    def validate_scoring_llm(cls, values):
+        if values.get('scoring_llm') is None:
+            raise ValueError("A scoring_llm must be provided for motivation_aware_scoring.")
+        return values