aiqtoolkit 1.2.0a20250706__py3-none-any.whl → 1.2.0a20250730__py3-none-any.whl

aiq/experimental/inference_time_scaling/functions/its_tool_wrapper_function.py

@@ -0,0 +1,146 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+
+from pydantic import BaseModel
+from pydantic import Field
+
+from aiq.builder.builder import Builder
+from aiq.builder.framework_enum import LLMFrameworkEnum
+from aiq.builder.function import Function
+from aiq.builder.function_info import FunctionInfo
+from aiq.cli.register_workflow import register_function
+from aiq.data_models.component_ref import FunctionRef
+from aiq.data_models.component_ref import LLMRef
+from aiq.data_models.function import FunctionBaseConfig
+from aiq.utils.string_utils import convert_to_str
+
+logger = logging.getLogger(__name__)
+
+
+class ITSToolWrapperFunctionConfig(FunctionBaseConfig, name="its_tool_wrapper"):
+    """
+    Configuration for the ITSToolWrapperFunction, which is used to wrap a function that will be executed
+    in the inference time scaling pipeline.
+
+    This function is responsible for turning an 'objective' or description for the tool into tool input.
+
+    NOTE: Only supports LLMs with structured output.
+    """
+
+    augmented_fn: FunctionRef = Field(description="The name of the function to reason on.")
+
+    input_llm: LLMRef = Field(description="The LLM that will generate input to the function.")
+    verbose: bool = Field(default=False, description="Whether to log detailed information.")
+
+    downstream_template: str = Field(
+        description="The template for the input LLM to generate structured input to the function.",
+        default=("You are highly sophisticated generalist AI assistant. Your objective is to act as a"
+                 " conduit between a user's task for a function and the function itself. You will be given a general "
+                 "description of the task, or pseudo input for a function. You will also be provided with description "
+                 "of the function, its input schema, and the output schema. Your task is to generate structured input "
+                 "to the function based on the description of the task and the function's input schema. If you do not "
+                 "have enough information to generate structured input, you should respond with 'NOT ENOUGH "
+                 "INFORMATION'. \n\n The description of the function is: {function_description}\n\n"
+                 "The input schema of the function is: {input_schema}\n\n"
+                 "The output schema of the function is: {output_schema}\n\n"
+                 "The description of the task is: {task_description}\n\n"
+                 "The structured input to the function is: "))
+
+    tool_description: str | None = Field(description="The description of the tool to be used for the function.",
+                                         default=None)
+
+
+@register_function(config_type=ITSToolWrapperFunctionConfig, framework_wrappers=[LLMFrameworkEnum.LANGCHAIN])
+async def register_its_tool_wrapper_function(
+    config: ITSToolWrapperFunctionConfig,
+    builder: Builder,
+):
+    """
+    Register the ITSToolWrapperFunction with the provided builder and configuration.
+    """
+
+    try:
+        from langchain_core.language_models import BaseChatModel
+        from langchain_core.prompts import PromptTemplate
+    except ImportError:
+        raise ImportError("langchain-core is not installed. Please install it to use SingleShotMultiPlanPlanner.\n"
+                          "This error can be resolved by installing aiqtoolkit-langchain.")
+
+    augmented_function: Function = builder.get_function(config.augmented_fn)
+    input_llm: BaseChatModel = await builder.get_llm(config.input_llm, wrapper_type=LLMFrameworkEnum.LANGCHAIN)
+
+    if not augmented_function.has_single_output:
+        raise ValueError("ITSToolWrapperFunction only supports functions with a single output.")
+
+    if not augmented_function.has_single_output:
+        raise ValueError("ITSToolWrapperFunction only supports functions with a single output.")
+
+    if augmented_function.description and augmented_function.description != "":
+        augmented_function_desc = augmented_function.description
+    else:
+        if not config.tool_description:
+            raise ValueError(f"Function {config.augmented_fn} does not have a description. Cannot augment "
+                             f"function without a description and without a tool description.")
+
+        augmented_function_desc = config.tool_description
+
+    fn_input_schema: BaseModel = augmented_function.input_schema
+    fn_output_schema: BaseModel = augmented_function.single_output_schema
+
+    runnable_llm = input_llm.with_structured_output(schema=fn_input_schema)
+
+    template = PromptTemplate(
+        template=config.downstream_template,
+        input_variables=["function_description", "input_schema", "output_schema", "task_description"],
+        validate_template=True)
+
+    function_description = (f"\nDescription: {augmented_function_desc}\n" +
+                            "\n Input should be a thorough description with all relevant information on what "
+                            f"the tool should do.  The tool requires information about "
+                            f"{fn_input_schema.model_fields}")
+
+    async def single_inner(input_message: str) -> fn_output_schema:
+        """
+        Inner function to handle the streaming output of the ITSToolWrapperFunction.
+        It generates structured input for the augmented function based on the input message.
+        """
+
+        prompt = await template.ainvoke(
+            input={
+                "function_description": augmented_function_desc,
+                "input_schema": fn_input_schema,
+                "output_schema": fn_output_schema,
+                "task_description": input_message
+            })
+
+        prompt = prompt.to_string()
+
+        if config.verbose:
+            logger.info("ITSToolWrapperFunction: Generated prompt: %s", prompt)
+
+        llm_parsed = await runnable_llm.ainvoke(prompt)
+
+        if not llm_parsed:
+            logger.warning("ITSToolWrapperFunction: LLM parsing error")
+            return "Not enough information"
+
+        # Call the augmented function with the structured input
+        result = await augmented_function.acall_invoke(llm_parsed)
+
+        return result
+
+    yield FunctionInfo.from_fn(fn=single_inner, description=function_description, converters=[convert_to_str])

aiq/experimental/inference_time_scaling/functions/plan_select_execute_function.py

@@ -0,0 +1,224 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+from collections.abc import AsyncGenerator
+
+from pydantic import Field
+
+from aiq.builder.builder import Builder
+from aiq.builder.framework_enum import LLMFrameworkEnum
+from aiq.builder.function_info import FunctionInfo
+from aiq.cli.register_workflow import register_function
+from aiq.data_models.api_server import AIQChatRequest
+from aiq.data_models.component_ref import FunctionRef
+from aiq.data_models.component_ref import ITSStrategyRef
+from aiq.data_models.function import FunctionBaseConfig
+from aiq.experimental.inference_time_scaling.models.its_item import ITSItem
+from aiq.experimental.inference_time_scaling.models.stage_enums import PipelineTypeEnum
+from aiq.experimental.inference_time_scaling.models.stage_enums import StageTypeEnum
+
+logger = logging.getLogger(__name__)
+
+
+class PlanSelectExecuteFunctionConfig(FunctionBaseConfig, name="plan_select_execute_function"):
+    """
+    Defines an aiqtoolkit function that performs reasoning on the input data.
+    Output is passed to the next function in the workflow.
+
+    Designed to be used with an InterceptingFunction.
+    """
+
+    augmented_fn: FunctionRef = Field(description="The name of the function to reason on.")
+
+    planner: ITSStrategyRef = Field(description="The configuration for the planner.")
+    editor: ITSStrategyRef | None = Field(description="The configuration for the editor.", default=None)
+    scorer: ITSStrategyRef | None = Field(description="The configuration for the scorer.", default=None)
+    selector: ITSStrategyRef = Field(description="The configuration for the selector.")
+
+    verbose: bool = Field(default=False, description="Whether to log detailed information.")
+    agent_context_prompt_template: str = Field(
+        description="The template for the agent context prompt. This prompt is used to provide context about the agent",
+        default=("\nThe agent system has the following description:\n"
+                 "{description}\n"
+                 "And has access to the following tools with functionality:\n"
+                 "{tools}\n\n"))
+
+    downstream_template: str = Field(
+        description=("The template for the downstream prompt. This prompt is used to provide the reasoning output to"
+                     " the executing agent"),
+        default=("Answer the following question based on message history: {input_text}"
+                 "\n\nHere is a plan for execution that you could use to guide you if you wanted to:"
+                 "\n\n{reasoning_output}"
+                 "\n\nNOTE: Remember to follow your guidance on how to format output, etc."
+                 "\n\n You must respond with the answer to the original question directly to the user."))
+
+
+@register_function(config_type=PlanSelectExecuteFunctionConfig, framework_wrappers=[LLMFrameworkEnum.LANGCHAIN])
+async def plan_select_execute_function(config: PlanSelectExecuteFunctionConfig, builder: Builder):
+    """
+    Build a ExecutionPlanningFunction from the provided config.
+
+    Args:
+        config (ExecutionPlanningFunctionConfig): The config for the ExecutionPlanningFunction.
+        builder (Builder): The Builder instance to use for building the function.
+
+    Returns:
+        ExecutionPlanningFunction: The built ExecutionPlanningFunction.
+    """
+
+    try:
+        from langchain_core.prompts import PromptTemplate
+    except ImportError:
+        raise ImportError("langchain-core is not installed. Please install it to use SingleShotMultiPlanPlanner.\n"
+                          "This error can be resolved by installing aiqtoolkit-langchain.")
+
+    # Get the augmented function's description
+    augmented_function = builder.get_function(config.augmented_fn)
+
+    # For now, we rely on runtime checking for type conversion
+
+    if augmented_function.description and augmented_function.description != "":
+        augmented_function_desc = augmented_function.description
+    else:
+        raise ValueError(f"Function {config.augmented_fn} does not have a description. Cannot augment "
+                         f"function without a description.")
+
+    # Get the function dependencies of the augmented function
+    function_used_tools = builder.get_function_dependencies(config.augmented_fn).functions
+    tool_list = "Tool: Description\n"
+
+    for tool in function_used_tools:
+        tool_impl = builder.get_function(tool)
+        tool_list += f"- {tool}: {tool_impl.description if hasattr(tool_impl, 'description') else ''}\n"
+
+    # Draft the reasoning prompt for the augmented function
+    template = PromptTemplate(template=config.agent_context_prompt_template,
+                              input_variables=["description", "tools"],
+                              validate_template=True)
+
+    downstream_template = PromptTemplate(template=config.downstream_template,
+                                         input_variables=["input_text", "reasoning_output"],
+                                         validate_template=True)
+
+    planner = await builder.get_its_strategy(strategy_name=config.planner,
+                                             pipeline_type=PipelineTypeEnum.PLANNING,
+                                             stage_type=StageTypeEnum.SEARCH)
+
+    selector = await builder.get_its_strategy(strategy_name=config.selector,
+                                              pipeline_type=PipelineTypeEnum.PLANNING,
+                                              stage_type=StageTypeEnum.SELECTION)
+
+    if config.editor:
+        editor = await builder.get_its_strategy(strategy_name=config.editor,
+                                                pipeline_type=PipelineTypeEnum.PLANNING,
+                                                stage_type=StageTypeEnum.EDITING)
+    else:
+        editor = None
+
+    if config.scorer:
+        scorer = await builder.get_its_strategy(strategy_name=config.scorer,
+                                                pipeline_type=PipelineTypeEnum.PLANNING,
+                                                stage_type=StageTypeEnum.SCORING)
+    else:
+        scorer = None
+
+    async def planning_pipeline(prompt, context):
+
+        plans = await planner.ainvoke([ITSItem()], prompt, context)
+
+        if editor:
+            plans = await editor.ainvoke(plans, prompt, context)
+        if scorer:
+            plans = await scorer.ainvoke(plans, prompt, context)
+
+        selected_plan = (await selector.ainvoke(plans, prompt, context))[0]
+
+        return selected_plan
+
+    streaming_inner_fn = None
+    single_inner_fn = None
+
+    if augmented_function.has_streaming_output:
+
+        async def streaming_inner(
+                input_message: AIQChatRequest) -> AsyncGenerator[augmented_function.streaming_output_type]:
+            """
+            Perform reasoning on the input text.
+
+            Args:
+                input_message (AIQChatRequest): The input text to reason on.
+            """
+
+            input_text = "".join([str(message.model_dump()) + "\n" for message in input_message.messages])
+
+            context_prompt = await template.ainvoke(input={"description": augmented_function_desc, "tools": tool_list})
+
+            context_prompt = context_prompt.to_string()
+
+            # Run the ITS pipeline
+            planning_item: ITSItem = await planning_pipeline(prompt=input_text, context=context_prompt)
+
+            output = await downstream_template.ainvoke(input={
+                "input_text": input_text, "reasoning_output": planning_item.plan
+            })
+
+            output = output.to_string()
+
+            if config.verbose:
+                logger.info("Reasoning plan and input to agent: \n\n%s", output)
+
+            async for chunk in augmented_function.acall_stream(output):
+                yield chunk
+
+        streaming_inner_fn = streaming_inner
+
+    if augmented_function.has_single_output:
+
+        async def single_inner(input_message: AIQChatRequest) -> augmented_function.single_output_type:
+            """
+            Perform reasoning on the input text.
+
+            Args:
+                input_message (AIQChatRequest): The input text to reason on.
+            """
+
+            input_text = "".join([str(message.model_dump()) + "\n" for message in input_message.messages])
+
+            context_prompt = await template.ainvoke(input={"description": augmented_function_desc, "tools": tool_list})
+
+            context_prompt = context_prompt.to_string()
+
+            # Run the ITS pipeline
+            planning_item: ITSItem = await planning_pipeline(prompt=input_text, context=context_prompt)
+
+            output = await downstream_template.ainvoke(input={
+                "input_text": input_text, "reasoning_output": planning_item.plan
+            })
+
+            output = output.to_string()
+
+            if config.verbose:
+                logger.info("Reasoning plan and input to agent: \n\n%s", output)
+
+            return await augmented_function.acall_invoke(output)
+
+        single_inner_fn = single_inner
+
+    yield FunctionInfo.create(
+        single_fn=single_inner_fn,
+        stream_fn=streaming_inner_fn,
+        description=("Function that runs an ITS execution planner on input and sends plan downstream"),
+        converters=augmented_function.converter_list)

aiq/experimental/inference_time_scaling/models/editor_config.py

@@ -0,0 +1,132 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import typing
+
+from pydantic import Field
+from pydantic import model_validator
+
+from aiq.data_models.component_ref import LLMRef
+from aiq.data_models.its_strategy import ITSStrategyBaseConfig
+
+
+class LLMAsAJudgeEditorConfig(ITSStrategyBaseConfig, name="llm_as_a_judge_editor"):
+    """
+    Configuration for the LLMAsAJudgeEditor.
+    """
+    num_feedback: int = Field(default=10,
+                              description="Number of feedback items to generate for each plan during editing. "
+                              "This can help in refining the plans based on feedback.")
+
+    # If strategy is provided, LLM must be
+    editing_llm: LLMRef | typing.Any | None = Field(
+        default=None,
+        description="The LLM to use for editing the plans. This can be a callable or an instance of an LLM client.")
+
+    # If strategy is LLM_AS_A_JUDGE, ensure that the feedback_llm is provided.
+    feedback_llm: LLMRef | typing.Any | None = Field(default=None,
+                                                     description="The LLM to use for generating feedback on the plans."
+                                                     " This can be a callable or an instance of an LLM client.")
+
+    editor_template: str = Field(default=(
+        "You are an expert at improving execution plans. You will be given a plan and feedback on that plan."
+        " Your task is to create an improved version of the plan that addresses the feedback "
+        "while maintaining its strengths.\n\n"
+        "Here is the context:\n\n"
+        "{context}\n\n"
+        "**Input:** \n{original_prompt}\n\n"
+        "**Original Plan:**\n{plan}\n\n"
+        "**Feedback on the Plan:**\n{feedback}\n\n"
+        "Please provide an improved version of the plan that addresses"
+        " the feedback points. Maintain the same structure and "
+        "step-by-step format, but enhance the content. Do not include explanations of your changes, just provide the "
+        "improved plan directly:\n\n"
+        "Begin the final improve plan with 'EDITED PLAN:'"),
+                                 description="The template to use for editing the planning items based on feedback.")
+
+    feedback_template: str = Field(
+        default=("You are an expert at evaluating execution plans. You will be given a plan and "
+                 "need to provide {num_feedback} "
+                 "specific points of feedback about its strengths and weaknesses.\n\n"
+                 "Your feedback should cover aspects like:\n"
+                 "- Comprehensiveness of the plan\n"
+                 "- Logical flow and sequencing\n"
+                 "- Appropriate use of available tools\n"
+                 "- Potential edge cases or failure points\n"
+                 "- Efficiency and optimization opportunities\n\n"
+                 "Here is the context and plan to evaluate:\n\n"
+                 "{context}\n\n"
+                 "**Objective:** \n{original_prompt}\n\n"
+                 "**Plan to Evaluate:**\n{plan}\n\n"
+                 "Please provide exactly {num_feedback} numbered points of feedback, including "
+                 "both strengths and areas for improvement. Begin the feedback with 'FEEDBACK:' and provide"
+                 "{num_feedback} specific feedback points."),
+        description="The template to use for generating feedback for each planning item.")
+
+    @model_validator(mode="before")
+    def validate_strategies(cls, values: dict[str, typing.Any]) -> dict[str, typing.Any]:
+
+        if values.get('editing_llm') is None:
+            raise ValueError('editing_llm must be provided when editing_strategy is set.')
+        # If editing strategy is LLM_AS_A_JUDGE, feedback_llm must also be provided
+        if (values.get('feedback_llm') is None):
+            raise ValueError('feedback_llm must be provided when editing_strategy is LLM_AS_A_JUDGE.')
+
+        return values
+
+
+class IterativePlanRefinementConfig(ITSStrategyBaseConfig, name="iterative_plan_refinement"):
+    """Configuration for an 'iterative plan refinement' strategy."""
+    editor_llm: LLMRef | typing.Any | None = Field(
+        default=None, description="The LLM to use for generating and refining the plan across multiple iterations.")
+    num_iterations: int = Field(default=3, description="How many refinement steps to perform.")
+    refinement_template: str = Field(
+        default=("You have the current plan:\n{current_plan}\n\n"
+                 "The plan was generated to achieve the following objective:\n{original_prompt}\n\n"
+                 "Using an agent system with the following description:\n{context}\n\n"
+                 "Refine or improve it to achieve the objective better."
+                 "Output the updated plan, beginning with:\nEDITED PLAN:\n"),
+        description="Prompt used in each iteration to refine the plan.")
+
+    @model_validator(mode="before")
+    def validate_iterative_strategies(cls, values: dict) -> dict:
+        if not values.get('editor_llm'):
+            raise ValueError('planning_llm must be provided for iterative plan refinement.')
+        if values.get('num_iterations', 0) < 1:
+            raise ValueError('num_iterations must be >= 1 for iterative plan refinement.')
+        return values
+
+
+class MotivationAwareSummarizationConfig(ITSStrategyBaseConfig, name="motivation_aware_editing"):
+    """
+    Configuration for the MotivationAwareSummarization strategy.
+    """
+    editor_llm: LLMRef | typing.Any | None = Field(
+        default=None,
+        description="The LLM to use for editing the plans. This can be a callable or an instance of an LLM client.")
+
+    editor_template: str = Field(
+        default=("You are an expert at summarizing key information from relevant documents based on an input task"
+                 "and motivation. Given a task and motivation, and documents, your task is to create a concise "
+                 "a summarized response to the task and motivation grounded in the documents .\n\n"
+                 "Here is the task:\n\n"
+                 "{task}\n\n"
+                 "Here is the motivation:\n\n"
+                 "{motivation}\n\n"
+                 "and here are the documents:\n\n"
+                 "{output}\n\n"
+                 "Please respond with a concise summary that addresses the task and motivation, in at most one"
+                 "or two sentences. Do not include any other output except the summary. "),
+        description="The template to use for summarizing documents.")

aiq/experimental/inference_time_scaling/models/its_item.py

@@ -0,0 +1,48 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import typing
+
+from pydantic import BaseModel
+from pydantic import ConfigDict
+from pydantic import Field
+
+
+class ITSItem(BaseModel):
+    """
+    Represents an item in the Inference Time Scaling (ITS) functions and pipelines
+    """
+    model_config = ConfigDict(extra="allow")
+
+    input: typing.Any | None = Field(default=None,
+                                     description="Input to the function or pipeline. "
+                                     "This can be a structured tool call, or other info.")
+    output: typing.Any | None = Field(default=None,
+                                      description="Output from the function or pipeline. "
+                                      "This can be a structured tool call, or other info.")
+    plan: typing.Any | None = Field(default=None, description="Search plan for downstream agent(s).")
+    feedback: str | None = Field(default=None,
+                                 description="Feedback "
+                                 "provided by feedback steps to improve the plan.")
+    score: float | None = Field(default=None,
+                                description="Score of the plan based on feedback or other evaluation criteria. "
+                                "This can be used to rank plans.")
+    metadata: typing.Any | None = Field(default=None,
+                                        description="Additional information. This can be"
+                                        " a structured tool call, or other info not "
+                                        "in the plan.")
+    name: str | None = Field(default=None,
+                             description="Name of the item or function"
+                             ", used for identification in pipelines.")

aiq/experimental/inference_time_scaling/models/scoring_config.py

@@ -0,0 +1,112 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import typing
+
+from pydantic import Field
+from pydantic import model_validator
+
+from aiq.data_models.component_ref import LLMRef
+from aiq.data_models.its_strategy import ITSStrategyBaseConfig
+
+
+class LLMBasedPlanScoringConfig(ITSStrategyBaseConfig, name="llm_based_plan_scoring"):
+    """
+    Configuration for LLMBasedScoring.
+    """
+    scoring_llm: LLMRef | typing.Any | None = Field(
+        default=None,
+        description="The LLM to use for scoring the plans. This can be a callable or an instance of an LLM client.")
+
+    scoring_template: str = Field(
+        default=("You are an expert reasoning model tasked with scoring the following execution plan based on its"
+                 "quality and relevance to the provided input to an agent system.\n\n"
+                 "The agent system's role is:\n{context}\n\n"
+                 "It has been tasked with achieving the following goal: \n{original_prompt}\n\n"
+                 "The following plan has been generated to achieve this goal:\n\n{plan}\n\n"
+                 "Score the plan on a scale from 1 to 10, where 10 is the best. "
+                 "Return the final score as a floating point number preceded by `FINAL SCORE:` without any "
+                 "other text before or after it\n"),
+        description="The template to use for scoring the plans.")
+
+    @model_validator(mode="before")
+    def validate_strategies(cls, values: dict[str, typing.Any]) -> dict[str, typing.Any]:
+        """
+        Ensure that the scoring_llm is provided when using LLMBasedScoring.
+        """
+        if values.get('scoring_llm') is None:
+            raise ValueError('scoring_llm must be provided when scorer_type is set to LLM_BASED_SCORING.')
+
+        return values
+
+
+class LLMBasedAgentScoringConfig(ITSStrategyBaseConfig, name="llm_based_agent_scoring"):
+    """
+    Configuration for LLMBasedScoring.
+    """
+    scoring_llm: LLMRef | typing.Any | None = Field(
+        default=None,
+        description="The LLM to use for scoring the plans. This can be a callable or an instance of an LLM client.")
+
+    scoring_template: str = Field(
+        description="Prompt template to use for scoring the function output",
+        default=("You are an expert reasoning model tasked with scoring the following "
+                 "result of an agent system based on its input and objective. Judge"
+                 " the quality and relevance of the answer to score it.\n\n"
+                 "The agent system's objective is:\n{objective}\n\n"
+                 "It has been tasked with achieving the following goal: \n{input}\n\n"
+                 "The following output has been generated by the agent:\n\n{output}\n\n"
+                 "Score the result on a scale from 1 to 10, where 10 is the best. "
+                 "Return the final score as a floating point number preceded by `FINAL SCORE:` without any "
+                 "other text before or after it\n"),
+    )
+
+    @model_validator(mode="before")
+    def validate_strategies(cls, values: dict[str, typing.Any]) -> dict[str, typing.Any]:
+        """
+        Ensure that the scoring_llm is provided when using LLMBasedScoring.
+        """
+        if values.get('scoring_llm') is None:
+            raise ValueError('scoring_llm must be provided when scorer_type is set to LLM_BASED_SCORING.')
+
+        return values
+
+
+class MotivationAwareScoringConfig(ITSStrategyBaseConfig, name="motivation_aware_scoring"):
+    """
+    Configuration for a scoring strategy that considers both the original input (task)
+    and the motivation (from metadata) along with the current output.
+    """
+
+    scoring_llm: LLMRef | None = Field(
+        default=None, description="The LLM used to evaluate how well the output addresses the task plus motivation.")
+
+    scoring_template: str = Field(
+        default=("You are an expert at assessing the quality of an output in relation to its task and motivation.\n"
+                 "Task: {task}\n"
+                 "Motivation: {motivation}\n"
+                 "Output: {output}\n"
+                 "On a scale from 1 to 10 (10 being the best), how well does this output fulfill "
+                 "the original task in the context "
+                 "of the provided motivation? Note that the task might answer one part of a bigger question "
+                 "which should count as a satisfactory response and should not receive a lower score.\n"
+                 "Return the final score as a floating point number preceded by 'FINAL SCORE:'."),
+        description="The prompt template used to evaluate and score the output.")
+
+    @model_validator(mode="before")
+    def validate_scoring_llm(cls, values):
+        if values.get('scoring_llm') is None:
+            raise ValueError("A scoring_llm must be provided for motivation_aware_scoring.")
+        return values