aiqtoolkit 1.1.0a20250429__py3-none-any.whl

aiq/eval/swe_bench_evaluator/evaluate.py

@@ -0,0 +1,215 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import logging
+import os
+import shutil
+from pathlib import Path
+
+from aiq.data_models.swe_bench_model import SWEBenchInput
+from aiq.data_models.swe_bench_model import SWEBenchOutput
+from aiq.eval.evaluator.evaluator_model import EvalInput
+from aiq.eval.evaluator.evaluator_model import EvalOutput
+
+try:
+    import swebench.harness.run_evaluation as swebench_eval
+    from swebench.harness.constants import MAP_REPO_VERSION_TO_SPECS
+except ImportError as exc:
+    raise ImportError("Please install swebench to use this evaluator") from exc
+
+logger = logging.getLogger(__name__)
+
+
+class SweBenchEvaluator:
+
+    def __init__(self, run_id: str, max_workers: int, output_dir: Path):
+
+        self.run_id = run_id
+        self.max_workers = max_workers
+        self.output_dir = output_dir
+
+        # metadata
+        self._unsupported_repos = []
+        self._swe_bench_inputs = []
+        self._swe_bench_outputs = []
+        self._model_name_or_path = "no_llm"
+
+    def get_model_name_from_output(self, workflow_output: list[dict]) -> str | None:
+        """Fetch the `model_name_or_path` from the first entry in the list."""
+        return workflow_output[0].get("model_name_or_path") if workflow_output else None
+
+    @staticmethod
+    def empty_report_dir(report_dir: Path):
+        """Remove the current contents of the report directory."""
+        os.makedirs(report_dir, exist_ok=True)
+
+        # Iterate through all files in the directory and remove them
+        for item in report_dir.iterdir():
+            if item.is_file():  # Remove files only
+                item.unlink()
+            elif item.is_dir():  # Remove subdirectories and their contents
+                shutil.rmtree(item)
+
+    @staticmethod
+    def move_report_and_logs(swe_bench_report_file: str, logs_dir: str, report_dir: Path):
+        """ Temorary function to move the report and logs to the output directory"""
+        try:
+            shutil.move(swe_bench_report_file, report_dir)
+        except Exception as e:
+            logger.exception("Error moving report file: %s", e, exc_info=True)
+
+        try:
+            dest_logs_dir = os.path.join(report_dir, 'logs')
+            shutil.move(logs_dir, dest_logs_dir)
+        except Exception as e:
+            logger.exception("Error moving logs directory: %s", e, exc_info=True)
+
+    def is_repo_supported(self, repo: str, version: str) -> bool:
+        """Check if the repo is supported by swebench"""
+
+        try:
+            _ = MAP_REPO_VERSION_TO_SPECS[repo][str(version)]
+        except KeyError:
+            self._unsupported_repos.append({repo, version})
+            return False
+        return True
+
+    def process_eval_input(self, eval_input: EvalInput) -> tuple[Path, Path]:
+        """Converts EvalInput into lists of SWEBenchInput and SWEBenchOutput models and applies filtering."""
+        # Convert input_obj and output_obj JSON strings to SWEBenchInput and SWEBenchOutput models
+        swebench_inputs = []
+        swebench_outputs = []
+
+        for item in eval_input.eval_input_items:
+            try:
+                swebench_input = SWEBenchInput.model_validate_json(item.input_obj)  # Convert input JSON to model
+                swebench_input.version = str(swebench_input.version)  # Convert version to string
+                swebench_inputs.append(swebench_input)
+
+                if item.output_obj:  # Convert output JSON to model if available
+                    swebench_output = SWEBenchOutput.model_validate_json(item.output_obj)
+                    swebench_outputs.append(swebench_output)
+                    # this is bit of a hack to match the swe_bench harness
+                    self._model_name_or_path = swebench_output.model_name_or_path
+
+            except Exception as e:
+                logger.exception("Failed to parse EvalInputItem %s: %s", item.id, e, exc_info=True)
+
+        # Filter out repos/version not supported by SWEBench
+        supported_inputs = [
+            swebench for swebench in swebench_inputs if self.is_repo_supported(swebench.repo, swebench.version)
+        ]
+
+        if not supported_inputs:
+            logger.error("No supported instances; nothing to evaluate")
+            return None, None
+
+        if len(supported_inputs) < len(swebench_inputs):
+            logger.warning("The following repos are not supported by SWEBench and were skipped:\n %s",
+                           {s.repo
+                            for s in swebench_inputs if s not in supported_inputs})
+
+        # Write SWEBenchInput to file
+        workflow_input_file = self.output_dir / "aiq_workflow_input.json"
+        workflow_input_file.parent.mkdir(parents=True, exist_ok=True)
+        Path(workflow_input_file).write_text(json.dumps([swebench.model_dump() for swebench in supported_inputs],
+                                                        indent=2),
+                                             encoding="utf-8")
+        logger.info("Workflow input written to %s", workflow_input_file)
+
+        # Filter SWEBenchOutput to include only instance_ids present in SWEBenchInput
+        valid_instance_ids = {swebench.instance_id for swebench in supported_inputs}
+        filtered_outputs = [output for output in swebench_outputs if output.instance_id in valid_instance_ids]
+
+        if not filtered_outputs:
+            logger.error("No supported outputs; nothing to evaluate")
+            return None, None
+
+        # Write SWEBenchOutput to file
+        workflow_output_file = self.output_dir / "aiq_workflow_output.json"
+        Path(workflow_output_file).write_text(json.dumps([output.model_dump() for output in filtered_outputs],
+                                                         indent=2),
+                                              encoding="utf-8")
+        logger.info("Workflow output written to %s", workflow_output_file)
+
+        self._swe_bench_inputs = supported_inputs
+        self._swe_bench_outputs = filtered_outputs
+        return workflow_input_file, workflow_output_file
+
+    def build_eval_output(self):
+        """Builds the EvalOutput object from the SWEBenchOutput models and the average score."""
+        # WIP: Build a score based on eval run logs
+        for swebench_output in self._swe_bench_outputs:
+            yield {"id": swebench_output.instance_id, "score": "-", "reasoning": "-"}
+
+    @staticmethod
+    def compute_score(success_cnt: int, total_cnt: int) -> float:
+        if total_cnt == 0:
+            return 0.0
+        score = success_cnt / total_cnt
+        return min(max(score, 0.0), 1.0)
+
+    async def evaluate(self, eval_input: EvalInput) -> EvalOutput:
+        '''Run the swebench evaluation and store the report in the output directory'''
+
+        # Process the EvalInput
+        workflow_input_file, workflow_output_file = self.process_eval_input(eval_input)
+        if not workflow_input_file or not workflow_output_file:
+            # nothing to evaluate
+            return EvalOutput(average_score=0.0, eval_output_items=[])
+
+        report_dir = self.output_dir / "swe_bench_reports"
+        self.empty_report_dir(report_dir)
+
+        logger.info("Starting swe_bench run %s", self.run_id)
+        swebench_eval.main(dataset_name=str(workflow_input_file),
+                           split="dev",
+                           instance_ids=[],
+                           predictions_path=str(workflow_output_file),
+                           max_workers=self.max_workers,
+                           force_rebuild=False,
+                           cache_level="env",
+                           clean=False,
+                           open_file_limit=4096,
+                           run_id=self.run_id,
+                           timeout=1800,
+                           namespace=None,
+                           rewrite_reports=False,
+                           modal=False,
+                           instance_image_tag='latest',
+                           report_dir=str(report_dir))
+        logger.info("Completed swe_bench run %s", self.run_id)
+
+        swe_bench_report_file = f"{self._model_name_or_path}.{self.run_id}.json"
+
+        # There is a bug in swebench because of which report_dir is being ignored. Copy the report to the output dir
+        self.move_report_and_logs(swe_bench_report_file=swe_bench_report_file, logs_dir="logs", report_dir=report_dir)
+        logger.info("SWE_bench report and logs written to %s directory", report_dir)
+
+        # read the swe_bench report file
+        report_file = report_dir / swe_bench_report_file
+        # if report file is not present, return empty EvalOutput
+        avg_score = 0.0
+        if report_file.exists():
+            with open(report_file, "r", encoding="utf-8") as f:
+                report = json.load(f)
+                resolved_instances = report.get("resolved_instances", 0)
+                total_instances = report.get("total_instances", 0)
+                avg_score = self.compute_score(resolved_instances, total_instances)
+
+        # Build the EvalOutput from self._swe_bench_outputs and avg_score
+        eval_output_items = list(self.build_eval_output())
+        return EvalOutput(average_score=avg_score, eval_output_items=eval_output_items)

aiq/eval/swe_bench_evaluator/register.py

@@ -0,0 +1,36 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from pydantic import Field
+
+from aiq.builder.builder import EvalBuilder
+from aiq.builder.evaluator import EvaluatorInfo
+from aiq.cli.register_workflow import register_evaluator
+from aiq.data_models.evaluator import EvaluatorBaseConfig
+
+
+class SweBenchEvaluatorConfig(EvaluatorBaseConfig, name="swe_bench"):
+    """Code patch evaluation for SWE Bench problems."""
+
+    run_id: str = Field(description="swe-bench test harness run identifier.")
+
+
+@register_evaluator(config_type=SweBenchEvaluatorConfig)
+async def register_swe_bench_evaluator(config: SweBenchEvaluatorConfig, builder: EvalBuilder):
+
+    from .evaluate import SweBenchEvaluator
+    _evaluator = SweBenchEvaluator(config.run_id, builder.get_max_concurrency(), builder.get_output_dir())
+
+    yield EvaluatorInfo(config=config, evaluate_fn=_evaluator.evaluate, description="SWE Bench Evaluator")

aiq/eval/trajectory_evaluator/evaluate.py

@@ -0,0 +1,118 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import asyncio
+import logging
+
+from langchain.evaluation import TrajectoryEvalChain
+from langchain_core.language_models import BaseChatModel
+from langchain_core.tools import BaseTool
+from tqdm import tqdm
+
+from aiq.eval.evaluator.evaluator_model import EvalInput
+from aiq.eval.evaluator.evaluator_model import EvalInputItem
+from aiq.eval.evaluator.evaluator_model import EvalOutput
+from aiq.eval.evaluator.evaluator_model import EvalOutputItem
+from aiq.eval.utils.tqdm_position_registry import TqdmPositionRegistry
+
+logger = logging.getLogger(__name__)
+
+
+class TrajectoryEvaluator:
+
+    def __init__(
+        self,
+        llm: BaseChatModel,
+        tools: list[BaseTool] | None = None,
+        max_concurrency: int = 8,
+    ):
+
+        self.llm = llm
+        self.tools = tools
+        self.max_concurrency = max_concurrency
+        self.semaphore = asyncio.Semaphore(self.max_concurrency)
+        # Initialize trajectory evaluation chain
+        self.traj_eval_chain = TrajectoryEvalChain.from_llm(llm=self.llm,
+                                                            tools=self.tools,
+                                                            return_reasoning=True,
+                                                            requires_reference=True)
+        logger.debug("Trajectory evaluation chain initialized.")
+
+    async def evaluate(self, eval_input: EvalInput) -> EvalOutput:
+        """
+        Evaluates the agent trajectories using trajectory evaluation chain.
+        """
+
+        num_records = len(eval_input.eval_input_items)
+        logger.info("Running trajectory evaluation with %d records", num_records)
+        from aiq.data_models.intermediate_step import IntermediateStepType
+        from aiq.eval.intermediate_step_adapter import IntermediateStepAdapter
+
+        intermediate_step_adapter = IntermediateStepAdapter()
+        event_filter = [IntermediateStepType.LLM_END, IntermediateStepType.TOOL_END]
+
+        async def process_item(item: EvalInputItem) -> tuple[float, dict]:
+            """
+            Evaluate a single EvalInputItem asynchronously and return a tuple of-
+            1. score
+            2. reasoning for the score
+            """
+            question = item.input_obj
+            generated_answer = item.output_obj
+            agent_trajectory = intermediate_step_adapter.get_agent_actions(item.trajectory, event_filter)
+            try:
+                eval_result = await self.traj_eval_chain.aevaluate_agent_trajectory(
+                    input=question,
+                    agent_trajectory=agent_trajectory,
+                    prediction=generated_answer,
+                )
+            except Exception as e:
+                logger.exception("Error evaluating trajectory for question: %s, Error: %s", question, e, exc_info=True)
+                return 0.0, f"Error evaluating trajectory: {e}"
+
+            reasoning = {
+                "reasoning": eval_result["reasoning"],
+                "trajectory": [(action.model_dump(), output) for (action, output) in agent_trajectory]
+            }
+            return eval_result["score"], reasoning
+
+        async def wrapped_process(item: EvalInputItem) -> tuple[float, dict]:
+            async with self.semaphore:
+                result = await process_item(item)
+                pbar.update(1)
+                return result
+
+        # Execute all evaluations asynchronously
+        try:
+            tqdm_position = TqdmPositionRegistry.claim()
+            pbar = tqdm(total=len(eval_input.eval_input_items), desc="Evaluating Trajectory", position=tqdm_position)
+            results = await asyncio.gather(*[wrapped_process(item) for item in eval_input.eval_input_items])
+        finally:
+            pbar.close()
+            TqdmPositionRegistry.release(tqdm_position)
+
+        # Extract scores and reasonings
+        sample_scores, sample_reasonings = zip(*results) if results else ([], [])
+
+        # Compute average score
+        avg_score = round(sum(sample_scores) / len(sample_scores), 2) if sample_scores else 0.0
+
+        # Construct EvalOutputItems
+        eval_output_items = [
+            EvalOutputItem(id=item.id, score=score, reasoning=reasoning)
+            for item, score, reasoning in zip(eval_input.eval_input_items, sample_scores, sample_reasonings)
+        ]
+
+        return EvalOutput(average_score=avg_score, eval_output_items=eval_output_items)

aiq/eval/trajectory_evaluator/register.py

@@ -0,0 +1,40 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from pydantic import Field
+
+from aiq.builder.builder import EvalBuilder
+from aiq.builder.evaluator import EvaluatorInfo
+from aiq.cli.register_workflow import register_evaluator
+from aiq.data_models.evaluator import EvaluatorBaseConfig
+
+
+class TrajectoryEvaluatorConfig(EvaluatorBaseConfig, name="trajectory"):
+    """Agent Trajectory Evaluation."""
+
+    llm_name: str = Field(description="LLM as a judge.")
+
+
+@register_evaluator(config_type=TrajectoryEvaluatorConfig)
+async def register_trajectory_evaluator(config: TrajectoryEvaluatorConfig, builder: EvalBuilder):
+    from aiq.builder.framework_enum import LLMFrameworkEnum
+
+    from .evaluate import TrajectoryEvaluator
+    llm = await builder.get_llm(config.llm_name, wrapper_type=LLMFrameworkEnum.LANGCHAIN)
+    tools = builder.get_all_tools(wrapper_type=LLMFrameworkEnum.LANGCHAIN)
+
+    _evaluator = TrajectoryEvaluator(llm, tools, builder.get_max_concurrency())
+
+    yield EvaluatorInfo(config=config, evaluate_fn=_evaluator.evaluate, description="Trajectory Evaluator")

aiq/eval/utils/output_uploader.py

@@ -0,0 +1,131 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import asyncio
+import logging
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+import aioboto3
+from botocore.exceptions import NoCredentialsError
+from tqdm import tqdm
+
+from aiq.data_models.evaluate import EvalOutputConfig
+
+logger = logging.getLogger(__name__)
+
+
+class OutputUploader:
+    """
+    Run custom scripts and upload evaluation outputs using the configured s3
+    credentials.
+    """
+
+    def __init__(self, output_config: EvalOutputConfig, job_id: str | None = None):
+        self.output_config = output_config
+        self._s3_client = None
+        self.job_id = job_id
+
+    @property
+    def s3_config(self):
+        return self.output_config.s3
+
+    async def _upload_file(self, s3_client, bucket, s3_key, local_path, pbar):
+        try:
+            await s3_client.upload_file(str(local_path), bucket, s3_key)
+            logger.info("Uploaded %s to s3://%s/%s", local_path, bucket, s3_key)
+            pbar.update(1)
+        except Exception as e:
+            logger.error("Failed to upload %s to s3://%s/%s: %s", local_path, bucket, s3_key, e)
+            raise
+
+    async def upload_directory(self):
+        """
+        Upload the contents of the local output directory to the remote S3 bucket in parallel.
+        """
+        if not self.output_config.s3:
+            logger.info("No S3 config provided; skipping upload.")
+            return
+
+        local_dir = self.output_config.dir
+        bucket = self.s3_config.bucket
+        remote_prefix = self.output_config.remote_dir or ""
+        if self.job_id:
+            remote_prefix = str(Path(remote_prefix) / f"jobs/{self.job_id}")
+
+        file_entries = []
+        for root, _, files in os.walk(local_dir):
+            for file in files:
+                local_path = Path(root) / file
+                relative_path = local_path.relative_to(local_dir)
+                s3_path = Path(remote_prefix) / relative_path
+                s3_key = str(s3_path).replace("\\", "/")  # Normalize for S3
+                file_entries.append((local_path, s3_key))
+
+        session = aioboto3.Session()
+        try:
+            async with session.client(
+                    "s3",
+                    endpoint_url=self.s3_config.endpoint_url,
+                    aws_access_key_id=self.s3_config.access_key,
+                    aws_secret_access_key=self.s3_config.secret_key,
+            ) as s3_client:
+                with tqdm(total=len(file_entries), desc="Uploading files to S3") as pbar:
+                    upload_tasks = [
+                        self._upload_file(s3_client, bucket, s3_key, local_path, pbar)
+                        for local_path, s3_key in file_entries
+                    ]
+                    await asyncio.gather(*upload_tasks)
+
+        except NoCredentialsError as e:
+            logger.error("AWS credentials not available: %s", e)
+            raise
+        except Exception as e:
+            logger.error("Failed to upload files to S3: %s", e)
+            raise
+
+    def run_custom_scripts(self):
+        """
+        Run custom Python scripts defined in the EvalOutputConfig.
+        Each script is run with its kwargs passed as command-line arguments.
+        The output directory is passed as the first argument.
+        """
+        for _, script_config in self.output_config.custom_scripts.items():
+            script_path = script_config.script
+            if not script_path.exists():
+                logger.error("Custom script %s does not exist.", script_path)
+                continue
+
+            # use python interpreter
+            args = [sys.executable, str(script_path)]
+            # add output directory as first keyword argument
+            args.append("--output_dir")
+            args.append(str(self.output_config.dir))
+            if script_config.kwargs:
+                for key, value in script_config.kwargs.items():
+                    args.append(f"--{key}")
+                    args.append(str(value))
+
+            display_args = " ".join(f'"{arg}"' if " " in arg else arg for arg in args[1:])
+
+            try:
+                logger.info("Running custom script: %s %s", script_path, display_args)
+                subprocess.run(args, check=True, text=True)
+                logger.info("Custom script %s completed successfully.", script_path)
+            except subprocess.CalledProcessError as e:
+                logger.error("Custom script %s failed with return code %s", script_path, e.returncode)
+                raise

aiq/eval/utils/tqdm_position_registry.py

@@ -0,0 +1,40 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+class TqdmPositionRegistry:
+    """
+    A simple registry for tqdm positions.
+    """
+    _positions = set()
+    _max_positions = 100
+
+    @classmethod
+    def claim(cls) -> int:
+        """
+        Claim a tqdm position in the range of 0-99.
+        """
+        for i in range(cls._max_positions):
+            if i not in cls._positions:
+                cls._positions.add(i)
+                return i
+        raise RuntimeError("No available tqdm positions.")
+
+    @classmethod
+    def release(cls, pos: int):
+        """
+        Release a tqdm position.
+        """
+        cls._positions.discard(pos)

aiq/front_ends/__init__.py

@@ -0,0 +1,14 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

aiq/front_ends/console/__init__.py

@@ -0,0 +1,14 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

aiq/front_ends/console/console_front_end_config.py

@@ -0,0 +1,32 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from pathlib import Path
+
+from pydantic import Field
+
+from aiq.data_models.front_end import FrontEndBaseConfig
+
+
+class ConsoleFrontEndConfig(FrontEndBaseConfig, name="console"):
+    """
+    A front end that allows an AgentIQ workflow to be run from the console.
+    """
+
+    input_query: list[str] | None = Field(default=None,
+                                          alias="input",
+                                          description="A single input to submit the the workflow.")
+    input_file: Path | None = Field(default=None,
+                                    description="Path to a json file of inputs to submit to the workflow.")