aiqtoolkit 1.1.0a20250429__py3-none-any.whl

aiq/data_models/registry_handler.py

@@ -0,0 +1,26 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import typing
+
+from .common import BaseModelRegistryTag
+from .common import TypedBaseModel
+
+
+class RegistryHandlerBaseConfig(TypedBaseModel, BaseModelRegistryTag):
+    pass
+
+
+RegistryHandlerBaseConfigT = typing.TypeVar("RegistryHandlerBaseConfigT", bound=RegistryHandlerBaseConfig)

aiq/data_models/retriever.py

@@ -0,0 +1,30 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import typing
+
+from aiq.data_models.common import BaseModelRegistryTag
+from aiq.data_models.common import TypedBaseModel
+
+
+class RetrieverBaseConfig(TypedBaseModel, BaseModelRegistryTag):
+    """
+    The base level config object for a retriever object. Retrievers use different provider clients (e.g., Milvus) to
+    provide an interface for searching for and retrieving documents from the configured data store.
+    """
+    pass
+
+
+RetrieverBaseConfigT = typing.TypeVar("RetrieverBaseConfigT", bound=RetrieverBaseConfig)

aiq/data_models/step_adaptor.py

@@ -0,0 +1,64 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+from enum import Enum
+
+from pydantic import BaseModel
+from pydantic import Field
+from pydantic import model_validator
+
+from aiq.data_models.intermediate_step import IntermediateStepType
+
+logger = logging.getLogger(__name__)
+
+
+class StepAdaptorMode(str, Enum):
+    DEFAULT = "default"
+    CUSTOM = "custom"
+    OFF = "off"
+
+
+class StepAdaptorConfig(BaseModel):
+    """
+    Configures how intermediate steps are filtered and normalized by the StepAdaptor.
+
+    Args:
+        mode (StepAdaptorMode): One of:
+            - 'current' => pass only LLM (all LLM_* events) + TOOL_END
+            - 'end_events_only' => pass only LLM_END and TOOL_END
+            - 'custom' => pass only the events in custom_event_types
+        custom_event_types (list[IntermediateStepType]):
+            If mode == 'custom', we only pass events whose event_type is in this list.
+            Otherwise, this field is ignored.
+    """
+    mode: StepAdaptorMode = StepAdaptorMode.DEFAULT
+    custom_event_types: list[IntermediateStepType] = Field(default_factory=list)
+
+    @model_validator(mode="after")
+    def check_custom_event_types(self) -> "StepAdaptorConfig":
+        """
+        Validates custom configurations
+        """
+        if self.mode != StepAdaptorMode.CUSTOM and self.custom_event_types:
+            logger.warning("Ignoring custom_event_types because mode is not 'custom'")
+            self.custom_event_types = []
+        elif self.mode == StepAdaptorMode.CUSTOM and not self.custom_event_types:
+            logger.warning("No custom_event_types provided for custom mode. Defaulting to CUSTOM_START and CUSTOM_END")
+            self.custom_event_types = [IntermediateStepType.CUSTOM_START, IntermediateStepType.CUSTOM_END]
+        elif self.mode == StepAdaptorMode.OFF:
+            logger.warning("StepAdaptor is disabled. Ignoring all intermediate event types")
+            self.custom_event_types = []
+        return self

aiq/data_models/streaming.py

@@ -0,0 +1,33 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import inspect
+import typing
+from collections.abc import Callable
+
+_T = typing.TypeVar("_T")
+
+
+class Streaming(typing.Generic[_T]):
+
+    def __init__(self, *, convert: Callable[[typing.Any], _T] = None):
+        self.convert = convert
+
+        sig = inspect.signature(self.convert)
+
+        if (sig.return_annotation == inspect.Parameter.empty):
+            raise ValueError("convert must have a return annotation")
+
+        self.single_output_type = sig.return_annotation

aiq/data_models/swe_bench_model.py

@@ -0,0 +1,54 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+
+from pydantic import BaseModel
+from pydantic import ConfigDict
+from pydantic import field_validator
+
+
+class SWEBenchInput(BaseModel):
+    # Allow extra fields in the model_config to support derived models
+    model_config = ConfigDict(extra="allow")
+
+    repo: str
+    instance_id: str
+    base_commit: str
+    patch: str
+    test_patch: str
+    problem_statement: str
+    hints_text: str
+    created_at: str | int
+    version: float
+    FAIL_TO_PASS: list[str]
+    PASS_TO_PASS: list[str]
+    environment_setup_commit: str
+
+    # Handle improperly formatted JSON strings for list fields
+    @field_validator("FAIL_TO_PASS", "PASS_TO_PASS", mode="before")
+    def parse_list_fields(cls, value):  # pylint: disable=no-self-argument
+        if isinstance(value, str):
+            # Attempt to parse the string as a list
+            return json.loads(value)
+        return value
+
+
+class SWEBenchOutput(BaseModel):
+    model_config = ConfigDict(protected_namespaces=(), populate_by_name=True)
+
+    instance_id: str
+    model_name_or_path: str
+    model_patch: str

aiq/data_models/telemetry_exporter.py

@@ -0,0 +1,26 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import typing
+
+from .common import BaseModelRegistryTag
+from .common import TypedBaseModel
+
+
+class TelemetryExporterBaseConfig(TypedBaseModel, BaseModelRegistryTag):
+    pass
+
+
+TelemetryExporterConfigT = typing.TypeVar("TelemetryExporterConfigT", bound=TelemetryExporterBaseConfig)

aiq/embedder/langchain_client.py

@@ -0,0 +1,41 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from aiq.builder.builder import Builder
+from aiq.builder.framework_enum import LLMFrameworkEnum
+from aiq.cli.register_workflow import register_embedder_client
+from aiq.embedder.nim_embedder import NIMEmbedderModelConfig
+
+
+@register_embedder_client(config_type=NIMEmbedderModelConfig, wrapper_type=LLMFrameworkEnum.LANGCHAIN)
+async def nim_langchain(embedder_config: NIMEmbedderModelConfig, builder: Builder):
+
+    from langchain_nvidia_ai_endpoints import NVIDIAEmbeddings
+
+    yield NVIDIAEmbeddings(**embedder_config.model_dump(exclude={"type"}, by_alias=True))
+
+
+@register_embedder_client(config_type=NIMEmbedderModelConfig, wrapper_type=LLMFrameworkEnum.LLAMA_INDEX)
+async def nim_llamaindex(embedder_config: NIMEmbedderModelConfig, builder: Builder):
+
+    from llama_index.embeddings.nvidia import NVIDIAEmbedding  # pylint: disable=no-name-in-module
+
+    config_obj = {
+        **embedder_config.model_dump(exclude={"type", "model_name"}, by_alias=True),
+        "model":
+            embedder_config.model_name,
+    }
+
+    yield NVIDIAEmbedding(**config_obj)

aiq/embedder/nim_embedder.py

@@ -0,0 +1,58 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import typing
+
+from pydantic import AfterValidator
+from pydantic import AliasChoices
+from pydantic import ConfigDict
+from pydantic import Field
+
+from aiq.builder.builder import Builder
+from aiq.builder.embedder import EmbedderProviderInfo
+from aiq.cli.register_workflow import register_embedder_provider
+from aiq.data_models.embedder import EmbedderBaseConfig
+
+allowed_truncate_values = ["NONE", "START", "END"]
+
+
+def option_in_allowed_values(v):
+    """Ensures option is allowed"""
+    assert v in allowed_truncate_values
+    return v
+
+
+TruncationOption = typing.Annotated[str, AfterValidator(option_in_allowed_values)]
+
+
+class NIMEmbedderModelConfig(EmbedderBaseConfig, name="nim"):
+    """A NVIDIA Inference Microservice (NIM) embedder provider to be used with an embedder client."""
+
+    api_key: str | None = Field(default=None, description="NVIDIA API key to interact with hosted NIM.")
+    base_url: str | None = Field(default=None, description="Base url to the hosted NIM.")
+    model_name: str = Field(validation_alias=AliasChoices("model_name", "model"),
+                            serialization_alias="model",
+                            description="The model name for the hosted NIM.")
+    truncate: TruncationOption = Field(default="NONE",
+                                       description=("The truncation strategy if the input on the "
+                                                    "server side if it's too large."))
+
+    model_config = ConfigDict(protected_namespaces=())
+
+
+@register_embedder_provider(config_type=NIMEmbedderModelConfig)
+async def nim_embedder_model(embedder_config: NIMEmbedderModelConfig, builder: Builder):
+
+    yield EmbedderProviderInfo(config=embedder_config, description="A NIM model for use with an Embedder client.")

aiq/embedder/openai_embedder.py

@@ -0,0 +1,42 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from pydantic import AliasChoices
+from pydantic import ConfigDict
+from pydantic import Field
+
+from aiq.builder.builder import Builder
+from aiq.builder.embedder import EmbedderProviderInfo
+from aiq.cli.register_workflow import register_embedder_provider
+from aiq.data_models.embedder import EmbedderBaseConfig
+
+
+class OpenAIEmbedderModelConfig(EmbedderBaseConfig, name="openai"):
+    """An OpenAI LLM provider to be used with an LLM client."""
+
+    model_config = ConfigDict(protected_namespaces=())
+
+    api_key: str | None = Field(default=None, description="OpenAI API key to interact with hosted model.")
+    base_url: str | None = Field(default=None, description="Base url to the hosted model.")
+    model_name: str = Field(validation_alias=AliasChoices("model_name", "model"),
+                            serialization_alias="model",
+                            description="The OpenAI hosted model name.")
+    max_retries: int = Field(default=2, description="The max number of retries for the request.")
+
+
+@register_embedder_provider(config_type=OpenAIEmbedderModelConfig)
+async def openai_llm(config: OpenAIEmbedderModelConfig, builder: Builder):
+
+    yield EmbedderProviderInfo(config=config, description="An OpenAI model for use with an Embedder client.")

aiq/embedder/register.py

@@ -0,0 +1,24 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# pylint: disable=unused-import
+# flake8: noqa
+# isort:skip_file
+
+# Import any providers which need to be automatically registered here
+from . import nim_embedder
+from . import openai_embedder
+# Import any clients which need to be automatically registered here
+from . import langchain_client

aiq/eval/__init__.py

@@ -0,0 +1,14 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

aiq/eval/config.py

@@ -0,0 +1,42 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from pathlib import Path
+
+from pydantic import BaseModel
+
+
+class EvaluationRunConfig(BaseModel):
+    """
+    Parameters used for a single evaluation run.
+    """
+    config_file: Path
+    dataset: str | None  # dataset file path can be specified in the config file
+    result_json_path: str = "$"
+    skip_workflow: bool = False
+    skip_completed_entries: bool = False
+    endpoint: str | None = None  # only used when running the workflow remotely
+    endpoint_timeout: int = 300
+    reps: int = 1
+    override: tuple[tuple[str, str], ...] = ()
+
+
+class EvaluationRunOutput(BaseModel):
+    """
+    Output of a single evaluation run.
+    """
+    workflow_output_file: Path | None
+    evaluator_output_files: list[Path]
+    workflow_interrupted: bool

aiq/eval/dataset_handler/dataset_downloader.py

@@ -0,0 +1,106 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import logging
+from pathlib import Path
+
+import boto3
+import requests
+from botocore.exceptions import NoCredentialsError
+
+from aiq.data_models.dataset_handler import EvalDatasetConfig
+
+logger = logging.getLogger(__name__)
+
+
+class DatasetDownloader:
+    """
+    Download remote datasets using signed URLs or S3 credentials.
+
+    One DatasetDownloader object is needed for each dataset to be downloaded.
+    """
+
+    def __init__(self, dataset_config: EvalDatasetConfig):
+        self.dataset_config = dataset_config
+        self._s3_client = None
+
+    @property
+    def s3_config(self):
+        return self.dataset_config.s3
+
+    @property
+    def s3_client(self):
+        """Lazy init the S3 client."""
+        if not self._s3_client:
+            try:
+                self._s3_client = boto3.client("s3",
+                                               endpoint_url=self.s3_config.endpoint_url,
+                                               aws_access_key_id=self.s3_config.access_key,
+                                               aws_secret_access_key=self.s3_config.secret_key)
+            except NoCredentialsError as e:
+                logger.error("AWS credentials not available: %s", e)
+                raise
+            except Exception as e:
+                logger.error("Failed to initialize S3 client: %s", e)
+                raise
+        return self._s3_client
+
+    @staticmethod
+    def ensure_directory_exists(file_path: str):
+        """Ensure the directory for the file exists."""
+        Path(file_path).parent.mkdir(parents=True, exist_ok=True)
+
+    def download_with_signed_url(self, remote_file_path: str, local_file_path: str, timeout: int = 300):
+        """Download a file using a signed URL."""
+        try:
+            response = requests.get(remote_file_path, stream=True, timeout=timeout)
+            response.raise_for_status()
+            with open(local_file_path, "wb") as file:
+                for chunk in response.iter_content(chunk_size=8192):
+                    file.write(chunk)
+            logger.info("File downloaded successfully to %s using signed URL.", local_file_path)
+        except requests.exceptions.RequestException as e:
+            logger.error("Error downloading file using signed URL: %s", e)
+            raise
+
+    def download_with_boto3(self, remote_file_path: str, local_file_path: str):
+        """Download a file using boto3 and credentials."""
+        try:
+            self.s3_client.download_file(self.dataset_config.s3.bucket, remote_file_path, local_file_path)
+            logger.info("File downloaded successfully to %s using S3 client.", local_file_path)
+        except Exception as e:
+            logger.error("Error downloading file from S3: %s", e)
+            raise
+
+    @staticmethod
+    def is_file_path_url(file_path: str) -> bool:
+        """Check if the file path is a URL."""
+        return file_path.startswith("http")
+
+    def download_file(self, remote_file_path: str, local_file_path: str):
+        """Download a file using the appropriate method."""
+        self.ensure_directory_exists(local_file_path)
+        if self.is_file_path_url(remote_file_path):
+            logger.info("Using signed URL to download the file %s...", remote_file_path)
+            self.download_with_signed_url(remote_file_path, local_file_path, timeout=120)
+        else:
+            logger.info("Using S3 credentials to download the file %s...", remote_file_path)
+            self.download_with_boto3(remote_file_path, local_file_path)
+
+    def download_dataset(self):
+        """Download datasets defined in the evaluation configuration."""
+        if self.dataset_config.remote_file_path:
+            logger.info("Downloading remote dataset %s")
+            self.download_file(remote_file_path=self.dataset_config.remote_file_path,
+                               local_file_path=self.dataset_config.file_path)

aiq/eval/dataset_handler/dataset_filter.py

@@ -0,0 +1,52 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pandas as pd
+
+from aiq.data_models.dataset_handler import EvalFilterConfig
+
+
+class DatasetFilter:
+    """
+    Apply allowlist and denylist filters to the DataFrame based on specified column filters.
+        - If a allowlist is provided, only keep rows matching the filter values.
+        - If a denylist is provided, remove rows matching the filter values.
+        - If the filter column does not exist in the DataFrame, the filtering is skipped for that column.
+
+    This is a utility class that is dataset agnostic and can be used to filter any DataFrame based on the provided
+    filter configuration.
+    """
+
+    def __init__(self, filter_config: EvalFilterConfig):
+
+        self.filter_config = filter_config
+
+    def apply_filters(self, df) -> pd.DataFrame:
+
+        filtered_df = df.copy()
+
+        # Apply allowlist (only keep specified rows)
+        if self.filter_config.allowlist:
+            for column, values in self.filter_config.allowlist.field.items():
+                if column in filtered_df.columns:
+                    filtered_df = filtered_df[filtered_df[column].isin(values)]
+
+        # Apply denylist (remove specified rows)
+        if self.filter_config.denylist:
+            for column, values in self.filter_config.denylist.field.items():
+                if column in filtered_df.columns:
+                    filtered_df = filtered_df[~filtered_df[column].isin(values)]
+
+        return filtered_df