dcs-sdk 1.6.4__py3-none-any.whl → 1.6.6__py3-none-any.whl

dcs_core/core/validation/base.py

@@ -0,0 +1,230 @@
+#  Copyright 2022-present, the Waterdip Labs Pvt. Ltd.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+import datetime
+import json
+import sys
+import traceback
+from abc import ABC, abstractmethod
+from typing import Optional, Tuple, Union
+
+from loguru import logger
+
+from dcs_core.core.common.models.configuration import (
+    DataSourceLanguageSupport,
+    ValidationConfig,
+)
+from dcs_core.core.common.models.validation import (
+    ConditionType,
+    DeltaValidationInfo,
+    ValidationFunction,
+    ValidationInfo,
+)
+from dcs_core.core.datasource.manager import DataSource
+
+
+class ValidationIdentity:
+    @staticmethod
+    def generate_identity(
+        validation_function: ValidationFunction,
+        validation_name: str,
+        data_source_name: str = None,
+        dataset_name: str = None,
+        field_name: str = None,
+    ) -> str:
+        """
+        Generate a unique identifier for a metric
+        """
+
+        identifiers = []
+
+        if data_source_name is not None:
+            identifiers.append(data_source_name)
+        if dataset_name:
+            identifiers.append(dataset_name)
+        if field_name:
+            identifiers.append(field_name)
+        if validation_function:
+            identifiers.append(validation_function.value)
+        if validation_name:
+            identifiers.append(validation_name)
+        return ".".join([str(p) for p in identifiers])
+
+
+class Validation(ABC):
+    """
+    Validation is a class that represents a validation that is generated by a data source.
+    """
+
+    def __init__(
+        self,
+        name: str,
+        validation_config: ValidationConfig,
+        data_source: DataSource,
+        dataset_name: str,
+        field_name: str = None,
+        **kwargs,
+    ):
+        self.name = name
+        self.validation_config = validation_config
+        self.data_source = data_source
+        self.dataset_name = dataset_name
+        self.field_name = field_name
+
+        self.query = validation_config.query
+
+        self.threshold = validation_config.threshold
+        self.where_filter = None
+        self.values = None
+        self.regex_pattern = validation_config.regex
+
+        if validation_config.where:
+            if data_source.language_support == DataSourceLanguageSupport.DSL_ES:
+                self.where_filter = json.loads(validation_config.where)
+            elif data_source.language_support == DataSourceLanguageSupport.SQL:
+                self.where_filter = validation_config.where
+        if validation_config.values:
+            if data_source.language_support == DataSourceLanguageSupport.SQL:
+                self.values = validation_config.values
+
+    def get_validation_identity(self) -> str:
+        return ValidationIdentity.generate_identity(
+            validation_function=self.validation_config.get_validation_function,
+            validation_name=self.name,
+            data_source_name=self.data_source.data_source_name,
+            dataset_name=self.dataset_name,
+            field_name=self.field_name,
+        )
+
+    def _validate_threshold(self, metric_value) -> Tuple[bool, Optional[str]]:
+        for operator, value in self.threshold.__dict__.items():
+            if value is not None:
+                if ConditionType.GTE == operator:
+                    if metric_value < value:
+                        return (
+                            False,
+                            f"Less than threshold value of {value}",
+                        )
+                elif ConditionType.LTE == operator:
+                    if metric_value > value:
+                        return (
+                            False,
+                            f"Greater than threshold value of {value}",
+                        )
+                elif ConditionType.GT == operator:
+                    if metric_value <= value:
+                        return (
+                            False,
+                            f"Less than or equal to threshold value of {value}",
+                        )
+                elif ConditionType.LT == operator:
+                    if metric_value >= value:
+                        return (
+                            False,
+                            f"Greater than or equal to threshold value of {value}",
+                        )
+                elif ConditionType.EQ == operator:
+                    if metric_value != value:
+                        return (
+                            False,
+                            f"Not equal to the value of {value}",
+                        )
+        return True, None
+
+    @abstractmethod
+    def _generate_metric_value(self, **kwargs) -> Union[float, int]:
+        pass
+
+    def get_validation_info(self, **kwargs) -> Union[ValidationInfo, None]:
+        try:
+            metric_value = self._generate_metric_value(**kwargs)
+            tags = {
+                "name": self.name,
+            }
+
+            value = ValidationInfo(
+                name=self.name,
+                identity=self.get_validation_identity(),
+                data_source_name=self.data_source.data_source_name,
+                dataset=self.dataset_name,
+                validation_function=self.validation_config.get_validation_function,
+                field=self.field_name,
+                value=metric_value,
+                timestamp=datetime.datetime.utcnow(),
+                tags=tags,
+            )
+            if self.threshold is not None:
+                value.is_valid, value.reason = self._validate_threshold(metric_value)
+
+            return value
+        except Exception as e:
+            traceback.print_exc(file=sys.stdout)
+            logger.error(f"Failed to generate metric {self.name}: {str(e)}")
+            return None
+
+
+class DeltaValidation(Validation, ABC):
+    def __init__(
+        self,
+        name: str,
+        validation_config: ValidationConfig,
+        data_source: DataSource,
+        dataset_name: str,
+        reference_data_source: DataSource,
+        reference_dataset_name: str,
+        reference_field_name: str = None,
+        **kwargs,
+    ):
+        super().__init__(name, validation_config, data_source, dataset_name, **kwargs)
+        self.reference_data_source = reference_data_source
+        self.reference_dataset_name = reference_dataset_name
+        self.reference_field_name = reference_field_name
+
+    @abstractmethod
+    def _generate_reference_metric_value(self, **kwargs) -> Union[float, int]:
+        pass
+
+    def get_validation_info(self, **kwargs) -> Union[ValidationInfo, None]:
+        try:
+            metric_value = self._generate_metric_value(**kwargs)
+            reference_metric_value = self._generate_reference_metric_value(**kwargs)
+            delta_value = abs(metric_value - reference_metric_value)
+
+            tags = {
+                "name": self.name,
+            }
+
+            value = DeltaValidationInfo(
+                name=self.name,
+                identity=self.get_validation_identity(),
+                data_source_name=self.data_source.data_source_name,
+                dataset=self.dataset_name,
+                validation_function=self.validation_config.get_validation_function,
+                field=self.field_name,
+                value=delta_value,
+                source_value=metric_value,
+                reference_value=reference_metric_value,
+                reference_datasource_name=self.reference_data_source.data_source_name,
+                reference_dataset=self.reference_dataset_name,
+                timestamp=datetime.datetime.utcnow(),
+                tags=tags,
+            )
+            if self.threshold is not None:
+                value.is_valid, value.reason = self._validate_threshold(delta_value)
+
+            return value
+        except Exception as e:
+            traceback.print_exc(file=sys.stdout)
+            logger.error(f"Failed to generate metric {self.name}: {str(e)}")
+            return None

dcs_core/core/validation/completeness_validation.py

@@ -0,0 +1,153 @@
+#  Copyright 2022-present, the Waterdip Labs Pvt. Ltd.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+import re
+from typing import Union
+
+from dcs_core.core.datasource.search_datasource import SearchIndexDataSource
+from dcs_core.core.datasource.sql_datasource import SQLDataSource
+from dcs_core.core.validation.base import Validation
+from dcs_core.integrations.databases.oracle import OracleDataSource
+
+
+class CountNullValidation(Validation):
+    def _generate_metric_value(self, **kwargs) -> Union[float, int]:
+        if isinstance(self.data_source, SQLDataSource):
+            if isinstance(self.data_source, OracleDataSource) and self.where_filter:
+                self.where_filter = re.sub(r"(\b[a-zA-Z_]+\b)(?=\s*[=<>])", r'"\1"', self.where_filter)
+            return self.data_source.query_get_null_count(
+                table=self.dataset_name,
+                field=f'"{self.field_name}"' if isinstance(self.data_source, OracleDataSource) else self.field_name,
+                filters=self.where_filter if self.where_filter is not None else None,
+            )
+        elif isinstance(self.data_source, SearchIndexDataSource):
+            return self.data_source.query_get_null_count(
+                index_name=self.dataset_name,
+                field=self.field_name,
+                filters=self.where_filter if self.where_filter else None,
+            )
+        else:
+            raise ValueError("Invalid data source type")
+
+
+class PercentageNullValidation(Validation):
+    def _generate_metric_value(self, **kwargs) -> Union[float, int]:
+        if isinstance(self.data_source, SQLDataSource):
+            if isinstance(self.data_source, OracleDataSource) and self.where_filter:
+                self.where_filter = re.sub(r"(\b[a-zA-Z_]+\b)(?=\s*[=<>])", r'"\1"', self.where_filter)
+            return self.data_source.query_get_null_percentage(
+                table=self.dataset_name,
+                field=f'"{self.field_name}"' if isinstance(self.data_source, OracleDataSource) else self.field_name,
+                filters=self.where_filter if self.where_filter is not None else None,
+            )
+        elif isinstance(self.data_source, SearchIndexDataSource):
+            return self.data_source.query_get_null_percentage(
+                index_name=self.dataset_name,
+                field=self.field_name,
+                filters=self.where_filter if self.where_filter else None,
+            )
+        else:
+            raise ValueError("Invalid data source type")
+
+
+class CountEmptyStringValidation(Validation):
+    def _generate_metric_value(self, **kwargs) -> Union[float, int]:
+        if isinstance(self.data_source, SQLDataSource):
+            if isinstance(self.data_source, OracleDataSource) and self.where_filter:
+                self.where_filter = re.sub(r"(\b[a-zA-Z_]+\b)(?=\s*[=<>])", r'"\1"', self.where_filter)
+            return self.data_source.query_get_empty_string_count(
+                table=self.dataset_name,
+                field=f'"{self.field_name}"' if isinstance(self.data_source, OracleDataSource) else self.field_name,
+                filters=self.where_filter if self.where_filter is not None else None,
+            )
+        elif isinstance(self.data_source, SearchIndexDataSource):
+            return self.data_source.query_get_empty_string_count(
+                index_name=self.dataset_name,
+                field=self.field_name,
+                filters=self.where_filter if self.where_filter else None,
+            )
+        else:
+            raise ValueError("Invalid data source type")
+
+
+class PercentageEmptyStringValidation(Validation):
+    def _generate_metric_value(self, **kwargs) -> Union[float, int]:
+        if isinstance(self.data_source, SQLDataSource):
+            if isinstance(self.data_source, OracleDataSource) and self.where_filter:
+                self.where_filter = re.sub(r"(\b[a-zA-Z_]+\b)(?=\s*[=<>])", r'"\1"', self.where_filter)
+            return self.data_source.query_get_empty_string_percentage(
+                table=self.dataset_name,
+                field=f'"{self.field_name}"' if isinstance(self.data_source, OracleDataSource) else self.field_name,
+                filters=self.where_filter if self.where_filter is not None else None,
+            )
+        elif isinstance(self.data_source, SearchIndexDataSource):
+            return self.data_source.query_get_empty_string_percentage(
+                index_name=self.dataset_name,
+                field=self.field_name,
+                filters=self.where_filter if self.where_filter else None,
+            )
+        else:
+            raise ValueError("Invalid data source type")
+
+
+class CountAllSpaceValidation(Validation):
+    def _generate_metric_value(self, **kwargs) -> Union[float, int]:
+        if isinstance(self.data_source, SQLDataSource):
+            return self.data_source.query_get_all_space_count(
+                table=self.dataset_name,
+                field=self.field_name,
+                operation="count",
+                filters=self.where_filter if self.where_filter is not None else None,
+            )
+        else:
+            raise ValueError("Invalid data source type")
+
+
+class PercentageAllSpaceValidation(Validation):
+    def _generate_metric_value(self, **kwargs) -> Union[float, int]:
+        if isinstance(self.data_source, SQLDataSource):
+            return self.data_source.query_get_all_space_count(
+                table=self.dataset_name,
+                field=self.field_name,
+                operation="percent",
+                filters=self.where_filter if self.where_filter is not None else None,
+            )
+        else:
+            raise ValueError("Invalid data source type")
+
+
+class CountNullKeywordValidation(Validation):
+    def _generate_metric_value(self, **kwargs) -> Union[float, int]:
+        if isinstance(self.data_source, SQLDataSource):
+            return self.data_source.query_get_null_keyword_count(
+                table=self.dataset_name,
+                field=self.field_name,
+                operation="count",
+                filters=self.where_filter if self.where_filter is not None else None,
+            )
+        else:
+            raise ValueError("Invalid data source type")
+
+
+class PercentageNullKeywordValidation(Validation):
+    def _generate_metric_value(self, **kwargs) -> Union[float, int]:
+        if isinstance(self.data_source, SQLDataSource):
+            return self.data_source.query_get_null_keyword_count(
+                table=self.dataset_name,
+                field=self.field_name,
+                operation="percent",
+                filters=self.where_filter if self.where_filter is not None else None,
+            )
+        else:
+            raise ValueError("Invalid data source type")

dcs_core/core/validation/custom_query_validation.py

@@ -0,0 +1,24 @@
+#  Copyright 2022-present, the Waterdip Labs Pvt. Ltd.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+from dcs_core.core.datasource.sql_datasource import SQLDataSource
+from dcs_core.core.validation.base import Validation
+
+
+class CustomSqlValidation(Validation):
+    def _generate_metric_value(self):
+        if isinstance(self.data_source, SQLDataSource):
+            return self.data_source.query_get_custom_sql(query=self.query)
+        else:
+            raise ValueError("Invalid data source type")

dcs_core/core/validation/manager.py

@@ -0,0 +1,282 @@
+#  Copyright 2022-present, the Waterdip Labs Pvt. Ltd.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+from typing import Dict
+
+from dcs_core.core.common.models.configuration import (
+    Configuration,
+    ValidationConfigByDataset,
+)
+from dcs_core.core.common.models.validation import ValidationFunction
+from dcs_core.core.datasource.manager import DataSourceManager
+from dcs_core.core.validation.base import DeltaValidation, Validation
+from dcs_core.core.validation.completeness_validation import (  # noqa F401 this is used in globals
+    CountAllSpaceValidation,
+    CountEmptyStringValidation,
+    CountNullKeywordValidation,
+    CountNullValidation,
+    PercentageAllSpaceValidation,
+    PercentageEmptyStringValidation,
+    PercentageNullKeywordValidation,
+    PercentageNullValidation,
+)
+from dcs_core.core.validation.custom_query_validation import (  # noqa F401 this is used in globals
+    CustomSqlValidation,
+)
+from dcs_core.core.validation.numeric_validation import (  # noqa F401 this is used in globals
+    AvgValidation,
+    CountNegativeValidation,
+    CountZeroValidation,
+    MaxValidation,
+    MinValidation,
+    Percentile20Validation,
+    Percentile40Validation,
+    Percentile60Validation,
+    Percentile80Validation,
+    Percentile90Validation,
+    PercentNegativeValidation,
+    PercentZeroValidation,
+    StdDevValidation,
+    SumValidation,
+    VarianceValidation,
+)
+from dcs_core.core.validation.reliability_validation import (  # noqa F401 this is used in globals
+    CountDocumentsValidation,
+    CountRowValidation,
+    DeltaCountRowValidation,
+    FreshnessValueMetric,
+)
+from dcs_core.core.validation.uniqueness_validation import (  # noqa F401 this is used in globals
+    CountDistinctValidation,
+    CountDuplicateValidation,
+)
+from dcs_core.core.validation.validity_validation import (  # noqa F401 this is used in globals
+    CountCUSIPValidation,
+    CountDateNotInFutureValidation,
+    CountEmailValidation,
+    CountFIGIValidation,
+    CountInvalidRegex,
+    CountInvalidValues,
+    CountISINValidation,
+    CountLatitudeValidation,
+    CountLEIValidation,
+    CountLongitudeValidation,
+    CountNotInFutureValidation,
+    CountPermIDValidation,
+    CountSEDOLValidation,
+    CountSSNValidation,
+    CountTimeStampValidation,
+    CountUSAPhoneValidation,
+    CountUSAStateCodeValidation,
+    CountUSAZipCodeValidation,
+    CountUUIDValidation,
+    CountValidRegex,
+    CountValidValues,
+    PercentCUSIPValidation,
+    PercentDateNotInFutureValidation,
+    PercentEmailValidation,
+    PercentFIGIValidation,
+    PercentInvalidRegex,
+    PercentInvalidValues,
+    PercentISINValidation,
+    PercentLatitudeValidation,
+    PercentLEIValidation,
+    PercentLongitudeValidation,
+    PercentNotInFutureValidation,
+    PercentPermIDValidation,
+    PercentSEDOLValidation,
+    PercentSSNValidation,
+    PercentTimeStampValidation,
+    PercentUSAPhoneValidation,
+    PercentUSAStateCodeValidation,
+    PercentUSAZipCodeValidation,
+    PercentUUIDValidation,
+    PercentValidRegex,
+    PercentValidValues,
+    StringLengthAverageValidation,
+    StringLengthMaxValidation,
+    StringLengthMinValidation,
+)
+
+
+class ValidationManager:
+    VALIDATION_CLASS_MAPPING = {
+        ValidationFunction.MIN.value: "MinValidation",
+        ValidationFunction.MAX.value: "MaxValidation",
+        ValidationFunction.AVG.value: "AvgValidation",
+        ValidationFunction.SUM.value: "SumValidation",
+        ValidationFunction.VARIANCE.value: "VarianceValidation",
+        ValidationFunction.STDDEV.value: "StdDevValidation",
+        ValidationFunction.COUNT_DUPLICATE.value: "CountDuplicateValidation",
+        ValidationFunction.COUNT_DISTINCT.value: "CountDistinctValidation",
+        ValidationFunction.COUNT_NULL.value: "CountNullValidation",
+        ValidationFunction.PERCENT_NULL.value: "PercentageNullValidation",
+        ValidationFunction.COUNT_EMPTY_STRING.value: "CountEmptyStringValidation",
+        ValidationFunction.PERCENT_EMPTY_STRING.value: "PercentageEmptyStringValidation",
+        ValidationFunction.CUSTOM_SQL.value: "CustomSqlValidation",
+        ValidationFunction.COUNT_DOCUMENTS.value: "CountDocumentsValidation",
+        ValidationFunction.COUNT_ROWS.value: "CountRowValidation",
+        ValidationFunction.DELTA_COUNT_ROWS.value: "DeltaCountRowValidation",
+        ValidationFunction.FRESHNESS.value: "FreshnessValueMetric",
+        ValidationFunction.COUNT_UUID.value: "CountUUIDValidation",
+        ValidationFunction.PERCENT_UUID.value: "PercentUUIDValidation",
+        ValidationFunction.COUNT_INVALID_VALUES.value: "CountInvalidValues",
+        ValidationFunction.PERCENT_INVALID_VALUES.value: "PercentInvalidValues",
+        ValidationFunction.COUNT_VALID_VALUES.value: "CountValidValues",
+        ValidationFunction.PERCENT_VALID_VALUES.value: "PercentValidValues",
+        ValidationFunction.COUNT_INVALID_REGEX.value: "CountInvalidRegex",
+        ValidationFunction.PERCENT_INVALID_REGEX.value: "PercentInvalidRegex",
+        ValidationFunction.COUNT_VALID_REGEX.value: "CountValidRegex",
+        ValidationFunction.PERCENT_VALID_REGEX.value: "PercentValidRegex",
+        ValidationFunction.COUNT_USA_PHONE.value: "CountUSAPhoneValidation",
+        ValidationFunction.PERCENT_USA_PHONE.value: "PercentUSAPhoneValidation",
+        ValidationFunction.COUNT_EMAIL.value: "CountEmailValidation",
+        ValidationFunction.PERCENT_EMAIL.value: "PercentEmailValidation",
+        ValidationFunction.STRING_LENGTH_MAX.value: "StringLengthMaxValidation",
+        ValidationFunction.STRING_LENGTH_MIN.value: "StringLengthMinValidation",
+        ValidationFunction.STRING_LENGTH_AVERAGE.value: "StringLengthAverageValidation",
+        ValidationFunction.COUNT_USA_STATE_CODE.value: "CountUSAStateCodeValidation",
+        ValidationFunction.PERCENT_USA_STATE_CODE.value: "PercentUSAStateCodeValidation",
+        ValidationFunction.COUNT_USA_ZIP_CODE.value: "CountUSAZipCodeValidation",
+        ValidationFunction.PERCENT_USA_ZIP_CODE.value: "PercentUSAZipCodeValidation",
+        ValidationFunction.COUNT_LATITUDE.value: "CountLatitudeValidation",
+        ValidationFunction.PERCENT_LATITUDE.value: "PercentLatitudeValidation",
+        ValidationFunction.COUNT_LONGITUDE.value: "CountLongitudeValidation",
+        ValidationFunction.PERCENT_LONGITUDE.value: "PercentLongitudeValidation",
+        ValidationFunction.COUNT_SSN.value: "CountSSNValidation",
+        ValidationFunction.PERCENT_SSN.value: "PercentSSNValidation",
+        ValidationFunction.COUNT_SEDOL.value: "CountSEDOLValidation",
+        ValidationFunction.PERCENT_SEDOL.value: "PercentSEDOLValidation",
+        ValidationFunction.COUNT_CUSIP.value: "CountCUSIPValidation",
+        ValidationFunction.PERCENT_CUSIP.value: "PercentCUSIPValidation",
+        ValidationFunction.COUNT_LEI.value: "CountLEIValidation",
+        ValidationFunction.PERCENT_LEI.value: "PercentLEIValidation",
+        ValidationFunction.COUNT_FIGI.value: "CountFIGIValidation",
+        ValidationFunction.PERCENT_FIGI.value: "PercentFIGIValidation",
+        ValidationFunction.COUNT_ISIN.value: "CountISINValidation",
+        ValidationFunction.PERCENT_ISIN.value: "PercentISINValidation",
+        ValidationFunction.COUNT_PERM_ID.value: "CountPermIDValidation",
+        ValidationFunction.PERCENT_PERM_ID.value: "PercentPermIDValidation",
+        ValidationFunction.PERCENTILE_20.value: "Percentile20Validation",
+        ValidationFunction.PERCENTILE_40.value: "Percentile40Validation",
+        ValidationFunction.PERCENTILE_60.value: "Percentile60Validation",
+        ValidationFunction.PERCENTILE_80.value: "Percentile80Validation",
+        ValidationFunction.PERCENTILE_90.value: "Percentile90Validation",
+        ValidationFunction.COUNT_ZERO.value: "CountZeroValidation",
+        ValidationFunction.PERCENT_ZERO.value: "PercentZeroValidation",
+        ValidationFunction.COUNT_NEGATIVE.value: "CountNegativeValidation",
+        ValidationFunction.PERCENT_NEGATIVE.value: "PercentNegativeValidation",
+        ValidationFunction.COUNT_ALL_SPACE.value: "CountAllSpaceValidation",
+        ValidationFunction.PERCENT_ALL_SPACE.value: "PercentageAllSpaceValidation",
+        ValidationFunction.COUNT_NULL_KEYWORD.value: "CountNullKeywordValidation",
+        ValidationFunction.PERCENT_NULL_KEYWORD.value: "PercentageNullKeywordValidation",
+        ValidationFunction.COUNT_TIMESTAMP_STRING.value: "CountTimeStampValidation",
+        ValidationFunction.PERCENT_TIMESTAMP_STRING.value: "PercentTimeStampValidation",
+        ValidationFunction.COUNT_NOT_IN_FUTURE.value: "CountNotInFutureValidation",
+        ValidationFunction.PERCENT_NOT_IN_FUTURE.value: "PercentNotInFutureValidation",
+        ValidationFunction.COUNT_DATE_NOT_IN_FUTURE.value: "CountDateNotInFutureValidation",
+        ValidationFunction.PERCENT_DATE_NOT_IN_FUTURE.value: "PercentDateNotInFutureValidation",
+    }
+
+    def __init__(
+        self,
+        application_configs: Configuration,
+        data_source_manager: DataSourceManager,
+    ):
+        self.data_source_manager = data_source_manager
+        self.application_configs = application_configs
+        self.validation_configs: Dict[str, ValidationConfigByDataset] = application_configs.validations
+
+        """
+        Will store the validations in the following format:
+        {
+            "data_source_name": {
+                "dataset_name": {
+                    "validation_name": Validation
+                }
+            }
+        }
+        """
+        self.validations: Dict[str, Dict[str, Dict[str, Validation]]] = {}
+
+    def set_validation_configs(self, validations: Dict[str, ValidationConfigByDataset]):
+        self.validation_configs = validations
+
+    def build_validations(self):
+        for _, validation_by_dataset in self.validation_configs.items():
+            data_source_name = validation_by_dataset.data_source
+            dataset_name = validation_by_dataset.dataset
+
+            if data_source_name not in self.validations:
+                self.validations[data_source_name] = {}
+
+            if dataset_name not in self.validations[data_source_name]:
+                self.validations[data_source_name][dataset_name] = {}
+
+            for (
+                validation_name,
+                validation_config,
+            ) in validation_by_dataset.validations.items():
+                data_source = self.data_source_manager.get_data_source(data_source_name)
+                params = {}
+                if validation_config.get_is_delta_validation:
+                    reference_data_source = self.data_source_manager.get_data_source(
+                        validation_config.get_ref_data_source_name
+                    )
+                    base_class_name = self.VALIDATION_CLASS_MAPPING[validation_config.get_validation_function]
+                    validation: DeltaValidation = globals()[base_class_name](
+                        name=validation_name,
+                        data_source=data_source,
+                        dataset_name=dataset_name,
+                        validation_name=validation_name,
+                        validation_config=validation_config,
+                        field_name=validation_config.get_validation_field_name,
+                        reference_data_source=reference_data_source,
+                        reference_dataset_name=validation_config.get_ref_dataset_name,
+                        reference_field_name=validation_config.get_ref_field_name,
+                        **params,
+                    )
+                    self.validations[data_source_name][dataset_name][validation_name] = validation
+                else:
+                    validation: Validation = globals()[
+                        self.VALIDATION_CLASS_MAPPING[validation_config.get_validation_function]
+                    ](
+                        name=validation_name,
+                        data_source=data_source,
+                        dataset_name=dataset_name,
+                        validation_name=validation_name,
+                        validation_config=validation_config,
+                        field_name=validation_config.get_validation_field_name,
+                        **params,
+                    )
+                    self.validations[data_source_name][dataset_name][validation_name] = validation
+
+    def add_validation(self, validation: Validation):
+        data_source_name = validation.data_source.data_source_name
+        dataset_name = validation.dataset_name
+        validation_name = validation.name
+        if data_source_name not in self.validations:
+            self.validations[data_source_name] = {}
+
+        if dataset_name not in self.validations[data_source_name]:
+            self.validations[data_source_name][dataset_name] = {}
+
+        self.validations[data_source_name][dataset_name][validation_name] = validation
+
+    @property
+    def get_validations(self):
+        return self.validations
+
+    def get_validation(self, data_source_name: str, dataset_name: str, validation_name: str) -> Validation:
+        return self.validations[data_source_name][dataset_name][validation_name]