dcs-sdk 1.6.5__py3-none-any.whl

dcs_core/core/inspect.py

@@ -0,0 +1,163 @@
+#  Copyright 2022-present, the Waterdip Labs Pvt. Ltd.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+import sys
+import traceback
+from dataclasses import dataclass
+from datetime import datetime
+from typing import Dict, List, Optional, Union
+
+import requests
+from loguru import logger
+
+from dcs_core.core.common.models.configuration import Configuration
+from dcs_core.core.common.models.metric import (
+    CombinedMetrics,
+    DataSourceMetrics,
+    MetricValue,
+)
+from dcs_core.core.common.models.validation import ValidationInfo
+from dcs_core.core.configuration.configuration_parser import (
+    load_configuration,
+    load_configuration_from_yaml_str,
+)
+from dcs_core.core.datasource.manager import DataSourceManager
+from dcs_core.core.utils.tracking import (
+    create_inspect_event_json,
+    is_tracking_enabled,
+    send_event_json,
+)
+from dcs_core.core.utils.utils import truncate_error
+from dcs_core.core.validation.manager import ValidationManager
+
+requests.packages.urllib3.disable_warnings(requests.packages.urllib3.exceptions.InsecureRequestWarning)
+
+
+@dataclass
+class InspectOutput:
+    validations: Dict[str, ValidationInfo]
+    metrics: Optional[Dict[str, Union[DataSourceMetrics, CombinedMetrics]]] = None
+
+    def get_metric_values(self) -> List[MetricValue]:
+        """
+        This method returns the list of metric values
+        """
+        metric_values: List[MetricValue] = []
+        for ds_met in self.metrics.values():
+            if isinstance(ds_met, DataSourceMetrics):
+                for table_met in ds_met.table_metrics.values():
+                    for metric in table_met.metrics.values():
+                        metric_values.append(metric)
+                for index_met in ds_met.index_metrics.values():
+                    for metric in index_met.metrics.values():
+                        metric_values.append(metric)
+            else:
+                for metric in ds_met.metrics.values():
+                    metric_values.append(metric)
+        return metric_values
+
+    def get_inspect_info(self):
+        metrics_count, datasource_count, combined_metrics_count = 0, 0, 0
+        table_count, index_count = 0, 0
+        # for ds_met in self.metrics.values():
+        #     if isinstance(ds_met, DataSourceMetrics):
+        #         datasource_count = datasource_count + 1
+        #         for table_met in ds_met.table_metrics.values():
+        #             table_count = table_count + 1
+        #             metrics_count = metrics_count + len(
+        #                 list(table_met.metrics.values())
+        #             )
+        #         for index_met in ds_met.index_metrics.values():
+        #             index_count = index_count + 1
+        #             metrics_count = metrics_count + len(
+        #                 list(index_met.metrics.values())
+        #             )
+        #     else:
+        #         metrics_count += 1
+        return {
+            "metrics_count": metrics_count,
+            "datasource_count": datasource_count,
+            "table_count": table_count,
+            "index_count": index_count,
+        }
+
+
+class Inspect:
+    def __init__(
+        self,
+        configuration: Optional[Configuration] = None,
+    ):
+        if configuration is None:
+            self.configuration = Configuration()
+        else:
+            self.configuration = configuration
+
+        self.data_source_manager = DataSourceManager(self.configuration)
+        self.validation_manager = ValidationManager(
+            application_configs=self.configuration,
+            data_source_manager=self.data_source_manager,
+        )
+
+        self.execution_time_taken = 0
+        self.is_storage_enabled = False
+
+    def add_configuration_yaml_file(self, file_path: str):
+        load_configuration(configuration_path=file_path, configuration=self.configuration)
+
+    def add_validations_yaml_str(self, yaml_str: str):
+        load_configuration_from_yaml_str(yaml_string=yaml_str, configuration=self.configuration)
+
+    def add_spark_session(self, spark_session, data_source_name: str = "spark_df"):
+        self.configuration.add_spark_session(data_source_name, spark_session)
+
+    def run(self) -> InspectOutput:
+        """
+        This method starts the inspection process.
+        """
+        start = datetime.now()
+        error = None
+        inspect_info = None
+        try:
+            self.data_source_manager.connect()
+            self.validation_manager.build_validations()
+
+            validation_infos: Dict[str, ValidationInfo] = {}
+
+            for datasource, _ in self.validation_manager.get_validations.items():
+                for dataset, _ in self.validation_manager.get_validations[datasource].items():
+                    for _, validation in self.validation_manager.get_validations[datasource][dataset].items():
+                        validation_info = validation.get_validation_info()
+                        validation_infos[validation.get_validation_identity()] = validation_info
+
+            output = InspectOutput(validations=validation_infos)
+            inspect_info = output.get_inspect_info()
+
+            return output
+        except Exception as ex:
+            logger.error(f"Error while running inspection: {ex}")
+            traceback.print_exc(file=sys.stdout)
+            error = ex
+        finally:
+            end = datetime.now()
+            self.execution_time_taken = round((end - start).total_seconds(), 3)
+            logger.info(f"Inspection took {self.execution_time_taken} seconds")
+            err_message = truncate_error(repr(error))
+            if is_tracking_enabled():
+                event_json = create_inspect_event_json(
+                    runtime_seconds=self.execution_time_taken,
+                    inspect_info=inspect_info,
+                    error=err_message,
+                )
+                send_event_json(event_json)
+            if error:
+                logger.error(error)

dcs_core/core/logger/__init__.py

@@ -0,0 +1,13 @@
+#  Copyright 2022-present, the Waterdip Labs Pvt. Ltd.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.

dcs_core/core/logger/base.py

@@ -0,0 +1,32 @@
+#  Copyright 2022-present, the Waterdip Labs Pvt. Ltd.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+from abc import ABC, abstractmethod
+from typing import Dict
+
+
+class MetricLogger(ABC):
+    def __init__(self):
+        pass
+
+    @abstractmethod
+    def log(self, metric_name: str, metric_value: float, metric_tags: Dict[str, str] = None):
+        """
+        Log a metric to the logger
+        :param metric_name:
+        :param metric_value:
+        :param metric_tags:
+        :return:
+        """
+        pass

dcs_core/core/logger/default_logger.py

@@ -0,0 +1,94 @@
+#  Copyright 2022-present, the Waterdip Labs Pvt. Ltd.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+import json
+import sys
+from typing import Dict
+
+from loguru import logger
+
+from dcs_core.core.logger.base import MetricLogger
+
+
+class DefaultLogger(MetricLogger):
+    def __init__(self, **kwargs):
+        super().__init__()
+        self.time_format = "%Y-%m-%dT%H:%M:%S.%fZ"
+        self.project_name = "datachecks"
+
+        if "project_name" in kwargs:
+            self.project_name = kwargs["project_name"]
+        if "time_format" in kwargs:
+            self.time_format = kwargs["time_format"]
+        # logger.remove()
+        logger.add(
+            self._loguru_sink_serializer,
+            level="INFO",
+            enqueue=True,
+            serialize=True,
+            filter="datachecks.core.logger.default_logger",
+        )
+
+    def _loguru_sink_serializer(self, message):
+        record = message.record
+
+        simplified = {
+            "@timestamp": f"{record['time'].strftime(self.time_format)}",
+            "level": record["level"].name,
+            "message": record["message"],
+            "logger_name": record["name"],
+        }
+        if self.time_format.endswith("%fZ"):
+            simplified["@timestamp"] = f"{record['time'].strftime('%Y-%m-%dT%H:%M:%S.%fZ')[:-4]}Z"
+
+        if self.project_name is not None:
+            simplified["projectName"] = self.project_name
+
+        if record["extra"].get("extra"):
+            if "metric_name" in record["extra"]["extra"]:
+                simplified["metric_name"] = record["extra"]["extra"]["metric_name"]
+            if "metric_value" in record["extra"]["extra"]:
+                simplified["metric_value"] = record["extra"]["extra"]["metric_value"]
+            if "datasource_name" in record["extra"]["extra"]:
+                simplified["datasource_name"] = record["extra"]["extra"]["datasource_name"]
+            if "metric_type" in record["extra"]["extra"]:
+                simplified["metric_type"] = record["extra"]["extra"]["metric_type"]
+            if "identity" in record["extra"]["extra"]:
+                simplified["identity"] = record["extra"]["extra"]["identity"]
+            if "index_name" in record["extra"]["extra"]:
+                simplified["index_name"] = record["extra"]["extra"]["index_name"]
+            if "table_name" in record["extra"]["extra"]:
+                simplified["table_name"] = record["extra"]["extra"]["table_name"]
+            if "field_name" in record["extra"]["extra"]:
+                simplified["field_name"] = record["extra"]["extra"]["field_name"]
+
+        serialized = json.dumps(simplified)
+        print(serialized, file=sys.stdout)
+
+    def log(self, metric_name: str, metric_value: float, metric_tags: Dict[str, str] = None):
+        logger_extra_value = {
+            "metric_value": metric_value,
+            "metric_name": metric_name,
+            "datasource_name": metric_tags["dataSourceName"],
+            "metric_type": metric_tags["metricType"],
+            "identity": metric_tags["identity"],
+        }
+        if "index_name" in metric_tags and metric_tags["index_name"] is not None:
+            logger_extra_value["index_name"] = metric_tags["index_name"]
+        elif "table_name" in metric_tags and metric_tags["table_name"] is not None:
+            logger_extra_value["table_name"] = metric_tags["table_name"]
+        if "field_name" in metric_tags and metric_tags["field_name"] is not None:
+            logger_extra_value["field_name"] = metric_tags["field_name"]
+
+        logger.info("Logging metric value", extra={**logger_extra_value})

dcs_core/core/metric/__init__.py

@@ -0,0 +1,13 @@
+#  Copyright 2022-present, the Waterdip Labs Pvt. Ltd.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.

dcs_core/core/metric/base.py

@@ -0,0 +1,220 @@
+#  Copyright 2022-present, the Waterdip Labs Pvt. Ltd.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+import datetime
+import json
+from abc import ABC
+from typing import Optional, Tuple, Union
+
+from loguru import logger
+
+from dcs_core.core.common.models.metric import MetricsType, MetricValue
+from dcs_core.core.common.models.validation import ConditionType
+from dcs_core.core.datasource.base import DataSource
+from dcs_core.core.datasource.search_datasource import SearchIndexDataSource
+from dcs_core.core.datasource.sql_datasource import SQLDataSource
+
+
+class MetricIdentity:
+    @staticmethod
+    def generate_identity(
+        metric_type: MetricsType,
+        metric_name: str,
+        data_source: DataSource = None,
+        expression: str = None,
+        index_name: str = None,
+        table_name: str = None,
+        field_name: str = None,
+    ) -> str:
+        """
+        Generate a unique identifier for a metric
+        """
+
+        identifiers = []
+
+        if data_source is not None:
+            identifiers.append(data_source.data_source_name)
+        if index_name:
+            identifiers.append(index_name)
+        elif table_name:
+            identifiers.append(table_name)
+        if field_name:
+            identifiers.append(field_name)
+
+        identifiers.append(metric_type.value)
+        if metric_name:
+            identifiers.append(metric_name)
+        return ".".join([str(p) for p in identifiers])
+
+
+class Metric(ABC):
+    """
+    Metric is a class that represents a metric that is generated by a data source.
+    """
+
+    def __init__(
+        self,
+        name: str,
+        metric_type: MetricsType,
+        data_source: DataSource = None,
+        expression: str = None,
+        **kwargs,
+    ):
+        if metric_type == MetricsType.COMBINED:
+            if expression is None:
+                raise ValueError("Please give an expression for combined metric")
+        else:
+            if "index_name" in kwargs and "table_name" in kwargs:
+                if kwargs["index_name"] is not None and kwargs["table_name"] is not None:
+                    raise ValueError("Please give a value for table_name or index_name (but not both)")
+            if "index_name" not in kwargs and "table_name" not in kwargs:
+                raise ValueError("Please give a value for table_name or index_name")
+
+            self.index_name, self.table_name = None, None
+            if "index_name" in kwargs:
+                self.index_name = kwargs["index_name"]
+            if "table_name" in kwargs:
+                self.table_name = kwargs["table_name"]
+
+        self.name: str = name
+        self.metric_type = metric_type
+        self.data_source = data_source
+        self.expression = expression
+        self.filter_query = None
+
+        if "query" in kwargs and kwargs["query"] is not None:
+            self.custom_sql_query = kwargs["query"]
+
+        if "filters" in kwargs and kwargs["filters"] is not None:
+            filters = kwargs["filters"]
+            if "where" in filters and filters["where"] is not None:
+                if isinstance(data_source, SearchIndexDataSource):
+                    self.filter_query = json.loads(filters["where"])
+                elif isinstance(data_source, SQLDataSource):
+                    self.filter_query = filters["where"]
+
+        self.validation = None
+        if "validation" in kwargs and kwargs["validation"] is not None:
+            self.validation = kwargs["validation"]
+
+    def get_metric_identity(self) -> str:
+        return MetricIdentity.generate_identity(
+            metric_type=self.metric_type,
+            metric_name=self.name,
+            data_source=self.data_source,
+            expression=self.expression,
+        )
+
+    def _generate_metric_value(self, **kwargs) -> Union[float, int]:
+        pass
+
+    def get_metric_value(self, **kwargs) -> Union[MetricValue, None]:
+        try:
+            metric_value = self._generate_metric_value(**kwargs)
+            tags = {
+                "metric_name": self.name,
+            }
+            if self.metric_type.value == MetricsType.COMBINED.value:
+                value = MetricValue(
+                    identity=self.get_metric_identity(),
+                    metric_type=self.metric_type.value,
+                    value=metric_value,
+                    expression=self.expression,
+                    timestamp=datetime.datetime.utcnow(),
+                    tags=tags,
+                )
+            else:
+                value = MetricValue(
+                    identity=self.get_metric_identity(),
+                    metric_type=self.metric_type.value,
+                    value=metric_value,
+                    timestamp=datetime.datetime.utcnow(),
+                    data_source=self.data_source.data_source_name,
+                    expression=self.expression,
+                    tags=tags,
+                )
+            if self.validation is not None and self.validation.threshold is not None:
+                value.is_valid, value.reason = self.validate_metric(metric_value)
+
+            if "index_name" in self.__dict__ and self.__dict__["index_name"] is not None:
+                value.index_name = self.__dict__["index_name"]
+            elif "table_name" in self.__dict__ and self.__dict__["table_name"] is not None:
+                value.table_name = self.__dict__["table_name"]
+
+            if "field_name" in self.__dict__ and self.__dict__["field_name"] is not None:
+                value.field_name = self.__dict__["field_name"]
+
+            return value
+        except Exception as e:
+            logger.error(f"Failed to generate metric {self.name}: {str(e)}")
+            return None
+
+    def validate_metric(self, metric_value) -> Tuple[bool, Optional[str]]:
+        for operator, value in self.validation.threshold.__dict__.items():
+            if value is not None:
+                if ConditionType.GTE == operator:
+                    if metric_value < value:
+                        return (
+                            False,
+                            f"Less than threshold of {value}",
+                        )
+                elif ConditionType.LTE == operator:
+                    if metric_value > value:
+                        return (
+                            False,
+                            f"Greater than threshold of {value}",
+                        )
+                elif ConditionType.GT == operator:
+                    if metric_value <= value:
+                        return (
+                            False,
+                            f"Less than or equal to threshold of {value}",
+                        )
+                elif ConditionType.LT == operator:
+                    if metric_value >= value:
+                        return (
+                            False,
+                            f"Greater than or equal to threshold of {value}",
+                        )
+                elif ConditionType.EQ == operator:
+                    if metric_value != value:
+                        return (
+                            False,
+                            f"Not equal to {value}",
+                        )
+        return True, None
+
+
+class FieldMetrics(Metric, ABC):
+    def __init__(
+        self,
+        name: str,
+        metric_type: MetricsType,
+        data_source: Optional[DataSource] = None,
+        expression: Optional[str] = None,
+        **kwargs,
+    ):
+        super().__init__(
+            name=name,
+            data_source=data_source,
+            expression=expression,
+            metric_type=metric_type,
+            **kwargs,
+        )
+        if "field_name" in kwargs:
+            self.field_name = kwargs["field_name"]
+
+    @property
+    def get_field_name(self):
+        return self.field_name

dcs_core/core/metric/combined_metric.py

@@ -0,0 +1,98 @@
+#  Copyright 2022-present, the Waterdip Labs Pvt. Ltd.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+import datetime
+from typing import Dict, List, Union
+
+from loguru import logger
+from pyparsing import Forward, Group, Suppress, Word, alphas, delimitedList, nums
+
+from dcs_core.core.common.models.metric import MetricsType, MetricValue
+from dcs_core.core.metric.base import Metric, MetricIdentity
+
+
+class CombinedMetric(Metric):
+    """
+    CombinedMetric is a class that represents a metric test is generated by a data source.
+    """
+
+    def _find_metric_values(self, metric_arg: any, metric_values: List[MetricValue]) -> float:
+        """
+        Find the metric values for the metric identities
+        """
+        if metric_arg.isnumeric():
+            return float(metric_arg)
+        elif isinstance(metric_arg, str):
+            for metric_value in metric_values:
+                if metric_value.tags["metric_name"] == metric_arg:
+                    return metric_value.value
+            raise ValueError(f"Metric {metric_arg} not found in a {self.expression}")
+
+    def _metric_expression_parser(self, expression_str: str, metric_values: List[MetricValue]) -> Dict:
+        try:
+            fwd_expr = Forward()
+            operation = Word(alphas)
+            arguments = Word("_" + nums + alphas)
+            lp, rp = map(Suppress, "()")
+            reg_expression = operation + lp + delimitedList(fwd_expr) + rp
+            fwd_expr << (reg_expression | arguments)
+            fwd_expr.setParseAction(
+                lambda tokens: (
+                    self._find_metric_values(tokens[0], metric_values)
+                    if len(tokens) == 1
+                    else {"operation": tokens[0], "args": tokens[1:]}
+                )
+            )
+            return fwd_expr.parseString(expression_str, parseAll=True)[0]
+        except Exception as e:
+            raise ValueError(f"Invalid expression {expression_str}", e)
+
+    def _perform_operation(self, operation_data):
+        """
+        Perform the operation specified in the operation_data
+        """
+        operation = operation_data["operation"]
+        args = operation_data["args"]
+        if len(args) > 2:
+            raise ValueError("Operation must have only two arguments")
+            return
+        if isinstance(args[0], dict):
+            args[0] = self._perform_operation(args[0])
+
+        if isinstance(args[1], dict):
+            args[1] = self._perform_operation(args[1])
+
+        if operation == "sum":
+            return args[0] + args[1]
+        elif operation == "sub":
+            return args[0] - args[1]
+        elif operation == "mul":
+            return args[0] * args[1]
+        elif operation == "div":
+            return args[0] / args[1]
+        else:
+            raise ValueError("Invalid operation")
+
+    def get_metric_identity(self):
+        return MetricIdentity.generate_identity(
+            metric_type=MetricsType.COMBINED,
+            metric_name=self.name,
+            expression=self.expression,
+        )
+
+    def _generate_metric_value(self, metric_values: List[MetricValue]):
+        """
+        Generate the metric value for this metric
+        """
+        expression_data = self._metric_expression_parser(self.expression, metric_values)
+        return round(self._perform_operation(expression_data), 2)

dcs_core/core/metric/custom_metric.py

@@ -0,0 +1,34 @@
+#  Copyright 2022-present, the Waterdip Labs Pvt. Ltd.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+from dcs_core.core.common.models.metric import MetricsType
+from dcs_core.core.datasource.sql_datasource import SQLDataSource
+from dcs_core.core.metric.base import Metric, MetricIdentity
+
+
+class CustomSqlMetric(Metric):
+    def get_metric_identity(self):
+        return MetricIdentity.generate_identity(
+            metric_type=MetricsType.CUSTOM_SQL,
+            metric_name=self.name,
+            data_source=self.data_source,
+            table_name=self.table_name,
+        )
+
+    def _generate_metric_value(self):
+        if isinstance(self.data_source, SQLDataSource):
+            return self.data_source.query_get_custom_sql(query=self.custom_sql_query)
+
+        else:
+            raise ValueError("Invalid data source type")