dcs-sdk 1.6.4__py3-none-any.whl → 1.6.6__py3-none-any.whl

dcs_core/core/metric/manager.py

@@ -0,0 +1,137 @@
+#  Copyright 2022-present, the Waterdip Labs Pvt. Ltd.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+from typing import Dict
+
+from dcs_core.core.common.models.configuration import MetricConfiguration
+from dcs_core.core.common.models.data_source_resource import Field, Index, Table
+from dcs_core.core.common.models.metric import MetricsType
+from dcs_core.core.datasource.manager import DataSourceManager
+from dcs_core.core.metric.base import Metric
+from dcs_core.core.metric.combined_metric import CombinedMetric
+from dcs_core.core.metric.custom_metric import (  # noqa F401 this is used in globals
+    CustomSqlMetric,
+)
+from dcs_core.core.metric.numeric_metric import (  # noqa F401 this is used in globals
+    AvgMetric,
+    DistinctCountMetric,
+    DuplicateCountMetric,
+    EmptyStringCountMetric,
+    EmptyStringPercentageMetric,
+    MaxMetric,
+    MinMetric,
+    NullCountMetric,
+    NullPercentageMetric,
+    StddevMetric,
+    SumMetric,
+    VarianceMetric,
+)
+from dcs_core.core.metric.reliability_metric import (  # noqa F401 this is used in globals
+    DocumentCountMetric,
+    FreshnessValueMetric,
+    RowCountMetric,
+)
+
+
+class MetricManager:
+    METRIC_CLASS_MAPPING = {
+        MetricsType.DOCUMENT_COUNT.value: "DocumentCountMetric",
+        MetricsType.ROW_COUNT.value: "RowCountMetric",
+        MetricsType.FRESHNESS.value: "FreshnessValueMetric",
+        MetricsType.MAX.value: "MaxMetric",
+        MetricsType.MIN.value: "MinMetric",
+        MetricsType.AVG.value: "AvgMetric",
+        MetricsType.SUM.value: "SumMetric",
+        MetricsType.STDDEV.value: "StddevMetric",
+        MetricsType.VARIANCE.value: "VarianceMetric",
+        MetricsType.COMBINED.value: "CombinedMetric",
+        MetricsType.DUPLICATE_COUNT.value: "DuplicateCountMetric",
+        MetricsType.NULL_COUNT.value: "NullCountMetric",
+        MetricsType.DISTINCT_COUNT.value: "DistinctCountMetric",
+        MetricsType.NULL_PERCENTAGE.value: "NullPercentageMetric",
+        MetricsType.EMPTY_STRING_COUNT.value: "EmptyStringCountMetric",
+        MetricsType.EMPTY_STRING_PERCENTAGE.value: "EmptyStringPercentageMetric",
+        MetricsType.CUSTOM_SQL.value: "CustomSqlMetric",
+    }
+
+    def __init__(
+        self,
+        metric_config: Dict[str, MetricConfiguration],
+        data_source_manager: DataSourceManager,
+    ):
+        self.data_source_manager = data_source_manager
+        self.metrics: Dict[str, Metric] = {}
+        self.combined: Dict[str, Metric] = {}
+        if metric_config:
+            self._build_metrics(
+                config={k: v for (k, v) in metric_config.items() if v.metric_type != MetricsType.COMBINED.value}
+            )
+            self._build_combined_metrics(
+                config={k: v for (k, v) in metric_config.items() if v.metric_type == MetricsType.COMBINED.value}
+            )
+
+    def _build_metrics(self, config: Dict[str, MetricConfiguration]):
+        for metric_name, metric_config in config.items():
+            if isinstance(metric_config.resource, Field):
+                data_source = metric_config.resource.belongs_to.data_source
+            else:
+                data_source = metric_config.resource.data_source
+            params = {
+                "filters": metric_config.filters if metric_config.filters is not None else None,
+                "validation": metric_config.validation if metric_config.validation is not None else None,
+                "query": metric_config.query if metric_config.query is not None else None,
+            }
+            if isinstance(metric_config.resource, Index):
+                params["index_name"] = metric_config.resource.name
+            if isinstance(metric_config.resource, Table):
+                params["table_name"] = metric_config.resource.name
+            if isinstance(metric_config.resource, Field):
+                params["field_name"] = metric_config.resource.name
+                if isinstance(metric_config.resource.belongs_to, Table):
+                    params["table_name"] = metric_config.resource.belongs_to.name
+                elif isinstance(metric_config.resource.belongs_to, Index):
+                    params["index_name"] = metric_config.resource.belongs_to.name
+
+            metric: Metric = globals()[self.METRIC_CLASS_MAPPING[metric_config.metric_type]](
+                name=metric_config.name,
+                metric_type=MetricsType(metric_config.metric_type.lower()),
+                data_source=self.data_source_manager.get_data_source(data_source),
+                **params,
+            )
+
+            self.metrics[metric.get_metric_identity()] = metric
+
+    def add_metric(self, metric: Metric):
+        self.metrics[metric.get_metric_identity()] = metric
+
+    def _build_combined_metrics(self, config: Dict[str, MetricConfiguration]):
+        for metric_name, metric_config in config.items():
+            params = {
+                "filters": metric_config.filters if metric_config.filters else None,
+                "validation": metric_config.validation if metric_config.validation is not None else None,
+            }
+            metric: Metric = CombinedMetric(
+                name=metric_config.name,
+                metric_type=MetricsType(metric_config.metric_type.lower()),
+                expression=metric_config.expression,
+                **params,
+            )
+            self.combined[metric.get_metric_identity()] = metric
+
+    @property
+    def get_metrics(self):
+        return self.metrics
+
+    def get_metric(self, metric_identity: str):
+        return self.metrics.get(metric_identity, None)

dcs_core/core/metric/numeric_metric.py

@@ -0,0 +1,403 @@
+#  Copyright 2022-present, the Waterdip Labs Pvt. Ltd.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+
+from dcs_core.core.common.models.metric import MetricsType
+from dcs_core.core.datasource.search_datasource import SearchIndexDataSource
+from dcs_core.core.datasource.sql_datasource import SQLDataSource
+from dcs_core.core.metric.base import FieldMetrics, MetricIdentity
+
+
+class MinMetric(FieldMetrics):
+    """
+    MinMetric is a class that represents a metric test is generated by a data source.
+    """
+
+    def get_metric_identity(self):
+        return MetricIdentity.generate_identity(
+            metric_type=MetricsType.MIN,
+            metric_name=self.name,
+            data_source=self.data_source,
+            field_name=self.field_name,
+            table_name=self.table_name if self.table_name else None,
+            index_name=self.index_name if self.index_name else None,
+        )
+
+    def _generate_metric_value(self):
+        if isinstance(self.data_source, SQLDataSource):
+            return self.data_source.query_get_min(
+                table=self.table_name,
+                field=self.field_name,
+                filters=self.filter_query if self.filter_query else None,
+            )
+        elif isinstance(self.data_source, SearchIndexDataSource):
+            return self.data_source.query_get_min(
+                index_name=self.index_name,
+                field=self.field_name,
+                filters=self.filter_query if self.filter_query else None,
+            )
+        else:
+            raise ValueError("Invalid data source type")
+
+
+class MaxMetric(FieldMetrics):
+    """
+    MaxMetric is a class that represents a metric that is generated by a data source.
+    """
+
+    def get_metric_identity(self):
+        return MetricIdentity.generate_identity(
+            metric_type=MetricsType.MAX,
+            metric_name=self.name,
+            data_source=self.data_source,
+            field_name=self.field_name,
+            table_name=self.table_name if self.table_name else None,
+            index_name=self.index_name if self.index_name else None,
+        )
+
+    def _generate_metric_value(self):
+        if isinstance(self.data_source, SQLDataSource):
+            return self.data_source.query_get_max(
+                table=self.table_name,
+                field=self.field_name,
+                filters=self.filter_query if self.filter_query else None,
+            )
+        elif isinstance(self.data_source, SearchIndexDataSource):
+            return self.data_source.query_get_max(
+                index_name=self.index_name,
+                field=self.field_name,
+                filters=self.filter_query if self.filter_query else None,
+            )
+        else:
+            raise ValueError("Invalid data source type")
+
+
+class AvgMetric(FieldMetrics):
+    """
+    AvgMetric is a class that represents a metric that is generated by a data source.
+    """
+
+    def get_metric_identity(self):
+        return MetricIdentity.generate_identity(
+            metric_type=MetricsType.AVG,
+            metric_name=self.name,
+            data_source=self.data_source,
+            field_name=self.field_name,
+            table_name=self.table_name if self.table_name else None,
+            index_name=self.index_name if self.index_name else None,
+        )
+
+    def _generate_metric_value(self):
+        if isinstance(self.data_source, SQLDataSource):
+            return self.data_source.query_get_avg(
+                table=self.table_name,
+                field=self.field_name,
+                filters=self.filter_query if self.filter_query else None,
+            )
+        elif isinstance(self.data_source, SearchIndexDataSource):
+            return self.data_source.query_get_avg(
+                index_name=self.index_name,
+                field=self.field_name,
+                filters=self.filter_query if self.filter_query else None,
+            )
+        else:
+            raise ValueError("Invalid data source type")
+
+
+class SumMetric(FieldMetrics):
+    """
+    SumMetric is a class that represents a metric that is generated by a data source.
+    """
+
+    def get_metric_identity(self):
+        return MetricIdentity.generate_identity(
+            metric_type=MetricsType.SUM,
+            metric_name=self.name,
+            data_source=self.data_source,
+            field_name=self.field_name,
+            table_name=self.table_name if self.table_name else None,
+            index_name=self.index_name if self.index_name else None,
+        )
+
+    def _generate_metric_value(self):
+        if isinstance(self.data_source, SQLDataSource):
+            return self.data_source.query_get_sum(
+                table=self.table_name,
+                field=self.field_name,
+                filters=self.filter_query if self.filter_query else None,
+            )
+        elif isinstance(self.data_source, SearchIndexDataSource):
+            return self.data_source.query_get_sum(
+                index_name=self.index_name,
+                field=self.field_name,
+                filters=self.filter_query if self.filter_query else None,
+            )
+        else:
+            raise ValueError("Invalid data source type")
+
+
+class VarianceMetric(FieldMetrics):
+    """
+    VarianceMetric is a class that represents a metric test is generated by a data source.
+    """
+
+    def get_metric_identity(self):
+        return MetricIdentity.generate_identity(
+            metric_type=MetricsType.VARIANCE,
+            metric_name=self.name,
+            data_source=self.data_source,
+            field_name=self.field_name,
+            table_name=self.table_name if self.table_name else None,
+            index_name=self.index_name if self.index_name else None,
+        )
+
+    def _generate_metric_value(self):
+        if isinstance(self.data_source, SQLDataSource):
+            return self.data_source.query_get_variance(
+                table=self.table_name,
+                field=self.field_name,
+                filters=self.filter_query if self.filter_query else None,
+            )
+        elif isinstance(self.data_source, SearchIndexDataSource):
+            return self.data_source.query_get_variance(
+                index_name=self.index_name,
+                field=self.field_name,
+                filters=self.filter_query if self.filter_query else None,
+            )
+        else:
+            raise ValueError("Invalid data source type")
+
+
+class StddevMetric(FieldMetrics):
+    """
+    StddevMetric is a class that represents a metric test generated by a data source.
+    """
+
+    def get_metric_identity(self):
+        return MetricIdentity.generate_identity(
+            metric_type=MetricsType.STDDEV,
+            metric_name=self.name,
+            data_source=self.data_source,
+            field_name=self.field_name,
+            table_name=self.table_name if self.table_name else None,
+            index_name=self.index_name if self.index_name else None,
+        )
+
+    def _generate_metric_value(self):
+        if isinstance(self.data_source, SQLDataSource):
+            return self.data_source.query_get_stddev(
+                table=self.table_name,
+                field=self.field_name,
+                filters=self.filter_query if self.filter_query else None,
+            )
+        elif isinstance(self.data_source, SearchIndexDataSource):
+            return self.data_source.query_get_stddev(
+                index_name=self.index_name,
+                field=self.field_name,
+                filters=self.filter_query if self.filter_query else None,
+            )
+        else:
+            raise ValueError("Invalid data source type")
+
+
+class DuplicateCountMetric(FieldMetrics):
+    """
+    DuplicateCountMetric is a class that represents a metric test is generated by a data source.
+    """
+
+    def get_metric_identity(self):
+        return MetricIdentity.generate_identity(
+            metric_type=MetricsType.DUPLICATE_COUNT,
+            metric_name=self.name,
+            data_source=self.data_source,
+            field_name=self.field_name,
+            table_name=self.table_name if self.table_name else None,
+            index_name=self.index_name if self.index_name else None,
+        )
+
+    def _generate_metric_value(self):
+        if isinstance(self.data_source, SQLDataSource):
+            return self.data_source.query_get_duplicate_count(
+                table=self.table_name,
+                field=self.field_name,
+                filters=self.filter_query if self.filter_query else None,
+            )
+        elif isinstance(self.data_source, SearchIndexDataSource):
+            return self.data_source.query_get_duplicate_count(
+                index_name=self.index_name,
+                field=self.field_name,
+                filters=self.filter_query if self.filter_query else None,
+            )
+        else:
+            raise ValueError("Invalid data source type")
+
+
+class NullCountMetric(FieldMetrics):
+    """
+    NullCountMetric is a class that represents a metric test is generated by a data source.
+    """
+
+    def get_metric_identity(self):
+        return MetricIdentity.generate_identity(
+            metric_type=MetricsType.NULL_COUNT,
+            metric_name=self.name,
+            data_source=self.data_source,
+            field_name=self.field_name,
+            table_name=self.table_name if self.table_name else None,
+            index_name=self.index_name if self.index_name else None,
+        )
+
+    def _generate_metric_value(self):
+        if isinstance(self.data_source, SQLDataSource):
+            return self.data_source.query_get_null_count(
+                table=self.table_name,
+                field=self.field_name,
+                filters=self.filter_query if self.filter_query else None,
+            )
+        elif isinstance(self.data_source, SearchIndexDataSource):
+            return self.data_source.query_get_null_count(
+                index_name=self.index_name,
+                field=self.field_name,
+                filters=self.filter_query if self.filter_query else None,
+            )
+        else:
+            raise ValueError("Invalid data source type")
+
+
+class NullPercentageMetric(FieldMetrics):
+    """
+    NullPercentageMetric is a class that represents a metric test is generated by a data source.
+    """
+
+    def get_metric_identity(self):
+        return MetricIdentity.generate_identity(
+            metric_type=MetricsType.NULL_PERCENTAGE,
+            metric_name=self.name,
+            data_source=self.data_source,
+            field_name=self.field_name,
+            table_name=self.table_name if self.table_name else None,
+            index_name=self.index_name if self.index_name else None,
+        )
+
+    def _generate_metric_value(self):
+        if isinstance(self.data_source, SQLDataSource):
+            return self.data_source.query_get_null_percentage(
+                table=self.table_name,
+                field=self.field_name,
+                filters=self.filter_query if self.filter_query else None,
+            )
+        elif isinstance(self.data_source, SearchIndexDataSource):
+            return self.data_source.query_get_null_percentage(
+                index_name=self.index_name,
+                field=self.field_name,
+                filters=self.filter_query if self.filter_query else None,
+            )
+        else:
+            raise ValueError("Invalid data source type")
+
+
+class DistinctCountMetric(FieldMetrics):
+    """
+    DistinctCountMetric is a class that represents a metric test generated by a data source.
+    """
+
+    def get_metric_identity(self):
+        return MetricIdentity.generate_identity(
+            metric_type=MetricsType.DISTINCT_COUNT,
+            metric_name=self.name,
+            data_source=self.data_source,
+            field_name=self.field_name,
+            table_name=self.table_name if self.table_name else None,
+            index_name=self.index_name if self.index_name else None,
+        )
+
+    def _generate_metric_value(self):
+        if isinstance(self.data_source, SQLDataSource):
+            return self.data_source.query_get_distinct_count(
+                table=self.table_name,
+                field=self.field_name,
+                filters=self.filter_query if self.filter_query else None,
+            )
+        elif isinstance(self.data_source, SearchIndexDataSource):
+            return self.data_source.query_get_distinct_count(
+                index_name=self.index_name,
+                field=self.field_name,
+                filters=self.filter_query if self.filter_query else None,
+            )
+        else:
+            raise ValueError("Invalid data source type")
+
+
+class EmptyStringCountMetric(FieldMetrics):
+    """
+    EmptyStringCountMetric is a class that represents a metric for counting empty strings in a data source.
+    """
+
+    def get_metric_identity(self):
+        return MetricIdentity.generate_identity(
+            metric_type=MetricsType.EMPTY_STRING_COUNT,
+            metric_name=self.name,
+            data_source=self.data_source,
+            field_name=self.field_name,
+            table_name=self.table_name if self.table_name else None,
+            index_name=self.index_name if self.index_name else None,
+        )
+
+    def _generate_metric_value(self):
+        if isinstance(self.data_source, SQLDataSource):
+            return self.data_source.query_get_empty_string_count(
+                table=self.table_name,
+                field=self.field_name,
+                filters=self.filter_query if self.filter_query else None,
+            )
+        elif isinstance(self.data_source, SearchIndexDataSource):
+            return self.data_source.query_get_empty_string_count(
+                index_name=self.index_name,
+                field=self.field_name,
+                filters=self.filter_query if self.filter_query else None,
+            )
+        else:
+            raise ValueError("Invalid data source type")
+
+
+class EmptyStringPercentageMetric(FieldMetrics):
+    """
+    EmptyStringPercentageMetric is a class that represents a metric for counting empty strings in a data source.
+    """
+
+    def get_metric_identity(self):
+        return MetricIdentity.generate_identity(
+            metric_type=MetricsType.EMPTY_STRING_PERCENTAGE,
+            metric_name=self.name,
+            data_source=self.data_source,
+            field_name=self.field_name,
+            table_name=self.table_name,
+            index_name=self.index_name,
+        )
+
+    def _generate_metric_value(self):
+        if isinstance(self.data_source, SQLDataSource):
+            return self.data_source.query_get_empty_string_percentage(
+                table=self.table_name,
+                field=self.field_name,
+                filters=self.filter_query if self.filter_query else None,
+            )
+        elif isinstance(self.data_source, SearchIndexDataSource):
+            return self.data_source.query_get_empty_string_percentage(
+                index_name=self.index_name,
+                field=self.field_name,
+                filters=self.filter_query if self.filter_query else None,
+            )
+        else:
+            raise ValueError("Invalid data source type")

dcs_core/core/metric/reliability_metric.py

@@ -0,0 +1,90 @@
+#  Copyright 2022-present, the Waterdip Labs Pvt. Ltd.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+from dcs_core.core.common.models.metric import MetricsType
+from dcs_core.core.datasource.search_datasource import SearchIndexDataSource
+from dcs_core.core.datasource.sql_datasource import SQLDataSource
+from dcs_core.core.metric.base import FieldMetrics, Metric, MetricIdentity
+
+
+class DocumentCountMetric(Metric):
+    """
+    DocumentCountMetrics is a class that represents a metric that is generated by a data source.
+    """
+
+    def validate_data_source(self):
+        return isinstance(self.data_source, SearchIndexDataSource)
+
+    def get_metric_identity(self):
+        return MetricIdentity.generate_identity(
+            metric_type=MetricsType.DOCUMENT_COUNT,
+            metric_name=self.name,
+            data_source=self.data_source,
+            index_name=self.index_name,
+        )
+
+    def _generate_metric_value(self):
+        if isinstance(self.data_source, SearchIndexDataSource):
+            return self.data_source.query_get_document_count(
+                index_name=self.index_name,
+                filters=self.filter_query if self.filter_query else None,
+            )
+        else:
+            raise ValueError("Invalid data source type")
+
+
+class RowCountMetric(Metric):
+    """
+    RowCountMetrics is a class that represents a metric that is generated by a data source.
+    """
+
+    def get_metric_identity(self):
+        return MetricIdentity.generate_identity(
+            metric_type=MetricsType.ROW_COUNT,
+            metric_name=self.name,
+            data_source=self.data_source,
+            table_name=self.table_name,
+        )
+
+    def _generate_metric_value(self):
+        if isinstance(self.data_source, SQLDataSource):
+            return self.data_source.query_get_row_count(
+                table=self.table_name,
+                filters=self.filter_query if self.filter_query else None,
+            )
+        else:
+            raise ValueError("Invalid data source type")
+
+
+class FreshnessValueMetric(FieldMetrics):
+    """
+    FreshnessMetric is a class that represents a metric that is generated by a data source.
+    """
+
+    def get_metric_identity(self):
+        return MetricIdentity.generate_identity(
+            metric_type=MetricsType.FRESHNESS,
+            metric_name=self.name,
+            data_source=self.data_source,
+            field_name=self.field_name,
+            table_name=self.table_name,
+            index_name=self.index_name,
+        )
+
+    def _generate_metric_value(self):
+        if isinstance(self.data_source, SQLDataSource):
+            return self.data_source.query_get_time_diff(table=self.table_name, field=self.field_name)
+        elif isinstance(self.data_source, SearchIndexDataSource):
+            return self.data_source.query_get_time_diff(index_name=self.index_name, field=self.field_name)
+        else:
+            raise ValueError("Invalid data source type")

dcs_core/core/profiling/__init__.py

@@ -0,0 +1,13 @@
+#  Copyright 2022-present, the Waterdip Labs Pvt. Ltd.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.