dcs-sdk 1.6.5__py3-none-any.whl

dcs_core/core/profiling/datasource_profiling.py

@@ -0,0 +1,136 @@
+#  Copyright 2022-present, the Waterdip Labs Pvt. Ltd.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+from datetime import datetime, timezone
+from typing import Dict, List, Union
+
+from dcs_core.core.common.models.metric import (
+    IndexMetrics,
+    MetricsType,
+    MetricValue,
+    TableMetrics,
+)
+from dcs_core.core.datasource.base import DataSource
+from dcs_core.core.datasource.sql_datasource import SQLDataSource
+from dcs_core.core.metric.base import MetricIdentity
+from dcs_core.core.profiling.numeric_field_profiling import NumericSQLFieldProfiler
+from dcs_core.core.profiling.text_field_profiling import TextSQLFieldProfiler
+
+
+class DataSourceProfiling:
+    """
+    This class is responsible for generating field profiles for a given data source.
+
+    """
+
+    def __init__(self, data_source: DataSource):
+        """
+        :param data_source: The data source for which field profiles are to be generated.
+        """
+        self._datasource = data_source
+        if isinstance(data_source, SQLDataSource):
+            self._tables: List[str] = data_source.query_get_table_metadata()
+            self._field_meta_data: Dict[str, Dict[str, str]] = {}
+            for table in self._tables:
+                self._field_meta_data[table] = data_source.query_get_column_metadata(table_name=table)
+
+    def _generate_sql_data_source_profiles(self) -> List[TableMetrics]:
+        """
+        This method generates field profiles for a SQL data source.
+        """
+        list_of_metric = []
+        for table, fields in self._field_meta_data.items():
+            table_metrics: List[MetricValue] = []
+
+            for field, data_type in fields.items():
+                # profile for numeric fields if the data type is numeric
+                if data_type in DataSource.NUMERIC_PYTHON_TYPES_FOR_PROFILING:
+                    metrics = self._generate_numeric_field_profile(table=table, field=field, data_type=data_type)
+                    table_metrics.extend(metrics)
+
+                # profile for numeric fields if the data type is text
+                elif data_type in DataSource.TEXT_PYTHON_TYPES_FOR_PROFILING:
+                    metrics = self._generate_text_field_profile(table=table, field=field, data_type=data_type)
+                    table_metrics.extend(metrics)
+
+            # add row count metrics
+            table_metrics.append(self._generate_sql_table_row_count(table=table))
+            # create a table metric list for a table
+            list_of_metric.append(
+                TableMetrics(
+                    table_name=table,
+                    metrics={metric.identity: metric for metric in table_metrics},
+                    data_source=self._datasource.data_source_name,
+                )
+            )
+
+        return list_of_metric
+
+    def _generate_sql_table_row_count(self, table: str) -> MetricValue:
+        if isinstance(self._datasource, SQLDataSource):
+            table_row_count = self._datasource.query_get_row_count(table=table)
+            return MetricValue(
+                identity=MetricIdentity.generate_identity(
+                    metric_name="",
+                    metric_type=MetricsType.ROW_COUNT,
+                    data_source=self._datasource,
+                    table_name=table,
+                ),
+                value=table_row_count,
+                data_source=self._datasource.data_source_name,
+                metric_type=MetricsType.ROW_COUNT,
+                table_name=table,
+                timestamp=datetime.now(timezone.utc),
+            )
+
+    def _generate_numeric_field_profile(self, table: str, field: str, data_type: str) -> List[MetricValue]:
+        """
+        This method generates a numeric field profile for a given field.
+        """
+        profiles = []
+        if isinstance(self._datasource, SQLDataSource):
+            profiler = NumericSQLFieldProfiler(
+                data_source=self._datasource,
+                table_name=table,
+                field_name=field,
+                data_type=data_type,
+            )
+            generate = profiler.generate()
+            profiles = generate.get_metric_values
+        return profiles
+
+    def _generate_text_field_profile(self, table: str, field: str, data_type: str) -> List[MetricValue]:
+        """
+        This method generates a text field profile for a given field.
+        """
+        profiles = []
+
+        if isinstance(self._datasource, SQLDataSource):
+            profiler = TextSQLFieldProfiler(
+                data_source=self._datasource,
+                table_name=table,
+                field_name=field,
+                data_type=data_type,
+            )
+            generate = profiler.generate()
+            profiles = generate.get_metric_values
+        return profiles
+
+    def generate(self) -> List[Union[TableMetrics, IndexMetrics]]:
+        """
+        This method generates field profiles for a given data source.
+        """
+        if isinstance(self._datasource, SQLDataSource):
+            return self._generate_sql_data_source_profiles()
+        else:
+            raise NotImplementedError(f"Profiling for {self._datasource.data_source_name} is not implemented.")

dcs_core/core/profiling/numeric_field_profiling.py

@@ -0,0 +1,72 @@
+#  Copyright 2022-present, the Waterdip Labs Pvt. Ltd.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+from datetime import datetime, timezone
+from typing import Dict
+
+from dcs_core.core.common.models.metric import MetricsType, MetricValue
+from dcs_core.core.common.models.profile import NumericFieldProfile
+from dcs_core.core.datasource.sql_datasource import SQLDataSource
+from dcs_core.core.metric.base import MetricIdentity
+
+
+class NumericSQLFieldProfiler:
+    """
+    NumericSQLFieldProfiler is a class that represents a numeric field profile that is generated by a data source.
+    """
+
+    def __init__(
+        self,
+        data_source: SQLDataSource,
+        table_name: str,
+        field_name: str,
+        data_type: str,
+    ):
+        self._data_source = data_source
+        self._table_name = table_name
+        self._field_name = field_name
+        self._data_type = data_type
+
+    def generate(self) -> NumericFieldProfile:
+        """
+        Generate a numeric field profile.
+        """
+        data: Dict = self._data_source.profiling_sql_aggregates_numeric(self._table_name, self._field_name)
+        return self._generate_field_profile(data)
+
+    def _generate_field_profile(self, data: Dict) -> NumericFieldProfile:
+        """
+        Generate a numeric field profile from the data provided.
+        """
+        profile = NumericFieldProfile(
+            field_name=self._field_name,
+            data_type=self._data_type,
+        )
+        timestamp = datetime.now(timezone.utc)
+        for key, value in data.items():
+            metric_value = MetricValue(
+                value=value,
+                identity=MetricIdentity.generate_identity(
+                    metric_name="",
+                    metric_type=MetricsType(key),
+                    data_source=self._data_source,
+                    table_name=self._table_name,
+                    field_name=self._field_name,
+                ),
+                metric_type=MetricsType(key),
+                timestamp=timestamp,
+                data_source=self._data_source.data_source_name,
+            )
+            setattr(profile, key, metric_value)
+        return profile

dcs_core/core/profiling/text_field_profiling.py

@@ -0,0 +1,67 @@
+#  Copyright 2022-present, the Waterdip Labs Pvt. Ltd.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+from datetime import datetime, timezone
+from typing import Dict
+
+from dcs_core.core.common.models.metric import MetricsType, MetricValue
+from dcs_core.core.common.models.profile import TextFieldProfile
+from dcs_core.core.datasource.sql_datasource import SQLDataSource
+from dcs_core.core.metric.base import MetricIdentity
+
+
+class TextSQLFieldProfiler:
+    def __init__(
+        self,
+        data_source: SQLDataSource,
+        table_name: str,
+        field_name: str,
+        data_type: str,
+    ):
+        self._data_source = data_source
+        self._table_name = table_name
+        self._field_name = field_name
+        self._data_type = data_type
+
+    def generate(self) -> TextFieldProfile:
+        data: Dict = self._data_source.profiling_sql_aggregates_string(self._table_name, self._field_name)
+        return self._generate_field_profile(data)
+
+    def _generate_field_profile(self, data: Dict) -> TextFieldProfile:
+        """
+        Generate a numeric field profile from the data provided.
+        """
+        profile = TextFieldProfile(
+            field_name=self._field_name,
+            data_type=self._data_type,
+        )
+        timestamp = datetime.now(timezone.utc)
+        for key, value in data.items():
+            metric_value = MetricValue(
+                value=value,
+                identity=MetricIdentity.generate_identity(
+                    metric_name="",
+                    metric_type=MetricsType(key),
+                    data_source=self._data_source,
+                    table_name=self._table_name,
+                    field_name=self._field_name,
+                ),
+                metric_type=MetricsType(key),
+                timestamp=timestamp,
+                data_source=self._data_source.data_source_name,
+                table_name=self._table_name,
+                field_name=self._field_name,
+            )
+            setattr(profile, key, metric_value)
+        return profile

dcs_core/core/repository/__init__.py

@@ -0,0 +1,13 @@
+#  Copyright 2022-present, the Waterdip Labs Pvt. Ltd.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.

dcs_core/core/repository/metric_repository.py

@@ -0,0 +1,77 @@
+#  Copyright 2022-present, the Waterdip Labs Pvt. Ltd.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+import abc
+from datetime import datetime
+from typing import Dict, List, Tuple
+
+from dcs_core.core.common.models.metric import MetricValue
+
+
+class MetricRepository(metaclass=abc.ABCMeta):
+    @abc.abstractmethod
+    def save_metric(self, metric_id, metric_value: MetricValue) -> int:
+        """
+        This method will save the metric_value for the given metric_id. The metric_value will be stored in two time
+        granularity - daily and hourly.
+        - daily: the time store value will be the start of the day. For example a metric value for 2020-01-01 10:30am
+            will be stored as 2020-01-01 00:00:00. This is to ensure that we can query the metric value for a given day.
+            If a metric is generated multiple times a day then the latest value will be stored and overwritten.
+        - hourly: the time store value will be the start of the hour. For example a metric value for 2020-01-01 10:30am
+            will be stored as 2020-01-01 10:00:00. This is to ensure that we can query the metric value for a given hour.
+            If a metric is generated multiple times an hour then the latest value will be stored and overwritten.
+        """
+        pass
+
+    @abc.abstractmethod
+    def save_all_metrics(self, metrics: List[MetricValue]) -> int:
+        """
+        This method will save all the metrics in the given list. The metric_value will be stored in two time
+        granularity - daily and hourly.
+        """
+        pass
+
+    @abc.abstractmethod
+    def get_metric_by_id(
+        self, metric_id, start_date: datetime = None, end_date: datetime = None
+    ) -> Tuple[List[MetricValue], List[MetricValue]]:
+        """
+        This method will return the metrics stored in the repository for the given metric_id. The metrics will be
+        returned as a tuple of two lists. The first list will contain the daily metrics and the second list will contain
+        the hourly metrics.
+
+        return value:
+        (
+            [daily_metric_1, daily_metric_2],
+            [hourly_metric_1, hourly_metric_2]
+        )
+        """
+        pass
+
+    @abc.abstractmethod
+    def get_all_metrics(
+        self, start_date: datetime = None, end_date: datetime = None
+    ) -> Dict[str, Tuple[List[MetricValue], List[MetricValue]]]:
+        """
+        This method will return all the metrics stored in the repository. The metrics will be returned as a dictionary
+        with the key being the metric_id and the value being a tuple of two lists. The first list will contain the
+        daily metrics and the second list will contain the hourly metrics.
+
+        return value:
+        {
+            "metric_id_1": ([daily_metric_1, daily_metric_2], [hourly_metric_1, hourly_metric_2]),
+            "metric_id_2": ([daily_metric_1, daily_metric_2], [hourly_metric_1, hourly_metric_2])
+        }
+        """
+        pass

dcs_core/core/utils/__init__.py

@@ -0,0 +1,13 @@
+#  Copyright 2022-present, the Waterdip Labs Pvt. Ltd.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.

dcs_core/core/utils/log.py

@@ -0,0 +1,29 @@
+#  Copyright 2022-present, the Waterdip Labs Pvt. Ltd.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+from loguru import Logger
+
+
+class LogLevel:
+    pass
+
+
+class Log:
+    def __init__(self, logger: Logger):
+        self.logger: Logger = logger
+        self.logs: list[Log] = []
+        self.verbose: bool = False
+
+    def log(self, level, message, location, doc, exception):
+        pass

dcs_core/core/utils/tracking.py

@@ -0,0 +1,105 @@
+#  Copyright 2022-present, the Waterdip Labs Pvt. Ltd.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+"""
+Module for anonymous telemetry.
+
+No credentials, data, personal information or anything private is collected (and will never be).
+"""
+import json
+import os
+import platform
+from time import time
+from typing import Optional
+from uuid import uuid4
+
+import requests
+from loguru import logger
+
+from dcs_core.core.utils.utils import truncate_error
+
+TRACKING_DISABLED = os.environ.get("DISABLE_DCS_ANONYMOUS_TELEMETRY", False)
+TRACK_URL = "https://hosted.rudderlabs.com/v1/track"
+TOKEN = "2U4Bsait5XpEyHnbFtJSjig7KH8"
+TIMEOUT = 8
+
+dcs_anonymous_id = None
+
+
+def is_tracking_enabled():
+    return not TRACKING_DISABLED
+
+
+def get_anonymous_id():
+    global dcs_anonymous_id
+    if dcs_anonymous_id is None:
+        dcs_anonymous_id = str(uuid4())
+    return dcs_anonymous_id
+
+
+def create_error_event(
+    exception: Exception,
+):
+    error = truncate_error(repr(exception))
+    return {
+        "event": "dcs_error",
+        "properties": {
+            "distinct_id": get_anonymous_id(),
+            "token": TOKEN,
+            "time": time(),
+            "os_type": os.name,
+            "os_version": platform.platform(),
+            "python_version": f"{platform.python_version()}/{platform.python_implementation()}",
+            "error": error,
+        },
+    }
+
+
+def create_inspect_event_json(
+    runtime_seconds: float,
+    inspect_info: Optional[dict] = None,
+    error: Optional[str] = None,
+):
+    return {
+        "event": "dcs_inspect_end",
+        "properties": {
+            "distinct_id": get_anonymous_id(),
+            "token": TOKEN,
+            "time": time(),
+            "runtime_seconds": runtime_seconds,
+            "os_type": os.name,
+            "os_version": platform.platform(),
+            "python_version": f"{platform.python_version()}/{platform.python_implementation()}",
+            "count_metrics": inspect_info.get("metrics_count", 0) if inspect_info else 0,
+            "count_datasource": inspect_info.get("datasource_count", 0) if inspect_info else 0,
+            "count_tables": inspect_info.get("table_count", 0) if inspect_info else 0,
+            "count_index": inspect_info.get("index_count", 0) if inspect_info else 0,
+            "error": error,
+        },
+    }
+
+
+def send_event_json(event_json):
+    if is_tracking_enabled():
+        headers = {
+            "Content-Type": "application/json",
+            "Authorization": "Basic MlU0QnNhaXQ1WHBFeUhuYkZ0SlNqaWc3S0g4Og==",
+        }
+        data = json.dumps(event_json).encode()
+        try:
+            response = requests.post(TRACK_URL, data=data, headers=headers)
+            if response.status_code != 200:
+                raise RuntimeError(response)
+        except Exception as e:
+            logger.debug(f"Failed to post to Rudderstack: {e}")

dcs_core/core/utils/utils.py

@@ -0,0 +1,44 @@
+#  Copyright 2022-present, the Waterdip Labs Pvt. Ltd.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+import json
+import re
+from datetime import datetime
+from pathlib import Path
+
+
+def truncate_error(error: str):
+    first_line = error.split("\n", 1)[0]
+    return re.sub("'(.*?)'", "'***'", first_line)
+
+
+def ensure_directory_exists(dir_path: str, create_if_not_exists=True):
+    dir_path = Path(dir_path)
+    if dir_path.exists() and dir_path.is_dir():
+        return True
+    elif create_if_not_exists:
+        dir_path.mkdir(parents=True)
+        return True
+    return False
+
+
+def write_to_file(file_path: str, data: str):
+    with open(file_path, "w") as file:
+        file.write(data)
+
+
+class EnhancedJSONEncoder(json.JSONEncoder):
+    def default(self, o):
+        if isinstance(o, datetime):
+            return o.isoformat()
+        return super().default(o)

dcs_core/core/validation/__init__.py

@@ -0,0 +1,13 @@
+#  Copyright 2022-present, the Waterdip Labs Pvt. Ltd.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.