dcs-sdk 1.7.0__tar.gz → 1.7.2__tar.gz

{dcs_sdk-1.7.0 → dcs_sdk-1.7.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dcs-sdk
-Version: 1.7.0
+Version: 1.7.2
 Summary: SDK for DataChecks
 Author: Waterdip Labs
 Author-email: hello@waterdip.ai
@@ -86,7 +86,7 @@ Requires-Dist: vertica-python (>=1.4.0) ; extra == "vertica" or extra == "all-db
 Description-Content-Type: text/markdown
 
 <h1 align="center">
-  DCS SDK v1.7.0
+  DCS SDK v1.7.2
 </h1>
 
 > SDK for DataChecks

{dcs_sdk-1.7.0 → dcs_sdk-1.7.2}/README.md

@@ -1,5 +1,5 @@
 <h1 align="center">
-  DCS SDK v1.7.0
+  DCS SDK v1.7.2
 </h1>
 
 > SDK for DataChecks

dcs_sdk-1.7.2/dcs_core/core/datasource/file_datasource.py

@@ -0,0 +1,124 @@
+#  Copyright 2022-present, the Waterdip Labs Pvt. Ltd.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+import os
+import uuid
+from abc import ABC, abstractmethod
+from contextlib import contextmanager
+from pathlib import Path
+from typing import Dict, Iterator
+
+import duckdb
+from loguru import logger
+
+from dcs_core.core.common.models.data_source_resource import RawColumnInfo
+from dcs_core.core.datasource.base import DataSource
+from dcs_core.integrations.databases.duck_db import DuckDb
+
+
+class FileDataSource(DataSource, ABC):
+    """
+    Abstract class for File data sources
+    """
+
+    def __init__(self, data_source_name: str, data_connection: Dict):
+        super().__init__(data_source_name, data_connection)
+        self.temp_dir_name = "tmp"
+
+    @contextmanager
+    def as_duckdb(self, table_name: str) -> Iterator["DuckDb"]:
+        """Returns a DuckDB instance for the given table name"""
+        duckdb_path = self.load_file_to_duckdb(table_name)
+        duck_db_ds = DuckDb(data_source_name=self.data_source_name, data_connection={"file_path": duckdb_path})
+        try:
+            duck_db_ds.connect()
+            yield duck_db_ds
+        finally:
+            duck_db_ds.close()
+
+    @abstractmethod
+    def query_get_table_names(self) -> dict:
+        """
+        Query to get table names
+        """
+        pass
+
+    @abstractmethod
+    def query_get_database_version(self) -> str:
+        """
+        Get the database version
+        :return: version string
+        """
+        pass
+
+    @abstractmethod
+    def _download_to_path(self, table_name: str, path: str) -> None:
+        """Vendor-specific download"""
+        pass
+
+    def load_file_to_duckdb(self, table_name: str) -> str:
+        """Template method"""
+        os.makedirs(self.temp_dir_name, exist_ok=True)
+
+        ext = Path(table_name).suffix
+        if not ext:
+            raise ValueError(f"Invalid file name {table_name}")
+
+        temp_path = f"{self.temp_dir_name}/{uuid.uuid4()}{ext}"
+
+        try:
+            self._download_to_path(table_name, temp_path)
+            return self._load_path_to_duckdb(temp_path, table_name)
+        finally:
+            if os.path.exists(temp_path):
+                os.remove(temp_path)
+                logger.info(f"Cleaned up temp file {temp_path}")
+
+    def _load_path_to_duckdb(self, path: str, table_name: str) -> str:
+        """Shared DuckDB loading logic"""
+        tmp_dir = self.temp_dir_name
+        duckdb_path = f"{tmp_dir}/{uuid.uuid4()}.duckdb"
+        table_stem = Path(table_name).stem
+
+        logger.info(f"Loading {path} into DuckDB")
+
+        conn = None
+        try:
+            conn = duckdb.connect(database=duckdb_path, read_only=False)
+            conn.execute(
+                f'CREATE TABLE "{table_stem}" AS SELECT * FROM read_csv_auto(?)',
+                [path],
+            )
+            logger.info(f"Successfully loaded data into {duckdb_path}")
+            return duckdb_path
+        except Exception as e:
+            logger.warning(f"read_csv_auto failed: {e}. Trying with ALL_VARCHAR=TRUE")
+            try:
+                if conn:
+                    conn.close()
+                conn = duckdb.connect(database=duckdb_path, read_only=False)
+                conn.execute(
+                    f'CREATE TABLE "{table_stem}" AS ' f"SELECT * FROM read_csv(?, ALL_VARCHAR=TRUE, SAMPLE_SIZE=-1)",
+                    [path],
+                )
+                logger.info(f"Successfully loaded data with ALL_VARCHAR into {duckdb_path}")
+                return duckdb_path
+            except Exception as fallback_error:
+                logger.error(f"Failed to load CSV into DuckDB: {fallback_error}")
+                if os.path.exists(duckdb_path):
+                    os.remove(duckdb_path)
+                raise
+        finally:
+            if conn:
+                conn.close()

dcs_sdk-1.7.2/dcs_core/integrations/databases/azure_blob.py

@@ -0,0 +1,106 @@
+#  Copyright 2022-present, the Waterdip Labs Pvt. Ltd.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+from typing import Any, Dict, Optional
+
+from azure.storage.blob import BlobServiceClient
+from loguru import logger
+
+from dcs_core.core.common.errors import (
+    DataChecksDataSourcesConnectionError,
+    DatachecksTableFetchError,
+)
+from dcs_core.core.datasource.file_datasource import FileDataSource
+
+
+class AzureBlobDataSource(FileDataSource):
+    def __init__(self, data_source_name: str, data_connection: Dict):
+        super().__init__(data_source_name, data_connection)
+        self.allowed_file_extensions = [".csv"]
+        self.blob_service_client: Optional[BlobServiceClient] = None
+        self.DEFAULT_NUMERIC_PRECISION = 16383
+        self.connection = None
+
+    def connect(self) -> Any:
+        """
+        Connect to the file data source
+        """
+        try:
+            account_name = self.data_connection.get("account_name")
+            container_name = self.data_connection.get("container_name")
+            account_key = self.data_connection.get("account_key")
+            endpoint_suffix = self.data_connection.get("endpoint_suffix", "core.windows.net")
+            connection_str = f"https://{account_name}.blob.{endpoint_suffix}"
+            blob_service_client = BlobServiceClient(account_url=connection_str, credential=account_key)
+            self.blob_service_client = blob_service_client
+            self.connection = blob_service_client.get_container_client(container=container_name)
+            return self.connection
+        except Exception as e:
+            raise DataChecksDataSourcesConnectionError(f"Failed to connect to Azure Blob Storage: {e}")
+
+    def is_connected(self) -> bool:
+        """
+        Check if the file data source is connected
+        """
+        return self.connection is not None
+
+    def close(self):
+        """
+        Close the connection
+        """
+        self.connection.close()
+        self.blob_service_client.close()
+        self.connection = None
+        self.blob_service_client = None
+
+    def query_get_table_names(self) -> dict:
+        """
+        Query to get table names (blob names in this case)
+        """
+        if not self.is_connected():
+            raise DataChecksDataSourcesConnectionError("Not connected to Azure Blob Storage")
+        try:
+            subfolder = self.data_connection.get("subfolder", "")
+            blob_iterator = self.connection.list_blobs(name_starts_with=subfolder)
+            blobs = [
+                blob.name
+                for blob in blob_iterator
+                if len(blob.name.split("/")) == 1 and blob.name.endswith(tuple(self.allowed_file_extensions))
+            ]
+            return {"table": blobs}
+        except Exception as e:
+            raise DatachecksTableFetchError(f"Failed to list blobs: {e}")
+
+    def safe_get(self, lst, idx, default=None):
+        return lst[idx] if 0 <= idx < len(lst) else default
+
+    def query_get_database_version(self) -> str:
+        """
+        Get the database version
+        :return: version string
+        """
+        api_version = self.blob_service_client.api_version
+        return api_version
+
+    def _download_to_path(self, table_name: str, path: str):
+        """Download blob to path"""
+        blob_client = self.connection.get_blob_client(blob=table_name)
+        logger.info(f"Downloading {table_name} to {path}")
+        try:
+            with open(path, "wb") as f:
+                stream = blob_client.download_blob()
+                for chunk in stream.chunks():
+                    f.write(chunk)
+        except Exception as e:
+            raise DataChecksDataSourcesConnectionError(f"Failed to download blob '{table_name}': {e}")

dcs_sdk-1.7.2/dcs_core/integrations/databases/duck_db.py

@@ -0,0 +1,141 @@
+#  Copyright 2022-present, the Waterdip Labs Pvt. Ltd.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+import os
+from pathlib import Path
+from typing import Any, Dict
+
+import duckdb
+from loguru import logger
+
+from dcs_core.core.common.errors import DataChecksDataSourcesConnectionError
+from dcs_core.core.common.models.data_source_resource import RawColumnInfo
+from dcs_core.core.datasource.sql_datasource import SQLDataSource
+
+
+class DuckDb(SQLDataSource):
+    def __init__(self, data_source_name: str, data_connection: Dict):
+        super().__init__(data_source_name, data_connection)
+        self.connection = None
+        self.use_sa_text_query = False
+        self.regex_patterns = {
+            "uuid": r"^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$",
+            "usa_phone": r"^(\+1[-.\s]?)?(\(?\d{3}\)?[-.\s]?)?\d{3}[-.\s]?\d{4}$",
+            "email": r"^[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}$",
+            "usa_zip_code": r"^[0-9]{5}(?:-[0-9]{4})?$",
+            "ssn": r"^[0-9]{3}-[0-9]{2}-[0-9]{4}$",
+            "sedol": r"^[B-DF-HJ-NP-TV-XZ0-9]{6}[0-9]$",
+            "lei": r"^[A-Z0-9]{18}[0-9]{2}$",
+            "cusip": r"^[0-9A-Z]{9}$",
+            "figi": r"^BBG[A-Z0-9]{9}$",
+            "isin": r"^[A-Z]{2}[A-Z0-9]{9}[0-9]$",
+            "perm_id": r"^\d{4}[- ]?\d{4}[- ]?\d{4}[- ]?\d{4}[- ]?\d{3}$",
+        }
+        self.DEFAULT_NUMERIC_PRECISION = 16383
+
+    def connect(self) -> Any:
+        """
+        Connect to the file data source
+        """
+        try:
+            file_path = self.data_connection.get("file_path")
+            self.connection = duckdb.connect(database=file_path)
+            return self.connection
+        except Exception as e:
+            raise DataChecksDataSourcesConnectionError(f"Failed to connect to DuckDB: {e}")
+
+    def is_connected(self) -> bool:
+        """
+        Check if the file data source is connected
+        """
+        return self.connection is not None
+
+    def close(self):
+        """
+        Close the connection
+        """
+        logger.info("Closing DuckDB connection")
+        if self.connection:
+            self.connection.close()
+        try:
+            fp = self.data_connection.get("file_path")
+            if fp and os.path.exists(fp):
+                os.remove(fp)
+        except Exception as e:
+            logger.error(f"Failed to remove the file {self.data_connection.get('file_path')}: {e}")
+
+    def qualified_table_name(self, table_name: str) -> str:
+        """
+        Get the qualified table name
+        :param table_name: name of the table
+        :return: qualified table name
+        """
+        return f'"{table_name}"'
+
+    def quote_column(self, column: str) -> str:
+        """
+        Quote the column name
+        :param column: name of the column
+        :return: quoted column name
+        """
+        return f'"{column}"'
+
+    def query_get_table_columns(
+        self,
+        table: str,
+        schema: str | None = None,
+    ) -> Dict[str, RawColumnInfo]:
+        """
+        Get the schema of a table.
+        :param table: table name
+        :return:  Dictionary with column names and their types
+        """
+        schema = schema or self.schema_name
+        info_schema_path = ["information_schema", "columns"]
+        if self.database:
+            database = self.quote_database(self.database)
+            info_schema_path.insert(0, database)
+
+        query = f"""
+            SELECT
+                column_name,
+                data_type,
+                CASE WHEN data_type IN ('TIMESTAMP', 'TIME') THEN datetime_precision ELSE NULL END AS datetime_precision,
+                CASE WHEN data_type = 'DECIMAL' THEN COALESCE(numeric_precision, 131072 + {self.DEFAULT_NUMERIC_PRECISION})
+                     WHEN data_type IN ('DOUBLE', 'REAL', 'FLOAT') THEN numeric_precision
+                     ELSE numeric_precision END AS numeric_precision,
+                CASE WHEN data_type = 'DECIMAL' THEN COALESCE(numeric_scale, {self.DEFAULT_NUMERIC_PRECISION}) ELSE numeric_scale END AS numeric_scale,
+                NULL AS collation_name,
+                CASE WHEN data_type = 'VARCHAR' THEN character_maximum_length ELSE NULL END AS character_maximum_length
+            FROM information_schema.columns
+            WHERE table_name = '{table}'
+            ORDER BY ordinal_position
+        """
+
+        rows = self.fetchall(query)
+        if not rows:
+            raise RuntimeError(f"{table}: Table, {schema}: Schema, does not exist, or has no columns")
+
+        column_info = {
+            r[0]: RawColumnInfo(
+                column_name=self.safe_get(r, 0),
+                data_type=self.safe_get(r, 1),
+                datetime_precision=self.safe_get(r, 2),
+                numeric_precision=self.safe_get(r, 3),
+                numeric_scale=self.safe_get(r, 4),
+                collation_name=self.safe_get(r, 5),
+                character_maximum_length=self.safe_get(r, 6),
+            )
+            for r in rows
+        }
+        return column_info

{dcs_sdk-1.7.0 → dcs_sdk-1.7.2}/dcs_core/integrations/databases/mssql.py

@@ -167,8 +167,6 @@ class MssqlDataSource(SQLDataSource):
         :return: Dictionary with index details
         """
         schema = schema or self.schema_name
-        table = table.upper()
-        schema = schema.upper()
 
         query = f"""
             SELECT
@@ -673,8 +671,6 @@ class MssqlDataSource(SQLDataSource):
 
             quoted_name = self.quote_column(name)
 
-            query_parts.append(f"COUNT(DISTINCT {quoted_name}) AS [{name}_distinct]")
-            query_parts.append(f"COUNT({quoted_name}) - COUNT(DISTINCT {quoted_name}) AS [{name}_duplicate]")
             query_parts.append(f"SUM(CASE WHEN {quoted_name} IS NULL THEN 1 ELSE 0 END) AS [{name}_is_null]")
 
             if dtype in (
@@ -690,13 +686,20 @@ class MssqlDataSource(SQLDataSource):
                 "money",
                 "smallmoney",
             ):
+                query_parts.append(f"COUNT({quoted_name}) - COUNT(DISTINCT {quoted_name}) AS [{name}_duplicate]")
                 query_parts.append(f"MIN({quoted_name}) AS [{name}_min]")
                 query_parts.append(f"MAX({quoted_name}) AS [{name}_max]")
                 query_parts.append(f"AVG(CAST({quoted_name} AS FLOAT)) AS [{name}_average]")
+                query_parts.append(f"COUNT(DISTINCT {quoted_name}) AS [{name}_distinct]")
 
-            elif dtype in ("varchar", "nvarchar", "char", "nchar", "text", "ntext"):
+            elif dtype in ("varchar", "nvarchar", "char", "nchar"):
                 query_parts.append(f"MAX(LEN({quoted_name})) AS [{name}_max_character_length]")
 
+            elif dtype in ("text", "ntext", "xml"):
+                query_parts.append(
+                    f"MAX(LEN(CAST({quoted_name} AS NVARCHAR(MAX)))) " f"AS [{name}_max_character_length]"
+                )
+
         if additional_queries:
             query_parts.extend(additional_queries)
 

{dcs_sdk-1.7.0 → dcs_sdk-1.7.2}/dcs_core/integrations/databases/oracle.py

@@ -143,8 +143,6 @@ class OracleDataSource(SQLDataSource):
         :return: Dictionary with index details
         """
         schema = schema or self.schema_name
-        table = table.upper()
-        schema = schema.upper()
 
         query = f"""
             SELECT
@@ -696,8 +694,8 @@ class OracleDataSource(SQLDataSource):
                 AND r_ac.OWNER = r_acc.OWNER
                 AND acc.POSITION = r_acc.POSITION
             WHERE ac.CONSTRAINT_TYPE = 'R'
-            AND ac.TABLE_NAME = '{table_name.upper()}'
-            AND ac.OWNER = '{schema.upper()}';
+            AND ac.TABLE_NAME = '{table_name}'
+            AND ac.OWNER = '{schema}';
         """
 
         try:

{dcs_sdk-1.7.0 → dcs_sdk-1.7.2}/dcs_core/integrations/databases/postgres.py

@@ -136,8 +136,6 @@ class PostgresDataSource(SQLDataSource):
         :return: Dictionary with index details
         """
         schema = schema or self.schema_name
-        table = table.lower()
-        schema = schema.lower()
 
         query = f"""
             SELECT

{dcs_sdk-1.7.0 → dcs_sdk-1.7.2}/dcs_sdk/__version__.py

@@ -12,4 +12,4 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-__version__ = "1.7.0"
+__version__ = "1.7.2"

{dcs_sdk-1.7.0 → dcs_sdk-1.7.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "dcs-sdk"
-version = "1.7.0"
+version = "1.7.2"
 description = "SDK for DataChecks"
 authors = ["Waterdip Labs <hello@waterdip.ai>"]
 readme = "README.md"

dcs_sdk-1.7.0/dcs_core/core/datasource/file_datasource.py

@@ -1,30 +0,0 @@
-#  Copyright 2022-present, the Waterdip Labs Pvt. Ltd.
-#
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-
-from typing import Dict
-
-from dcs_core.core.datasource.base import DataSource
-
-
-class FileDataSource(DataSource):
-    """
-    Abstract class for File data sources
-    """
-
-    def __init__(self, data_source_name: str, data_connection: Dict):
-        super().__init__(data_source_name, data_connection)
-
-    def load_file_to_duckdb(self, table_name: str):
-        """Load the file to duckdb"""
-        pass