dcs-sdk 1.6.9__tar.gz → 1.7.1__tar.gz

{dcs_sdk-1.6.9 → dcs_sdk-1.7.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dcs-sdk
-Version: 1.6.9
+Version: 1.7.1
 Summary: SDK for DataChecks
 Author: Waterdip Labs
 Author-email: hello@waterdip.ai
@@ -86,7 +86,7 @@ Requires-Dist: vertica-python (>=1.4.0) ; extra == "vertica" or extra == "all-db
 Description-Content-Type: text/markdown
 
 <h1 align="center">
-  DCS SDK v1.6.9
+  DCS SDK v1.7.1
 </h1>
 
 > SDK for DataChecks

{dcs_sdk-1.6.9 → dcs_sdk-1.7.1}/README.md

@@ -1,5 +1,5 @@
 <h1 align="center">
-  DCS SDK v1.6.9
+  DCS SDK v1.7.1
 </h1>
 
 > SDK for DataChecks

{dcs_sdk-1.6.9 → dcs_sdk-1.7.1}/dcs_core/core/common/models/configuration.py

@@ -44,6 +44,7 @@ class DataSourceType(str, Enum):
     DB2 = "db2"
     SYBASE = "sybase"
     AZURE_BLOB = "azure_blob"
+    DUCK_DB = "duck_db"
 
 
 class DataSourceLanguageSupport(str, Enum):
@@ -92,6 +93,8 @@ class DataSourceConnectionConfiguration:
     endpoint_suffix: Optional[str] = None
     subfolder_path: Optional[str] = None
 
+    file_path: Optional[str] = None
+
 
 @dataclass
 class DataSourceConfiguration:

{dcs_sdk-1.6.9 → dcs_sdk-1.7.1}/dcs_core/core/configuration/configuration_parser.py

@@ -81,6 +81,7 @@ class DataSourceConfigParser(ConfigParser):
             protocol=config["connection"].get("protocol"),
             driver=config["connection"].get("driver"),
             server=config["connection"].get("server"),
+            file_path=config["connection"].get("file_path"),
         )
         return connection_config
 

dcs_sdk-1.7.1/dcs_core/core/datasource/file_datasource.py

@@ -0,0 +1,124 @@
+#  Copyright 2022-present, the Waterdip Labs Pvt. Ltd.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+import os
+import uuid
+from abc import ABC, abstractmethod
+from contextlib import contextmanager
+from pathlib import Path
+from typing import Dict, Iterator
+
+import duckdb
+from loguru import logger
+
+from dcs_core.core.common.models.data_source_resource import RawColumnInfo
+from dcs_core.core.datasource.base import DataSource
+from dcs_core.integrations.databases.duck_db import DuckDb
+
+
+class FileDataSource(DataSource, ABC):
+    """
+    Abstract class for File data sources
+    """
+
+    def __init__(self, data_source_name: str, data_connection: Dict):
+        super().__init__(data_source_name, data_connection)
+        self.temp_dir_name = "tmp"
+
+    @contextmanager
+    def as_duckdb(self, table_name: str) -> Iterator["DuckDb"]:
+        """Returns a DuckDB instance for the given table name"""
+        duckdb_path = self.load_file_to_duckdb(table_name)
+        duck_db_ds = DuckDb(data_source_name=self.data_source_name, data_connection={"file_path": duckdb_path})
+        try:
+            duck_db_ds.connect()
+            yield duck_db_ds
+        finally:
+            duck_db_ds.close()
+
+    @abstractmethod
+    def query_get_table_names(self) -> dict:
+        """
+        Query to get table names
+        """
+        pass
+
+    @abstractmethod
+    def query_get_database_version(self) -> str:
+        """
+        Get the database version
+        :return: version string
+        """
+        pass
+
+    @abstractmethod
+    def _download_to_path(self, table_name: str, path: str) -> None:
+        """Vendor-specific download"""
+        pass
+
+    def load_file_to_duckdb(self, table_name: str) -> str:
+        """Template method"""
+        os.makedirs(self.temp_dir_name, exist_ok=True)
+
+        ext = Path(table_name).suffix
+        if not ext:
+            raise ValueError(f"Invalid file name {table_name}")
+
+        temp_path = f"{self.temp_dir_name}/{uuid.uuid4()}{ext}"
+
+        try:
+            self._download_to_path(table_name, temp_path)
+            return self._load_path_to_duckdb(temp_path, table_name)
+        finally:
+            if os.path.exists(temp_path):
+                os.remove(temp_path)
+                logger.info(f"Cleaned up temp file {temp_path}")
+
+    def _load_path_to_duckdb(self, path: str, table_name: str) -> str:
+        """Shared DuckDB loading logic"""
+        tmp_dir = self.temp_dir_name
+        duckdb_path = f"{tmp_dir}/{uuid.uuid4()}.duckdb"
+        table_stem = Path(table_name).stem
+
+        logger.info(f"Loading {path} into DuckDB")
+
+        conn = None
+        try:
+            conn = duckdb.connect(database=duckdb_path, read_only=False)
+            conn.execute(
+                f'CREATE TABLE "{table_stem}" AS SELECT * FROM read_csv_auto(?)',
+                [path],
+            )
+            logger.info(f"Successfully loaded data into {duckdb_path}")
+            return duckdb_path
+        except Exception as e:
+            logger.warning(f"read_csv_auto failed: {e}. Trying with ALL_VARCHAR=TRUE")
+            try:
+                if conn:
+                    conn.close()
+                conn = duckdb.connect(database=duckdb_path, read_only=False)
+                conn.execute(
+                    f'CREATE TABLE "{table_stem}" AS ' f"SELECT * FROM read_csv(?, ALL_VARCHAR=TRUE, SAMPLE_SIZE=-1)",
+                    [path],
+                )
+                logger.info(f"Successfully loaded data with ALL_VARCHAR into {duckdb_path}")
+                return duckdb_path
+            except Exception as fallback_error:
+                logger.error(f"Failed to load CSV into DuckDB: {fallback_error}")
+                if os.path.exists(duckdb_path):
+                    os.remove(duckdb_path)
+                raise
+        finally:
+            if conn:
+                conn.close()

{dcs_sdk-1.6.9 → dcs_sdk-1.7.1}/dcs_core/core/datasource/manager.py

@@ -58,6 +58,7 @@ class DataSourceManager:
         "db2": "DB2DataSource",
         "sybase": "SybaseDataSource",
         "azure_blob": "AzureBlobDataSource",
+        "duck_db": "DuckDb",
     }
 
     def __init__(self, config: Configuration):

{dcs_sdk-1.6.9 → dcs_sdk-1.7.1}/dcs_core/core/inspect.py

@@ -130,7 +130,6 @@ class Inspect:
         try:
             self.data_source_manager.connect()
             self.validation_manager.build_validations()
-
             validation_infos: Dict[str, ValidationInfo] = {}
 
             for datasource, _ in self.validation_manager.get_validations.items():

{dcs_sdk-1.6.9 → dcs_sdk-1.7.1}/dcs_core/integrations/databases/azure_blob.py

@@ -12,17 +12,21 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-import io
-from typing import Any, Dict, List, Optional
+import os
+import uuid
+from pathlib import Path
+from typing import Any, Dict, Optional
 
-import pandas as pd
+import duckdb
 from azure.storage.blob import BlobServiceClient
+from loguru import logger
 
 from dcs_core.core.common.errors import (
     DatachecksColumnFetchError,
     DataChecksDataSourcesConnectionError,
     DatachecksTableFetchError,
 )
+from dcs_core.core.common.models.data_source_resource import RawColumnInfo
 from dcs_core.core.datasource.file_datasource import FileDataSource
 
 
@@ -31,6 +35,7 @@ class AzureBlobDataSource(FileDataSource):
         super().__init__(data_source_name, data_connection)
         self.allowed_file_extensions = [".csv"]
         self.blob_service_client: Optional[BlobServiceClient] = None
+        self.DEFAULT_NUMERIC_PRECISION = 16383
         self.connection = None
 
     def connect(self) -> Any:
@@ -83,28 +88,8 @@ class AzureBlobDataSource(FileDataSource):
         except Exception as e:
             raise DatachecksTableFetchError(f"Failed to list blobs: {e}")
 
-    def query_get_table_columns(self, table: str) -> List[dict]:
-        """
-        Get column names for a table (CSV blob in this case).
-        """
-        if not self.is_connected():
-            raise DataChecksDataSourcesConnectionError("Not connected to Azure Blob Storage")
-
-        if not any(table.endswith(ext) for ext in self.allowed_file_extensions):
-            raise ValueError(f"Unsupported file type for {table}. Allowed: {self.allowed_file_extensions}")
-
-        try:
-            blob_client = self.connection.get_blob_client(blob=table)
-            download_stream = blob_client.download_blob()
-            data = download_stream.readall()
-            if table.endswith(".csv"):
-                df = pd.read_csv(io.BytesIO(data))
-            else:
-                raise ValueError(f"Unsupported file type for {table}. Allowed: {self.allowed_file_extensions}")
-
-            return [{"column_name": col, "column_type": "string"} for col in df.columns.tolist()]
-        except Exception as e:
-            raise DatachecksColumnFetchError(f"Failed to read columns from blob '{table}': {e}")
+    def safe_get(self, lst, idx, default=None):
+        return lst[idx] if 0 <= idx < len(lst) else default
 
     def query_get_database_version(self) -> str:
         """
@@ -113,3 +98,15 @@ class AzureBlobDataSource(FileDataSource):
         """
         api_version = self.blob_service_client.api_version
         return api_version
+
+    def _download_to_path(self, table_name: str, path: str):
+        """Download blob to path"""
+        blob_client = self.connection.get_blob_client(blob=table_name)
+        logger.info(f"Downloading {table_name} to {path}")
+        try:
+            with open(path, "wb") as f:
+                stream = blob_client.download_blob()
+                for chunk in stream.chunks():
+                    f.write(chunk)
+        except Exception as e:
+            raise DataChecksDataSourcesConnectionError(f"Failed to download blob '{table_name}': {e}")

dcs_sdk-1.7.1/dcs_core/integrations/databases/duck_db.py

@@ -0,0 +1,141 @@
+#  Copyright 2022-present, the Waterdip Labs Pvt. Ltd.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+import os
+from pathlib import Path
+from typing import Any, Dict
+
+import duckdb
+from loguru import logger
+
+from dcs_core.core.common.errors import DataChecksDataSourcesConnectionError
+from dcs_core.core.common.models.data_source_resource import RawColumnInfo
+from dcs_core.core.datasource.sql_datasource import SQLDataSource
+
+
+class DuckDb(SQLDataSource):
+    def __init__(self, data_source_name: str, data_connection: Dict):
+        super().__init__(data_source_name, data_connection)
+        self.connection = None
+        self.use_sa_text_query = False
+        self.regex_patterns = {
+            "uuid": r"^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$",
+            "usa_phone": r"^(\+1[-.\s]?)?(\(?\d{3}\)?[-.\s]?)?\d{3}[-.\s]?\d{4}$",
+            "email": r"^[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}$",
+            "usa_zip_code": r"^[0-9]{5}(?:-[0-9]{4})?$",
+            "ssn": r"^[0-9]{3}-[0-9]{2}-[0-9]{4}$",
+            "sedol": r"^[B-DF-HJ-NP-TV-XZ0-9]{6}[0-9]$",
+            "lei": r"^[A-Z0-9]{18}[0-9]{2}$",
+            "cusip": r"^[0-9A-Z]{9}$",
+            "figi": r"^BBG[A-Z0-9]{9}$",
+            "isin": r"^[A-Z]{2}[A-Z0-9]{9}[0-9]$",
+            "perm_id": r"^\d{4}[- ]?\d{4}[- ]?\d{4}[- ]?\d{4}[- ]?\d{3}$",
+        }
+        self.DEFAULT_NUMERIC_PRECISION = 16383
+
+    def connect(self) -> Any:
+        """
+        Connect to the file data source
+        """
+        try:
+            file_path = self.data_connection.get("file_path")
+            self.connection = duckdb.connect(database=file_path)
+            return self.connection
+        except Exception as e:
+            raise DataChecksDataSourcesConnectionError(f"Failed to connect to DuckDB: {e}")
+
+    def is_connected(self) -> bool:
+        """
+        Check if the file data source is connected
+        """
+        return self.connection is not None
+
+    def close(self):
+        """
+        Close the connection
+        """
+        logger.info("Closing DuckDB connection")
+        if self.connection:
+            self.connection.close()
+        try:
+            fp = self.data_connection.get("file_path")
+            if fp and os.path.exists(fp):
+                os.remove(fp)
+        except Exception as e:
+            logger.error(f"Failed to remove the file {self.data_connection.get('file_path')}: {e}")
+
+    def qualified_table_name(self, table_name: str) -> str:
+        """
+        Get the qualified table name
+        :param table_name: name of the table
+        :return: qualified table name
+        """
+        return f'"{table_name}"'
+
+    def quote_column(self, column: str) -> str:
+        """
+        Quote the column name
+        :param column: name of the column
+        :return: quoted column name
+        """
+        return f'"{column}"'
+
+    def query_get_table_columns(
+        self,
+        table: str,
+        schema: str | None = None,
+    ) -> Dict[str, RawColumnInfo]:
+        """
+        Get the schema of a table.
+        :param table: table name
+        :return:  Dictionary with column names and their types
+        """
+        schema = schema or self.schema_name
+        info_schema_path = ["information_schema", "columns"]
+        if self.database:
+            database = self.quote_database(self.database)
+            info_schema_path.insert(0, database)
+
+        query = f"""
+            SELECT
+                column_name,
+                data_type,
+                CASE WHEN data_type IN ('TIMESTAMP', 'TIME') THEN datetime_precision ELSE NULL END AS datetime_precision,
+                CASE WHEN data_type = 'DECIMAL' THEN COALESCE(numeric_precision, 131072 + {self.DEFAULT_NUMERIC_PRECISION})
+                     WHEN data_type IN ('DOUBLE', 'REAL', 'FLOAT') THEN numeric_precision
+                     ELSE numeric_precision END AS numeric_precision,
+                CASE WHEN data_type = 'DECIMAL' THEN COALESCE(numeric_scale, {self.DEFAULT_NUMERIC_PRECISION}) ELSE numeric_scale END AS numeric_scale,
+                NULL AS collation_name,
+                CASE WHEN data_type = 'VARCHAR' THEN character_maximum_length ELSE NULL END AS character_maximum_length
+            FROM information_schema.columns
+            WHERE table_name = '{table}'
+            ORDER BY ordinal_position
+        """
+
+        rows = self.fetchall(query)
+        if not rows:
+            raise RuntimeError(f"{table}: Table, {schema}: Schema, does not exist, or has no columns")
+
+        column_info = {
+            r[0]: RawColumnInfo(
+                column_name=self.safe_get(r, 0),
+                data_type=self.safe_get(r, 1),
+                datetime_precision=self.safe_get(r, 2),
+                numeric_precision=self.safe_get(r, 3),
+                numeric_scale=self.safe_get(r, 4),
+                collation_name=self.safe_get(r, 5),
+                character_maximum_length=self.safe_get(r, 6),
+            )
+            for r in rows
+        }
+        return column_info

{dcs_sdk-1.6.9 → dcs_sdk-1.7.1}/dcs_sdk/__version__.py

@@ -12,4 +12,4 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-__version__ = "1.6.9"
+__version__ = "1.7.1"

{dcs_sdk-1.6.9 → dcs_sdk-1.7.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "dcs-sdk"
-version = "1.6.9"
+version = "1.7.1"
 description = "SDK for DataChecks"
 authors = ["Waterdip Labs <hello@waterdip.ai>"]
 readme = "README.md"

dcs_sdk-1.6.9/dcs_core/core/datasource/file_datasource.py

@@ -1,26 +0,0 @@
-#  Copyright 2022-present, the Waterdip Labs Pvt. Ltd.
-#
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-
-from typing import Dict
-
-from dcs_core.core.datasource.base import DataSource
-
-
-class FileDataSource(DataSource):
-    """
-    Abstract class for File data sources
-    """
-
-    def __init__(self, data_source_name: str, data_connection: Dict):
-        super().__init__(data_source_name, data_connection)