dcs-sdk 1.6.9__py3-none-any.whl → 1.7.0__py3-none-any.whl

dcs_core/core/common/models/configuration.py

@@ -44,6 +44,7 @@ class DataSourceType(str, Enum):
     DB2 = "db2"
     SYBASE = "sybase"
     AZURE_BLOB = "azure_blob"
+    DUCK_DB = "duck_db"
 
 
 class DataSourceLanguageSupport(str, Enum):
@@ -92,6 +93,8 @@ class DataSourceConnectionConfiguration:
     endpoint_suffix: Optional[str] = None
     subfolder_path: Optional[str] = None
 
+    file_path: Optional[str] = None
+
 
 @dataclass
 class DataSourceConfiguration:

dcs_core/core/configuration/configuration_parser.py

@@ -81,6 +81,7 @@ class DataSourceConfigParser(ConfigParser):
             protocol=config["connection"].get("protocol"),
             driver=config["connection"].get("driver"),
             server=config["connection"].get("server"),
+            file_path=config["connection"].get("file_path"),
         )
         return connection_config
 

dcs_core/core/datasource/file_datasource.py

@@ -24,3 +24,7 @@ class FileDataSource(DataSource):
 
     def __init__(self, data_source_name: str, data_connection: Dict):
         super().__init__(data_source_name, data_connection)
+
+    def load_file_to_duckdb(self, table_name: str):
+        """Load the file to duckdb"""
+        pass

dcs_core/core/datasource/manager.py

@@ -58,6 +58,7 @@ class DataSourceManager:
         "db2": "DB2DataSource",
         "sybase": "SybaseDataSource",
         "azure_blob": "AzureBlobDataSource",
+        "duck_db": "DuckDb",
     }
 
     def __init__(self, config: Configuration):

dcs_core/core/inspect.py

@@ -130,7 +130,6 @@ class Inspect:
         try:
             self.data_source_manager.connect()
             self.validation_manager.build_validations()
-
             validation_infos: Dict[str, ValidationInfo] = {}
 
             for datasource, _ in self.validation_manager.get_validations.items():

dcs_core/integrations/databases/azure_blob.py

@@ -13,10 +13,17 @@
 #  limitations under the License.
 
 import io
+import os
+import uuid
+from concurrent.futures import ThreadPoolExecutor
+from pathlib import Path
+from queue import Empty, Queue
 from typing import Any, Dict, List, Optional
 
+import duckdb
 import pandas as pd
 from azure.storage.blob import BlobServiceClient
+from loguru import logger
 
 from dcs_core.core.common.errors import (
     DatachecksColumnFetchError,
@@ -113,3 +120,98 @@ class AzureBlobDataSource(FileDataSource):
         """
         api_version = self.blob_service_client.api_version
         return api_version
+
+    def _chunk_load_to_pandas(self, queue: Queue, result_df: list, timeout: float = 2.0):
+        """Consumer thread: read CSV chunks from queue & build final DataFrame"""
+        df = pd.DataFrame()
+        try:
+            while True:
+                try:
+                    data = queue.get(timeout=timeout)
+                except Empty:
+                    continue
+
+                if data is None:
+                    break
+
+                try:
+                    chunk = pd.read_csv(io.BytesIO(data), dtype=str)
+                    df = pd.concat([df, chunk], ignore_index=True)
+                except Exception as e:
+                    logger.error(f"[ERROR] Failed to read CSV chunk: {e}")
+                    continue
+
+        except Exception as e:
+            logger.error(f"[FATAL] Consumer crashed: {e}")
+
+        finally:
+            result_df.append(df)
+
+    def _load_blob_to_pandas(self, table_name: str):
+        blob_client = self.connection.get_blob_client(blob=table_name)
+        CHUNK_SIZE = 4 * 1024 * 1024
+        blob_size = blob_client.get_blob_properties().size
+        start = 0
+        queue = Queue()
+        result_df = []
+
+        with ThreadPoolExecutor(max_workers=1) as executor:
+            executor.submit(self._chunk_load_to_pandas, queue, result_df)
+
+            all_data = b""
+            while start < blob_size:
+                end = min(start + CHUNK_SIZE - 1, blob_size - 1)
+                data = blob_client.download_blob(offset=start, length=end - start + 1).readall()
+                all_data += data
+                queue.put(data)
+                start += CHUNK_SIZE
+
+            queue.put(None)
+        if not result_df or len(result_df) == 0:
+            raise ValueError("No data downloaded from Azure Blob Storage")
+        return result_df[0]
+
+    def _load_pd_to_duckdb(self, df: pd.DataFrame, table_name: str):
+        dir_name = "tmp"
+        if not os.path.exists(dir_name):
+            os.makedirs(dir_name)
+
+        duck_db_file_name = f"{dir_name}/{uuid.uuid4()}.duckdb"
+        file_path = None
+        try:
+            table_name = table_name
+
+            conn = duckdb.connect(database=duck_db_file_name, read_only=False)
+
+            file_path = duck_db_file_name
+
+            conn.register("df_view", df)
+
+            conn.execute(
+                f"""
+                CREATE OR REPLACE TABLE "{table_name}" AS
+                SELECT * FROM df_view;
+                """
+            )
+            conn.unregister("df_view")
+            conn.close()
+
+        except Exception as e:
+            logger.error(f"Error in loading CSV to DuckDB: {e}")
+            raise
+
+        return file_path
+
+    def load_file_to_duckdb(self, table_name: str):
+        logger.info(f"Loading {table_name} to pandas")
+        df: pd.DataFrame = self._load_blob_to_pandas(table_name)
+
+        if df is None or df.empty:
+            raise ValueError("No data downloaded from Azure Blob Storage")
+
+        name_only = Path(table_name).stem
+
+        logger.info(f"Loading {table_name} to duckdb")
+        file_path = self._load_pd_to_duckdb(df, name_only)
+
+        return file_path

dcs_core/integrations/databases/duck_db.py

@@ -0,0 +1,72 @@
+#  Copyright 2022-present, the Waterdip Labs Pvt. Ltd.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+import os
+from typing import Any, Dict
+
+import duckdb
+from loguru import logger
+
+from dcs_core.core.common.errors import DataChecksDataSourcesConnectionError
+from dcs_core.core.datasource.sql_datasource import SQLDataSource
+
+
+class DuckDb(SQLDataSource):
+    def __init__(self, data_source_name: str, data_connection: Dict):
+        super().__init__(data_source_name, data_connection)
+        self.connection = None
+        self.use_sa_text_query = False
+
+    def connect(self) -> Any:
+        """
+        Connect to the file data source
+        """
+        try:
+            file_path = self.data_connection.get("file_path")
+            self.connection = duckdb.connect(database=file_path)
+            return self.connection
+        except Exception as e:
+            raise DataChecksDataSourcesConnectionError(f"Failed to connect to DuckDB: {e}")
+
+    def is_connected(self) -> bool:
+        """
+        Check if the file data source is connected
+        """
+        return self.connection is not None
+
+    def close(self):
+        """
+        Close the connection
+        """
+        logger.info("Closing DuckDB connection")
+        self.connection.close()
+        try:
+            os.remove(self.data_connection.get("file_path"))
+        except Exception as e:
+            logger.error(f"Failed to remove the file {self.data_connection.get('file_path')}: {e}")
+
+    def qualified_table_name(self, table_name: str) -> str:
+        """
+        Get the qualified table name
+        :param table_name: name of the table
+        :return: qualified table name
+        """
+        return f'"{table_name}"'
+
+    def quote_column(self, column: str) -> str:
+        """
+        Quote the column name
+        :param column: name of the column
+        :return: quoted column name
+        """
+        return f'"{column}"'

dcs_sdk/__version__.py

@@ -12,4 +12,4 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-__version__ = "1.6.9"
+__version__ = "1.7.0"

{dcs_sdk-1.6.9.dist-info → dcs_sdk-1.7.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dcs-sdk
-Version: 1.6.9
+Version: 1.7.0
 Summary: SDK for DataChecks
 Author: Waterdip Labs
 Author-email: hello@waterdip.ai
@@ -86,7 +86,7 @@ Requires-Dist: vertica-python (>=1.4.0) ; extra == "vertica" or extra == "all-db
 Description-Content-Type: text/markdown
 
 <h1 align="center">
-  DCS SDK v1.6.9
+  DCS SDK v1.7.0
 </h1>
 
 > SDK for DataChecks

{dcs_sdk-1.6.9.dist-info → dcs_sdk-1.7.0.dist-info}/RECORD

@@ -51,7 +51,7 @@ dcs_core/core/__init__.py,sha256=8XyOIsx-uCpaEZUgfOrb0DCdvmz1TipNQdz01h7mun0,761
 dcs_core/core/common/__init__.py,sha256=RkfhRKLXEForLCs4rZkTf0qc_b0TokSggSAcKI4yfZg,610
 dcs_core/core/common/errors.py,sha256=nRczSqORCjcDngAuDsqzsc3_yZQzuUX26lPov0pTE1I,2268
 dcs_core/core/common/models/__init__.py,sha256=RkfhRKLXEForLCs4rZkTf0qc_b0TokSggSAcKI4yfZg,610
-dcs_core/core/common/models/configuration.py,sha256=cFFr_SiAqYR3NIFGfz4rJVVX-LuGu-9TJC47ghL3Tes,9396
+dcs_core/core/common/models/configuration.py,sha256=PDmDZMkjSxKho8XIPrE0rF_SPe3CRt5KIXOezOBHiGg,9457
 dcs_core/core/common/models/dashboard.py,sha256=_WV1kbs4cKlFZ5QcXyMdTmDSZLYxhvZWWWQzvHReMxM,814
 dcs_core/core/common/models/data_source_resource.py,sha256=rNvj5NjvEQi2irHYjClKBFZbp70LTX9oGCPDeFURlAI,1559
 dcs_core/core/common/models/metric.py,sha256=0Oxp7YvWZVy7zbmi4u_opBDeknsuzXmnOrK01pP2fQw,4843
@@ -60,15 +60,15 @@ dcs_core/core/common/models/validation.py,sha256=yGSL-hZgvKqSgj0nqNIqUm_DmNVlKbl
 dcs_core/core/common/models/widget.py,sha256=-IaZ5dAmPPZwMvpzJDQfEINfIPUsqS5rufBak1c7Y6A,1083
 dcs_core/core/configuration/__init__.py,sha256=RkfhRKLXEForLCs4rZkTf0qc_b0TokSggSAcKI4yfZg,610
 dcs_core/core/configuration/config_loader.py,sha256=xySV5DIJC7a1VioE2sq5X8rSInW-4qF4hm5bT-wxVlc,5637
-dcs_core/core/configuration/configuration_parser.py,sha256=KGOJqWbOWhTacuMwM1N55Kh6Ug-WrrjYLAaH9a8ynRk,11347
+dcs_core/core/configuration/configuration_parser.py,sha256=ue7tzWkOpamhXw_DJhr5ZkqVKIEbP1AiZ2bQegsbdzg,11408
 dcs_core/core/configuration/configuration_parser_arc.py,sha256=TOoPf12pEXLdkjEGJEGV6rJOMR8yqLedla6T1x6g-Xw,14057
 dcs_core/core/datasource/__init__.py,sha256=RkfhRKLXEForLCs4rZkTf0qc_b0TokSggSAcKI4yfZg,610
 dcs_core/core/datasource/base.py,sha256=YD_UuGuoORFJNX30IQMk6aitiiTCHaiAddSNgUBmRtA,1935
-dcs_core/core/datasource/file_datasource.py,sha256=HG4av7KUFTfH2UlAl4bqcNI6MxpbSOA26cDqxmLUqh0,913
-dcs_core/core/datasource/manager.py,sha256=3oBjIqV0YYjXubCDGVBJP_jzrv-oBgBA-octoa8Wvaw,4795
+dcs_core/core/datasource/file_datasource.py,sha256=ppjGOtzSaBV3DWIute0dvOHQxtfULgeIefsDFW3xZz8,1017
+dcs_core/core/datasource/manager.py,sha256=cuh6XAOCxn2b9SQxYwYurgBb6WUD8ZS6KRIg3FAloYU,4824
 dcs_core/core/datasource/search_datasource.py,sha256=_conk1Q_kywJhKHYyEScoKlVt_yRd05zuAISvDmXqjw,15014
 dcs_core/core/datasource/sql_datasource.py,sha256=dlX-E--hadl2q8XpMNRyZmLGC35tltBsGDzlyZqzqtw,40730
-dcs_core/core/inspect.py,sha256=QICJKcEpQClLacsfNClFoiF08M01QnJh_U2VsXRh1iA,6427
+dcs_core/core/inspect.py,sha256=H5MCR_o71hz1lvkueSMAJ5C4R93hogrEWl04iuwgrdM,6426
 dcs_core/core/logger/__init__.py,sha256=RkfhRKLXEForLCs4rZkTf0qc_b0TokSggSAcKI4yfZg,610
 dcs_core/core/logger/base.py,sha256=dbW48Y1t6okXy1dx8GSH3KpE5HX6pebHGuKqacJWF0E,1024
 dcs_core/core/logger/default_logger.py,sha256=7yieM99xnepxq5FSlu0TEhVd55X_kUGRqjiio5loQuU,4097
@@ -100,10 +100,11 @@ dcs_core/core/validation/uniqueness_validation.py,sha256=a6zm0_omiULKbQcDit8J913
 dcs_core/core/validation/validity_validation.py,sha256=358oAGH112oVxyPhDnfT-ypVaMAkpZ8pM73qogtdh9w,35297
 dcs_core/integrations/__init__.py,sha256=RkfhRKLXEForLCs4rZkTf0qc_b0TokSggSAcKI4yfZg,610
 dcs_core/integrations/databases/__init__.py,sha256=RkfhRKLXEForLCs4rZkTf0qc_b0TokSggSAcKI4yfZg,610
-dcs_core/integrations/databases/azure_blob.py,sha256=rOPj-dv3ZaGUrr_rLMn8xjZXuEjlzcdfZv2RcZgnbps,4674
+dcs_core/integrations/databases/azure_blob.py,sha256=XjFLWuWYoWh5DGow1zgYypHvGtP7qN2oFwlvRth2aQ0,7973
 dcs_core/integrations/databases/bigquery.py,sha256=26RuypLMmiARZIWkV_mxtnNL2yCs94YWerSGH5Nr10Q,7337
 dcs_core/integrations/databases/databricks.py,sha256=n4fm5m_mtRCdtjLGDvbNW18u7Ev234vDBjq_lxuOxns,1978
 dcs_core/integrations/databases/db2.py,sha256=hNGivvYCitp88ouZlCxp7iRQ-vnPiK1kL8x85NyGotk,26492
+dcs_core/integrations/databases/duck_db.py,sha256=J4ReqFmUFfbYbZlvjmjZfHiD5CkgX4TnzqljGf5wJCQ,2422
 dcs_core/integrations/databases/elasticsearch.py,sha256=6CTGs1WGrfgdDRNVt9DpOB0_z_znT6YoVj10E1WY-wQ,2152
 dcs_core/integrations/databases/mssql.py,sha256=g0MmoG8-xFphJ2oZl-q_OZ2oT6yz-lVY09JTIvIx4-0,38910
 dcs_core/integrations/databases/mysql.py,sha256=mUFLIGdbF_ktIlA19P7kq7holp5ZkRezGgN6TL_uiJ4,15815
@@ -133,7 +134,7 @@ dcs_core/report/static/index.js,sha256=p4wvku-zlXi0y4gWeSzV1amY0s4mjtUq2QsezARLV
 dcs_core/report/static/index.js.LICENSE.txt,sha256=bBDZBJVEDrqjCi7sfoF8CchjFn3hdcbNkP7ub7kbcXQ,201041
 dcs_sdk/__init__.py,sha256=RkfhRKLXEForLCs4rZkTf0qc_b0TokSggSAcKI4yfZg,610
 dcs_sdk/__main__.py,sha256=Qn8stIaQGrdLjHQ-H7xO0T-brtq5RWZoWU9QvqoarV8,683
-dcs_sdk/__version__.py,sha256=iSCeuxA5501nrospfk7ajv7gQWKLpPSH4npFVobI-gY,633
+dcs_sdk/__version__.py,sha256=eNGN1UbpUzU92QJAufllEa8THGec2TPMG38N7GY-Rc8,633
 dcs_sdk/cli/__init__.py,sha256=RkfhRKLXEForLCs4rZkTf0qc_b0TokSggSAcKI4yfZg,610
 dcs_sdk/cli/cli.py,sha256=jaO52UrMWLafcF_yhqllPkmYSTuO2sksFi30fYFdAB4,4406
 dcs_sdk/sdk/__init__.py,sha256=skrZcgWWJBL6NXTUERywJ3qRJRemgpDXyW7lPg1FJk8,2107
@@ -155,7 +156,7 @@ dcs_sdk/sdk/utils/similarity_score/levenshtein_distance_provider.py,sha256=puAWP
 dcs_sdk/sdk/utils/table.py,sha256=X8HxdYTWyx_oVrBWPsXlmA-xJKXXDBW9RrhlWNqA1As,18224
 dcs_sdk/sdk/utils/themes.py,sha256=Meo2Yldv4uyPpEqI7qdA28Aa6sxtwUU1dLKKm4QavjM,1403
 dcs_sdk/sdk/utils/utils.py,sha256=a9QGEVL8L7asbJm_VBwgKvJQknsvuqWS0uTUaHsDPiY,16463
-dcs_sdk-1.6.9.dist-info/METADATA,sha256=Od43VgAhrNLKIICnGJ3nSzmuUYJzHl7miH6OSysVE9U,7652
-dcs_sdk-1.6.9.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
-dcs_sdk-1.6.9.dist-info/entry_points.txt,sha256=XhODNz7UccgPOyklXgp7pIfTTXArd6-V0mImjhnhwto,80
-dcs_sdk-1.6.9.dist-info/RECORD,,
+dcs_sdk-1.7.0.dist-info/METADATA,sha256=y4bCqzJK1nn3mIkDJ8uAQwfgHNR_Gytw_X3hic0Jhi0,7652
+dcs_sdk-1.7.0.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
+dcs_sdk-1.7.0.dist-info/entry_points.txt,sha256=XhODNz7UccgPOyklXgp7pIfTTXArd6-V0mImjhnhwto,80
+dcs_sdk-1.7.0.dist-info/RECORD,,