dcs-sdk 1.6.7__py3-none-any.whl → 1.6.9__py3-none-any.whl

dcs_sdk/__version__.py

@@ -12,4 +12,4 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-__version__ = "1.6.7"
+__version__ = "1.6.9"

dcs_sdk/sdk/data_diff/data_differ.py

@@ -267,6 +267,10 @@ class DBTableDiffer:
                     ) from e
 
         except Exception as e:
+            self.cleanup_duckdb(
+                src=self.config.source.filepath,
+                target=self.config.target.filepath,
+            )
             raise RuntimeError(f"process_duckdb failed for {'source' if is_source else 'target'}: {e}") from e
 
     def _prepare_source_table(self) -> Optional[str]:
@@ -785,8 +789,10 @@ class DBTableDiffer:
                 with suppress(Exception):
                     db_connection.close()
 
-        safe_close(self.table1.database)
-        safe_close(self.table2.database)
+        if hasattr(self.table1, "database"):
+            safe_close(self.table1.database)
+        if hasattr(self.table2, "database"):
+            safe_close(self.table2.database)
 
         if self.source_db:
             self.source_db.drop_view_from_db(

dcs_sdk/sdk/utils/utils.py

@@ -158,7 +158,8 @@ def chunk_load_to_pandas(queue: Queue, result_df: list, timeout: float = 2.0):
                 break
 
             try:
-                df = pd.concat([df, pd.read_csv(io.BytesIO(data))], ignore_index=True)
+                chunk = pd.read_csv(io.BytesIO(data), dtype=str)
+                df = pd.concat([df, chunk], ignore_index=True)
             except Exception as e:
                 logger.error(f"[ERROR] Failed to read CSV chunk: {e}")
                 continue
@@ -232,21 +233,25 @@ def duck_db_load_pd_to_table(config: Comparison, is_source: bool = False, df: pd
         table_name = config.source.table if is_source else config.target.table
 
         conn = duckdb.connect(database=duck_db_file_name, read_only=False)
+        if is_source:
+            config.source.filepath = duck_db_file_name
+        else:
+            config.target.filepath = duck_db_file_name
 
         conn.register("df_view", df)
 
         conn.execute(
             f"""
-            CREATE OR REPLACE TABLE {table_name} AS
+            CREATE OR REPLACE TABLE "{table_name}" AS
             SELECT * FROM df_view;
             """
         )
 
         if pk_cols and len(pk_cols) > 0:
-            pk_cols_str = ", ".join(pk_cols)
+            quoted_pk_cols = ", ".join([f'"{col}"' for col in pk_cols])
             conn.execute(
                 f"""
-                CREATE INDEX idx_{table_name} ON {table_name} ({pk_cols_str});
+                CREATE INDEX "idx_{table_name}" ON "{table_name}" ({quoted_pk_cols});
                 """
             )
 
@@ -254,7 +259,7 @@ def duck_db_load_pd_to_table(config: Comparison, is_source: bool = False, df: pd
             view_name = f"{table_name}_query"
             conn.execute(
                 f"""
-                CREATE VIEW {view_name} AS {query};
+                CREATE VIEW "{view_name}" AS {query};
                 """
             )
 
@@ -265,10 +270,6 @@ def duck_db_load_pd_to_table(config: Comparison, is_source: bool = False, df: pd
         logger.error(f"Error in loading CSV to DuckDB: {e}")
         return False
 
-    if is_source:
-        config.source.filepath = duck_db_file_name
-    else:
-        config.target.filepath = duck_db_file_name
     return True
 
 
@@ -298,6 +299,12 @@ def duck_db_load_csv_to_table(config: Comparison, path, is_source: bool = False)
         try:
             table_name = generate_table_name(csv_file)
             conn = duckdb.connect(database=duck_db_file_name, read_only=False)
+            if is_source:
+                config.source.filepath = duck_db_file_name
+                config.source.table = table_name
+            else:
+                config.target.filepath = duck_db_file_name
+                config.target.table = table_name
             conn.execute(
                 """
                     CREATE OR REPLACE TABLE {} AS SELECT * FROM read_csv('{}',HEADER=True, UNION_BY_NAME=True, nullstr='NULL', all_varchar=True, IGNORE_ERRORS=TRUE);
@@ -307,14 +314,14 @@ def duck_db_load_csv_to_table(config: Comparison, path, is_source: bool = False)
             )
 
             if pk_cols and len(pk_cols) > 0:
-                pk_cols_str = ", ".join(pk_cols)
+                quoted_pk_cols = ", ".join(['"{}"'.format(col) for col in pk_cols])
                 conn.execute(
                     """
-                    CREATE INDEX idx_{} ON {} ({});
+                    CREATE INDEX "idx_{}" ON "{}" ({});
                     """.format(
                         table_name,
                         table_name,
-                        pk_cols_str,
+                        quoted_pk_cols,
                     )
                 )
 
@@ -322,7 +329,7 @@ def duck_db_load_csv_to_table(config: Comparison, path, is_source: bool = False)
                 table_name = f"{table_name}_query"
                 conn.execute(
                     """
-                    CREATE VIEW {} AS {};
+                    CREATE VIEW "{}" AS {};
                     """.format(
                         table_name, query
                     )
@@ -332,12 +339,6 @@ def duck_db_load_csv_to_table(config: Comparison, path, is_source: bool = False)
             logger.error(f"Error in loading CSV to DuckDB: {e}")
             return False
 
-    if is_source:
-        config.source.filepath = duck_db_file_name
-        config.source.table = table_name
-    else:
-        config.target.filepath = duck_db_file_name
-        config.target.table = table_name
     return True
 
 

{dcs_sdk-1.6.7.dist-info → dcs_sdk-1.6.9.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dcs-sdk
-Version: 1.6.7
+Version: 1.6.9
 Summary: SDK for DataChecks
 Author: Waterdip Labs
 Author-email: hello@waterdip.ai
@@ -86,12 +86,11 @@ Requires-Dist: vertica-python (>=1.4.0) ; extra == "vertica" or extra == "all-db
 Description-Content-Type: text/markdown
 
 <h1 align="center">
-  DCS SDK v1.6.7
+  DCS SDK v1.6.9
 </h1>
 
 > SDK for DataChecks
 
-
 ## Installation
 
 > Python version `>=3.10,<3.13`
@@ -108,24 +107,19 @@ $ pip install dcs-sdk[all-dbs]
 
 | Database          | Code Name    | Supported |
 | ----------------- | ------------ | --------- |
-| PostgreSQL        | `postgres`   | ✅         |
-| Snowflake         | `snowflake`  | ✅         |
-| Trino             | `trino`      | ✅         |
-| Databricks        | `databricks` | ✅         |
-| Oracle            | `oracle`     | ✅         |
-| MSSQL             | `mssql`      | ✅         |
-| MySQL             | `mysql`      | ✅         |
-| SAP Sybase IQ/ASE | `sybase`     | ✅         |
-| File              | `file`       | ✅         |
-| BigQuery          | `bigquery`   | ✅         |
-
-
-
+| PostgreSQL        | `postgres`   | ✅        |
+| Snowflake         | `snowflake`  | ✅        |
+| Trino             | `trino`      | ✅        |
+| Databricks        | `databricks` | ✅        |
+| Oracle            | `oracle`     | ✅        |
+| MSSQL             | `mssql`      | ✅        |
+| MySQL             | `mysql`      | ✅        |
+| SAP Sybase IQ/ASE | `sybase`     | ✅        |
+| File              | `file`       | ✅        |
+| BigQuery          | `bigquery`   | ✅        |
 
 ## Available Commands
 
-
-
 |    Option     | Short Option | Required |     Default     |                    Description                     |                                                 Example                                                  |
 | :-----------: | :----------: | :------: | :-------------: | :------------------------------------------------: | :------------------------------------------------------------------------------------------------------: |
 | --config-path |      -C      | **Yes**  |      None       |    Specify the file path for the configuration     |                        dcs-sdk run --config-path config.yaml --compare comp_name                         |
@@ -138,8 +132,6 @@ $ pip install dcs-sdk[all-dbs]
 | --report-path |              |    No    | dcs_report.html |       Specify the file path for HTML report        |     dcs-sdk run --config-path config.yaml --compare comp_name --html-report --report-path table.html     |
 |    --table    |              |    No    |      False      |         Display Comparison in table format         | dcs-sdk run --config-path config.yaml --compare comp_name --html-report --report-path table.html --table |
 
-
-
 ### Example Command [CLI]
 
 ```sh

{dcs_sdk-1.6.7.dist-info → dcs_sdk-1.6.9.dist-info}/RECORD

@@ -133,14 +133,14 @@ dcs_core/report/static/index.js,sha256=p4wvku-zlXi0y4gWeSzV1amY0s4mjtUq2QsezARLV
 dcs_core/report/static/index.js.LICENSE.txt,sha256=bBDZBJVEDrqjCi7sfoF8CchjFn3hdcbNkP7ub7kbcXQ,201041
 dcs_sdk/__init__.py,sha256=RkfhRKLXEForLCs4rZkTf0qc_b0TokSggSAcKI4yfZg,610
 dcs_sdk/__main__.py,sha256=Qn8stIaQGrdLjHQ-H7xO0T-brtq5RWZoWU9QvqoarV8,683
-dcs_sdk/__version__.py,sha256=_MZd1Vn40uGUurqdUiub-zDoYZlEiNqfaaWWlEJhxps,633
+dcs_sdk/__version__.py,sha256=iSCeuxA5501nrospfk7ajv7gQWKLpPSH4npFVobI-gY,633
 dcs_sdk/cli/__init__.py,sha256=RkfhRKLXEForLCs4rZkTf0qc_b0TokSggSAcKI4yfZg,610
 dcs_sdk/cli/cli.py,sha256=jaO52UrMWLafcF_yhqllPkmYSTuO2sksFi30fYFdAB4,4406
 dcs_sdk/sdk/__init__.py,sha256=skrZcgWWJBL6NXTUERywJ3qRJRemgpDXyW7lPg1FJk8,2107
 dcs_sdk/sdk/config/__init__.py,sha256=RkfhRKLXEForLCs4rZkTf0qc_b0TokSggSAcKI4yfZg,610
 dcs_sdk/sdk/config/config_loader.py,sha256=ZbSGQ56LsHv4_mxNhYrf6eoegO2R4PaqAs8iAghU73M,22435
 dcs_sdk/sdk/data_diff/__init__.py,sha256=RkfhRKLXEForLCs4rZkTf0qc_b0TokSggSAcKI4yfZg,610
-dcs_sdk/sdk/data_diff/data_differ.py,sha256=00lKfGU4xMeXuS_Wpvjf-TAgMiZ7r5_bv1EQsv1EdjQ,39050
+dcs_sdk/sdk/data_diff/data_differ.py,sha256=3uef0ED5TzMmXcJYx3OZtmS-K4tj_yjlsvkD9aoIFOc,39296
 dcs_sdk/sdk/rules/__init__.py,sha256=_BkKcE_jfdDQI_ECdOamJaefMKEXrKpYjPpnBQXl_Xs,657
 dcs_sdk/sdk/rules/rules_mappping.py,sha256=fxakVkf7B2cVkYSO946LTim_HmMsl6lBDBqZjTTsSPI,1292
 dcs_sdk/sdk/rules/rules_repository.py,sha256=x0Rli-wdnHAmXm5526go_qC3P-eFRt-4L7fs4hNqC-g,7564
@@ -154,8 +154,8 @@ dcs_sdk/sdk/utils/similarity_score/jaccard_provider.py,sha256=Jd0TvIGOULNTsiCL_F
 dcs_sdk/sdk/utils/similarity_score/levenshtein_distance_provider.py,sha256=puAWPnoWfNo4BN4-kXIUHrtrt5jLv3Vkw_NfHvjYrn4,1185
 dcs_sdk/sdk/utils/table.py,sha256=X8HxdYTWyx_oVrBWPsXlmA-xJKXXDBW9RrhlWNqA1As,18224
 dcs_sdk/sdk/utils/themes.py,sha256=Meo2Yldv4uyPpEqI7qdA28Aa6sxtwUU1dLKKm4QavjM,1403
-dcs_sdk/sdk/utils/utils.py,sha256=i-oEiSSs8DlBni9fwBTwXEnrl8FLxzZ0dLtUGxVwLWU,16276
-dcs_sdk-1.6.7.dist-info/METADATA,sha256=0kvVpF3PedRyXnaMEYfGnD73wdulAv28pr5b8edzjUQ,7670
-dcs_sdk-1.6.7.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
-dcs_sdk-1.6.7.dist-info/entry_points.txt,sha256=XhODNz7UccgPOyklXgp7pIfTTXArd6-V0mImjhnhwto,80
-dcs_sdk-1.6.7.dist-info/RECORD,,
+dcs_sdk/sdk/utils/utils.py,sha256=a9QGEVL8L7asbJm_VBwgKvJQknsvuqWS0uTUaHsDPiY,16463
+dcs_sdk-1.6.9.dist-info/METADATA,sha256=Od43VgAhrNLKIICnGJ3nSzmuUYJzHl7miH6OSysVE9U,7652
+dcs_sdk-1.6.9.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
+dcs_sdk-1.6.9.dist-info/entry_points.txt,sha256=XhODNz7UccgPOyklXgp7pIfTTXArd6-V0mImjhnhwto,80
+dcs_sdk-1.6.9.dist-info/RECORD,,