dcs-sdk 1.4.8__py3-none-any.whl → 1.4.9__py3-none-any.whl

data_diff/databases/base.py

@@ -54,6 +54,7 @@ from typing_extensions import Self
 from data_diff.abcs.compiler import AbstractCompiler, Compilable
 from data_diff.abcs.database_types import (
     JSON,
+    ArithAlphanumeric,
     Array,
     Boolean,
     ColType,
@@ -753,6 +754,8 @@ class BaseDialect(abc.ABC):
             return f"'{v.decode()}'"
         elif isinstance(v, Code):
             return v.code
+        elif isinstance(v, ArithAlphanumeric):
+            return f"'{v._str}'"
         return repr(v)
 
     def constant_values(self, rows) -> str:

data_diff/hashdiff_tables.py

@@ -522,7 +522,7 @@ class HashDiffer(TableDiffer):
 
         if count1 == 0 and count2 == 0:
             logger.debug(
-                "Uneven distribution of keys detected in segment %s..%s (big gaps in the key column). "
+                "Uneven distribution of keys detected in segment {}..{} (big gaps in the key column). "
                 "For better performance, we recommend to increase the bisection-threshold.",
                 table1.min_key,
                 table1.max_key,

data_diff/table_segment.py

@@ -14,6 +14,7 @@
 
 import logging
 import time
+from decimal import Decimal
 from itertools import product
 from typing import Container, Dict, List, Optional, Sequence, Tuple
 
@@ -24,7 +25,18 @@ from typing_extensions import Self
 
 from data_diff.abcs.database_types import DbKey, DbPath, DbTime, IKey, NumericType
 from data_diff.databases.base import Database
-from data_diff.queries.api import SKIP, Code, Count, Expr, max_, min_, table, this
+from data_diff.queries.api import (
+    SKIP,
+    Code,
+    Count,
+    Expr,
+    and_,
+    max_,
+    min_,
+    or_,
+    table,
+    this,
+)
 from data_diff.queries.extras import (
     ApplyFuncAndNormalizeAsString,
     Checksum,
@@ -54,6 +66,10 @@ def split_key_space(min_key: DbKey, max_key: DbKey, count: int) -> List[DbKey]:
         assert type(min_key) is type(max_key)
         checkpoints = min_key.range(max_key, count)
     else:
+        if isinstance(min_key, Decimal):
+            min_key = float(min_key)
+        if isinstance(max_key, Decimal):
+            max_key = float(max_key)
         checkpoints = split_space(min_key, max_key, count)
 
     assert all(min_key < x < max_key for x in checkpoints)
@@ -288,17 +304,65 @@ class TableSegment:
 
         return result
 
-    def get_sample_data(self, limit: int = 100) -> list:
-        "Download all the relevant values of the segment from the database"
+    # def get_sample_data(self, limit: int = 100) -> list:
+    #     "Download all the relevant values of the segment from the database"
+
+    #     exprs = []
+    #     for c in self.key_columns:
+    #         quoted = self.database.dialect.quote(c)
+    #         exprs.append(NormalizeAsString(Code(quoted), self._schema[c]))
+    #     if self.where:
+    #         select = self.source_table.select(*self._relevant_columns_repr).where(Code(self._where())).limit(limit)
+    #         self.key_columns
+    #     else:
+    #         select = self.source_table.select(*self._relevant_columns_repr).limit(limit)
+
+    #     start_time = time.monotonic()
+    #     result = self.database.query(select, List[Tuple])
+    #     query_time_ms = (time.monotonic() - start_time) * 1000
+    #     self._update_stats("row_fetch_queries_stats", query_time_ms)
+
+    def get_sample_data(self, limit: int = 100, sample_keys: Optional[List[List[DbKey]]] = None) -> list:
+        """
+        Download relevant values of the segment from the database.
+        If `sample_keys` is provided, it filters rows matching those composite keys.
+
+        Parameters:
+            limit (int): Maximum number of rows to return (default: 100).
+            sample_keys (Optional[List[List[DbKey]]]): List of composite keys to filter rows.
+                Each inner list must match the number of key_columns.
+
+        Returns:
+            list: List of tuples containing the queried row data.
+        """
+        select = self.make_select().select(*self._relevant_columns_repr)
 
-        exprs = []
-        for c in self.key_columns:
-            quoted = self.database.dialect.quote(c)
-            exprs.append(NormalizeAsString(Code(quoted), self._schema[c]))
-        if self.where:
-            select = self.source_table.select(*self._relevant_columns_repr).where(Code(self._where())).limit(limit)
+        filters = []
+
+        if sample_keys:
+            key_exprs = []
+            for key_values in sample_keys:
+                and_exprs = []
+                for col, val in safezip(self.key_columns, key_values):
+                    quoted = self.database.dialect.quote(col)
+                    schema = self._schema[col]
+                    if val is None:
+                        and_exprs.append(Code(quoted + " IS NULL"))
+                        continue
+                    mk_v = schema.make_value(val)
+                    constant_val = self.database.dialect._constant_value(mk_v)
+                    where_expr = f"{quoted} = {constant_val}"
+                    and_exprs.append(Code(where_expr))
+                if and_exprs:
+                    key_exprs.append(and_(*and_exprs))
+            if key_exprs:
+                filters.append(or_(*key_exprs))
+        if filters or self.where:
+            select = select.where(*filters)
         else:
-            select = self.source_table.select(*self._relevant_columns_repr).limit(limit)
+            logger.warning("No filters applied; fetching up to {} rows without key restrictions", limit)
+
+        select = select.limit(limit)
 
         start_time = time.monotonic()
         result = self.database.query(select, List[Tuple])

data_diff/utils.py

@@ -482,6 +482,9 @@ def number_to_human(n):
 
 
 def split_space(start, end, count) -> List[int]:
+    if isinstance(start, float) or isinstance(end, float):
+        step = (end - start) / (count + 1)
+        return [start + step * i for i in range(1, count + 1)]
     size = end - start
     assert count <= size, (count, size)
     return list(range(start, end, (size + 1) // (count + 1)))[1 : count + 1]

dcs_sdk/__version__.py

@@ -12,4 +12,4 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-__version__ = "1.4.8"
+__version__ = "1.4.9"

dcs_sdk/sdk/data_diff/data_differ.py

@@ -395,8 +395,10 @@ class DBTableDiffer:
                     error_message = f"Target table '{target_dataset.get('table_name')}' is empty"
                 is_table_empty = True
             if not is_table_empty and not self.config.schema_diff:
+                pks_len = len(self.table1.key_columns)
                 table_1_sample_data = self.table1.with_schema().get_sample_data(limit=100)
-                table_2_sample_data = self.table2.with_schema().get_sample_data(limit=100)
+                sample_keys = [list(row[:pks_len]) for row in table_1_sample_data]
+                table_2_sample_data = self.table2.with_schema().get_sample_data(limit=100, sample_keys=sample_keys)
                 self.diff_iter = diff_tables(
                     self.table1,
                     self.table2,
@@ -598,9 +600,14 @@ class DBTableDiffer:
                     columns_order_wise=columns_order_wise_target if columns_order_wise_target else [],
                 )
 
-                sample_value_column_names = list(self.table1.key_columns) + list(self.table1.extra_columns)
-                sample_value_source_dicts = [dict(zip(sample_value_column_names, row)) for row in table_1_sample_data]
-                sample_value_target_dicts = [dict(zip(sample_value_column_names, row)) for row in table_2_sample_data]
+                sample_value_column_names_src = list(self.table1.key_columns) + list(self.table1.extra_columns)
+                sample_value_column_names_tgt = list(self.table2.key_columns) + list(self.table2.extra_columns)
+                sample_value_source_dicts = [
+                    dict(zip(sample_value_column_names_src, row)) for row in table_1_sample_data
+                ]
+                sample_value_target_dicts = [
+                    dict(zip(sample_value_column_names_tgt, row)) for row in table_2_sample_data
+                ]
 
                 def get_pk(row, key_columns):
                     return tuple(row[k] for k in key_columns)

{dcs_sdk-1.4.8.dist-info → dcs_sdk-1.4.9.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: dcs-sdk
-Version: 1.4.8
+Version: 1.4.9
 Summary: SDK for DataChecks
 Author: Waterdip Labs
 Author-email: hello@waterdip.ai
@@ -60,7 +60,7 @@ Requires-Dist: vertica-python (>=1.4.0) ; extra == "vertica" or extra == "all-db
 Description-Content-Type: text/markdown
 
 <h1 align="center">
-  DCS SDK v1.4.8
+  DCS SDK v1.4.9
 </h1>
 
 > SDK for DataChecks

{dcs_sdk-1.4.8.dist-info → dcs_sdk-1.4.9.dist-info}/RECORD

@@ -6,7 +6,7 @@ data_diff/abcs/database_types.py,sha256=dHE6K6UtqFwX6LTjOqJu1OOb_XJBXODeMcZQ0cds
 data_diff/config.py,sha256=uRcoVVhPjVZqgQNwr18v6sPq06cGXDLemTUyitU57zA,4998
 data_diff/databases/__init__.py,sha256=NrBm1Paj7jkHZ_hQCD-4-Q1eeDdh3v9_bz1DkPDOv9g,1680
 data_diff/databases/_connect.py,sha256=nGsmtzDSKN8CK8zMkdcGZz0iExzkJDYw-PGebIkmQgc,11151
-data_diff/databases/base.py,sha256=X3i59bJwZJSSAZl9HaL3NQtAmx04zSoA-E-YpBWWWng,51801
+data_diff/databases/base.py,sha256=Fd_OzWOL97YiTXE_oYsBthXVVLvJGbU5Y4JgR77Sra0,51904
 data_diff/databases/bigquery.py,sha256=PDwSkmWRW26gUl2SMOyIsiYtgrqghptYqG8_SaaiXb4,14709
 data_diff/databases/clickhouse.py,sha256=5DsW8UpyYsWI8I3AlPUvHWYdWdWpqzRsvsVwgqEyaLw,7554
 data_diff/databases/databricks.py,sha256=6cdwfAspg1GIgWFlQByvFcW_Hz1mnJeI9_2kZhOP8b4,9334
@@ -24,7 +24,7 @@ data_diff/databases/vertica.py,sha256=2dSDZp6qOEvUVPldI5Tgn7Sm3dCpC3vNXJL3qb3FDv
 data_diff/diff_tables.py,sha256=Ey88gUr9Wh8UVsgRlBCY3CACIYfHL52PxJSrd821aqg,20060
 data_diff/errors.py,sha256=4Yru8yOwyuDuBlTABnGCvJMSpe6-rbLJpNnVHeTTyHU,745
 data_diff/format.py,sha256=QFDjdZaBVf_N-jfKiX4ppOUdpXTPZXmv1j0pc1RiOoc,10245
-data_diff/hashdiff_tables.py,sha256=1hIO6rsUHHAR5sgI-UaKf7xXbqLsXVTvnhPjRqSNeIE,27993
+data_diff/hashdiff_tables.py,sha256=K-JoMimAwACB3mqQLIdv5P5joYWyRMSZKcbDJWz5dlk,27993
 data_diff/info_tree.py,sha256=yHtFSoXuu6oBafLYOYQjUSKlB-DnAAd08U9HOEAdTPI,2799
 data_diff/joindiff_tables.py,sha256=fyrEYjyh2BX1vGibwVZLYM1V6JJTOY-uGXY-KInvMkw,17612
 data_diff/lexicographic_space.py,sha256=bBoCbbH1Mla9jNOq1b5RuwjAxSVU7gWkra673tPBwXQ,8305
@@ -37,20 +37,20 @@ data_diff/queries/base.py,sha256=pT-iaII7Nlu-w-Cuq9fhoNKX7-GSxkQ3Fk8K-tMkk60,964
 data_diff/queries/extras.py,sha256=aUm-ifj3BMlz4o4bbuHtmnvHZuptYAKGS5yWTHmNpvc,1270
 data_diff/query_utils.py,sha256=R7ZfRwcvv9Zf4zWXNln4tr_OxLmDI7CPmmCahYfHxlo,2101
 data_diff/schema.py,sha256=QoYSSB3k-svLXz680uRgsI4qjii8BFKOOQvheqtgEbs,2413
-data_diff/table_segment.py,sha256=W6S9kebBoSfglbsbyiWprMy3AM9oFgH-_CsJrjIgEbc,19018
+data_diff/table_segment.py,sha256=-zYdppv8lW8ZfDRzZvmARm1fYxGHhAYPurR0r0GmWh8,21350
 data_diff/thread_utils.py,sha256=_692ERjnWfHKaZsLdg7CNfkKiRd66y7_kpgDwzntp44,3831
-data_diff/utils.py,sha256=kOTSkS4L1TO3Crx5mclP9E4RgGlc7z2RwzSRmANwrzA,23163
+data_diff/utils.py,sha256=IHKaFfyizOiA1By0sfnTra7CJvgEDwr5KPGjev0kx0A,23328
 data_diff/version.py,sha256=Wk0ovyBlLEF2UaWLWEcVBLFElREtIxi7TU1hD3CuTFI,634
 dcs_sdk/__init__.py,sha256=RkfhRKLXEForLCs4rZkTf0qc_b0TokSggSAcKI4yfZg,610
 dcs_sdk/__main__.py,sha256=Qn8stIaQGrdLjHQ-H7xO0T-brtq5RWZoWU9QvqoarV8,683
-dcs_sdk/__version__.py,sha256=mTRqCxtg196g8dlefiZhyJ3s3O3JtAAx9-0RLHgGFDQ,633
+dcs_sdk/__version__.py,sha256=lS23jYuOvSiv7SJ1XHD4d1O93eassDRAcs2EuasDZ-0,633
 dcs_sdk/cli/__init__.py,sha256=RkfhRKLXEForLCs4rZkTf0qc_b0TokSggSAcKI4yfZg,610
 dcs_sdk/cli/cli.py,sha256=LyrRk972OL9pTqrvBeXWBu5rUDAN17lQ1g8FdSRW_8M,4299
 dcs_sdk/sdk/__init__.py,sha256=skrZcgWWJBL6NXTUERywJ3qRJRemgpDXyW7lPg1FJk8,2107
 dcs_sdk/sdk/config/__init__.py,sha256=RkfhRKLXEForLCs4rZkTf0qc_b0TokSggSAcKI4yfZg,610
 dcs_sdk/sdk/config/config_loader.py,sha256=Sg0gvleGrOcDaX41amgw6Uj6c_s4Zel6gQalIp9CltI,21416
 dcs_sdk/sdk/data_diff/__init__.py,sha256=RkfhRKLXEForLCs4rZkTf0qc_b0TokSggSAcKI4yfZg,610
-dcs_sdk/sdk/data_diff/data_differ.py,sha256=9Kl35REucew02V0WBWTTj6zM2TUlOrA7gJ4P8WGGb0Q,35212
+dcs_sdk/sdk/data_diff/data_differ.py,sha256=8WBDz-q-Li5OyJcG_a6JRnDPCJBvIsMki-DrD4EgDzQ,35575
 dcs_sdk/sdk/rules/__init__.py,sha256=_BkKcE_jfdDQI_ECdOamJaefMKEXrKpYjPpnBQXl_Xs,657
 dcs_sdk/sdk/rules/rules_mappping.py,sha256=fxakVkf7B2cVkYSO946LTim_HmMsl6lBDBqZjTTsSPI,1292
 dcs_sdk/sdk/rules/rules_repository.py,sha256=x0Rli-wdnHAmXm5526go_qC3P-eFRt-4L7fs4hNqC-g,7564
@@ -65,7 +65,7 @@ dcs_sdk/sdk/utils/similarity_score/levenshtein_distance_provider.py,sha256=puAWP
 dcs_sdk/sdk/utils/table.py,sha256=mXArkEYmxf7mz_DJvyHVHIN9RyxaF708I70n52JrptA,18268
 dcs_sdk/sdk/utils/themes.py,sha256=Meo2Yldv4uyPpEqI7qdA28Aa6sxtwUU1dLKKm4QavjM,1403
 dcs_sdk/sdk/utils/utils.py,sha256=vF2zAvgt__Y8limicWTEWRyn41SBVJN81ZCTBRy6hQg,11907
-dcs_sdk-1.4.8.dist-info/METADATA,sha256=csButDjidmwzDZrtJtyz0DT-KvisbwRxg2jUBuoMCv4,6221
-dcs_sdk-1.4.8.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
-dcs_sdk-1.4.8.dist-info/entry_points.txt,sha256=zQtrZL7YuaKtt6WPwihCTV1BRXnqBkaY6zUGdYJbBSg,49
-dcs_sdk-1.4.8.dist-info/RECORD,,
+dcs_sdk-1.4.9.dist-info/METADATA,sha256=JUBldmhqJC8gu9G2eftTjkcf8rgWsPgsPQkk5tlkHWw,6221
+dcs_sdk-1.4.9.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+dcs_sdk-1.4.9.dist-info/entry_points.txt,sha256=zQtrZL7YuaKtt6WPwihCTV1BRXnqBkaY6zUGdYJbBSg,49
+dcs_sdk-1.4.9.dist-info/RECORD,,