collate-data-diff 0.11.17__tar.gz → 0.11.19__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {collate_data_diff-0.11.17 → collate_data_diff-0.11.19}/PKG-INFO +5 -3
- {collate_data_diff-0.11.17 → collate_data_diff-0.11.19}/data_diff/__main__.py +5 -1
- {collate_data_diff-0.11.17 → collate_data_diff-0.11.19}/data_diff/abcs/database_types.py +2 -0
- {collate_data_diff-0.11.17 → collate_data_diff-0.11.19}/data_diff/databases/_connect.py +7 -1
- {collate_data_diff-0.11.17 → collate_data_diff-0.11.19}/data_diff/databases/base.py +7 -1
- {collate_data_diff-0.11.17 → collate_data_diff-0.11.19}/data_diff/databases/duckdb.py +5 -0
- {collate_data_diff-0.11.17 → collate_data_diff-0.11.19}/data_diff/databases/mssql.py +70 -13
- {collate_data_diff-0.11.17 → collate_data_diff-0.11.19}/data_diff/databases/mysql.py +9 -0
- {collate_data_diff-0.11.17 → collate_data_diff-0.11.19}/data_diff/databases/oracle.py +2 -1
- {collate_data_diff-0.11.17 → collate_data_diff-0.11.19}/data_diff/databases/postgresql.py +7 -6
- {collate_data_diff-0.11.17 → collate_data_diff-0.11.19}/data_diff/databases/presto.py +5 -0
- {collate_data_diff-0.11.17 → collate_data_diff-0.11.19}/data_diff/databases/snowflake.py +2 -1
- {collate_data_diff-0.11.17 → collate_data_diff-0.11.19}/data_diff/databases/trino.py +19 -1
- {collate_data_diff-0.11.17 → collate_data_diff-0.11.19}/data_diff/diff_tables.py +32 -8
- {collate_data_diff-0.11.17 → collate_data_diff-0.11.19}/data_diff/hashdiff_tables.py +65 -2
- {collate_data_diff-0.11.17 → collate_data_diff-0.11.19}/data_diff/joindiff_tables.py +18 -20
- {collate_data_diff-0.11.17 → collate_data_diff-0.11.19}/data_diff/table_segment.py +8 -2
- {collate_data_diff-0.11.17 → collate_data_diff-0.11.19}/pyproject.toml +8 -3
- {collate_data_diff-0.11.17 → collate_data_diff-0.11.19}/LICENSE +0 -0
- {collate_data_diff-0.11.17 → collate_data_diff-0.11.19}/README.md +0 -0
- {collate_data_diff-0.11.17 → collate_data_diff-0.11.19}/data_diff/__init__.py +0 -0
- {collate_data_diff-0.11.17 → collate_data_diff-0.11.19}/data_diff/abcs/__init__.py +0 -0
- {collate_data_diff-0.11.17 → collate_data_diff-0.11.19}/data_diff/abcs/compiler.py +0 -0
- {collate_data_diff-0.11.17 → collate_data_diff-0.11.19}/data_diff/cloud/__init__.py +0 -0
- {collate_data_diff-0.11.17 → collate_data_diff-0.11.19}/data_diff/cloud/data_source.py +0 -0
- {collate_data_diff-0.11.17 → collate_data_diff-0.11.19}/data_diff/cloud/datafold_api.py +0 -0
- {collate_data_diff-0.11.17 → collate_data_diff-0.11.19}/data_diff/config.py +0 -0
- {collate_data_diff-0.11.17 → collate_data_diff-0.11.19}/data_diff/databases/__init__.py +0 -0
- {collate_data_diff-0.11.17 → collate_data_diff-0.11.19}/data_diff/databases/bigquery.py +0 -0
- {collate_data_diff-0.11.17 → collate_data_diff-0.11.19}/data_diff/databases/clickhouse.py +0 -0
- {collate_data_diff-0.11.17 → collate_data_diff-0.11.19}/data_diff/databases/databricks.py +0 -0
- {collate_data_diff-0.11.17 → collate_data_diff-0.11.19}/data_diff/databases/redshift.py +0 -0
- {collate_data_diff-0.11.17 → collate_data_diff-0.11.19}/data_diff/databases/vertica.py +0 -0
- {collate_data_diff-0.11.17 → collate_data_diff-0.11.19}/data_diff/dbt.py +0 -0
- {collate_data_diff-0.11.17 → collate_data_diff-0.11.19}/data_diff/dbt_config_validators.py +0 -0
- {collate_data_diff-0.11.17 → collate_data_diff-0.11.19}/data_diff/dbt_parser.py +0 -0
- {collate_data_diff-0.11.17 → collate_data_diff-0.11.19}/data_diff/errors.py +0 -0
- {collate_data_diff-0.11.17 → collate_data_diff-0.11.19}/data_diff/format.py +0 -0
- {collate_data_diff-0.11.17 → collate_data_diff-0.11.19}/data_diff/info_tree.py +0 -0
- {collate_data_diff-0.11.17 → collate_data_diff-0.11.19}/data_diff/lexicographic_space.py +0 -0
- {collate_data_diff-0.11.17 → collate_data_diff-0.11.19}/data_diff/parse_time.py +0 -0
- {collate_data_diff-0.11.17 → collate_data_diff-0.11.19}/data_diff/py.typed +0 -0
- {collate_data_diff-0.11.17 → collate_data_diff-0.11.19}/data_diff/queries/__init__.py +0 -0
- {collate_data_diff-0.11.17 → collate_data_diff-0.11.19}/data_diff/queries/api.py +0 -0
- {collate_data_diff-0.11.17 → collate_data_diff-0.11.19}/data_diff/queries/ast_classes.py +0 -0
- {collate_data_diff-0.11.17 → collate_data_diff-0.11.19}/data_diff/queries/base.py +0 -0
- {collate_data_diff-0.11.17 → collate_data_diff-0.11.19}/data_diff/queries/extras.py +0 -0
- {collate_data_diff-0.11.17 → collate_data_diff-0.11.19}/data_diff/query_utils.py +0 -0
- {collate_data_diff-0.11.17 → collate_data_diff-0.11.19}/data_diff/schema.py +0 -0
- {collate_data_diff-0.11.17 → collate_data_diff-0.11.19}/data_diff/thread_utils.py +0 -0
- {collate_data_diff-0.11.17 → collate_data_diff-0.11.19}/data_diff/tracking.py +0 -0
- {collate_data_diff-0.11.17 → collate_data_diff-0.11.19}/data_diff/utils.py +0 -0
- {collate_data_diff-0.11.17 → collate_data_diff-0.11.19}/data_diff/version.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: collate-data-diff
|
|
3
|
-
Version: 0.11.
|
|
3
|
+
Version: 0.11.19
|
|
4
4
|
Summary: Command-line tool and Python library to efficiently diff rows across two different databases.
|
|
5
5
|
License: MIT
|
|
6
6
|
License-File: LICENSE
|
|
@@ -26,6 +26,7 @@ Classifier: Topic :: Database :: Database Engines/Servers
|
|
|
26
26
|
Classifier: Typing :: Typed
|
|
27
27
|
Provides-Extra: all-dbs
|
|
28
28
|
Provides-Extra: clickhouse
|
|
29
|
+
Provides-Extra: dbt
|
|
29
30
|
Provides-Extra: duckdb
|
|
30
31
|
Provides-Extra: mssql
|
|
31
32
|
Provides-Extra: mysql
|
|
@@ -41,20 +42,21 @@ Requires-Dist: attrs (>=23.1.0)
|
|
|
41
42
|
Requires-Dist: click (>=8.3.3)
|
|
42
43
|
Requires-Dist: clickhouse-driver ; extra == "clickhouse" or extra == "all-dbs"
|
|
43
44
|
Requires-Dist: cryptography (>=46.0.5) ; extra == "snowflake" or extra == "all-dbs"
|
|
44
|
-
Requires-Dist: dbt-core (>=1.0.0,<2.0.0)
|
|
45
|
+
Requires-Dist: dbt-core (>=1.0.0,<2.0.0) ; extra == "dbt"
|
|
45
46
|
Requires-Dist: deepdiff (>=8.6.1,<9.0)
|
|
46
47
|
Requires-Dist: dsnparse (<0.2.0)
|
|
47
48
|
Requires-Dist: duckdb ; extra == "duckdb" or extra == "all-dbs"
|
|
48
49
|
Requires-Dist: keyring
|
|
49
|
-
Requires-Dist: mashumaro[msgpack] (>=2.9,<3.11.0)
|
|
50
50
|
Requires-Dist: mysql-connector-python (>=8.0.29) ; (python_version < "3.9") and (extra == "mysql" or extra == "all-dbs")
|
|
51
51
|
Requires-Dist: mysql-connector-python (>=9.1) ; (python_version >= "3.9") and (extra == "mysql" or extra == "all-dbs")
|
|
52
52
|
Requires-Dist: oracledb ; extra == "oracle" or extra == "all-dbs"
|
|
53
|
+
Requires-Dist: packaging
|
|
53
54
|
Requires-Dist: preql (>=0.2.19) ; extra == "preql" or extra == "all-dbs"
|
|
54
55
|
Requires-Dist: presto-python-client ; extra == "presto" or extra == "all-dbs"
|
|
55
56
|
Requires-Dist: psycopg2 ; extra == "postgresql" or extra == "redshift" or extra == "all-dbs"
|
|
56
57
|
Requires-Dist: pydantic (>=2.0,<3)
|
|
57
58
|
Requires-Dist: pyodbc (>=4.0.39) ; extra == "mssql" or extra == "all-dbs"
|
|
59
|
+
Requires-Dist: requests (>=2.32.4)
|
|
58
60
|
Requires-Dist: rich
|
|
59
61
|
Requires-Dist: snowflake-connector-python (>=4.4.0,<5.0.0) ; (python_version >= "3.10") and (extra == "snowflake" or extra == "all-dbs")
|
|
60
62
|
Requires-Dist: tabulate (>=0.9.0)
|
|
@@ -15,7 +15,6 @@ from rich.logging import RichHandler
|
|
|
15
15
|
from data_diff import Database, DbPath
|
|
16
16
|
from data_diff.config import apply_config_from_file
|
|
17
17
|
from data_diff.databases._connect import connect
|
|
18
|
-
from data_diff.dbt import dbt_diff
|
|
19
18
|
from data_diff.diff_tables import Algorithm, TableDiffer
|
|
20
19
|
from data_diff.hashdiff_tables import HashDiffer, DEFAULT_BISECTION_THRESHOLD, DEFAULT_BISECTION_FACTOR
|
|
21
20
|
from data_diff.joindiff_tables import TABLE_WRITE_LIMIT, JoinDiffer
|
|
@@ -322,6 +321,11 @@ def main(conf, run, **kw) -> None:
|
|
|
322
321
|
if project_dir_override:
|
|
323
322
|
project_dir_override = os.path.expanduser(project_dir_override)
|
|
324
323
|
if kw["dbt"]:
|
|
324
|
+
try:
|
|
325
|
+
from data_diff.dbt import dbt_diff
|
|
326
|
+
except ImportError as e:
|
|
327
|
+
raise click.UsageError("--dbt needs dbt-core: pip install 'collate-data-diff[dbt]'") from e
|
|
328
|
+
|
|
325
329
|
dbt_diff(
|
|
326
330
|
log_status_handler=log_handlers.get("log_status_handler"),
|
|
327
331
|
profiles_dir_override=profiles_dir_override,
|
|
@@ -205,6 +205,8 @@ class Decimal(FractionalType, IKey): # Snowflake may use Decimal as a key
|
|
|
205
205
|
class StringType(ColType):
|
|
206
206
|
python_type = str
|
|
207
207
|
collation: Optional[Collation] = attrs.field(default=None, kw_only=True)
|
|
208
|
+
# A fixed-width column (CHAR) the database pads with trailing spaces
|
|
209
|
+
padded: bool = attrs.field(default=False, kw_only=True)
|
|
208
210
|
|
|
209
211
|
|
|
210
212
|
@attrs.define(frozen=True)
|
|
@@ -306,11 +306,17 @@ class Connect_SetUTC(Connect):
|
|
|
306
306
|
def _connection_created(self, db):
|
|
307
307
|
db = super()._connection_created(db)
|
|
308
308
|
try:
|
|
309
|
-
db.
|
|
309
|
+
set_utc = db.dialect.set_timezone_to_utc()
|
|
310
310
|
except NotImplementedError:
|
|
311
311
|
logging.debug(
|
|
312
312
|
f"Database '{db}' does not allow setting timezone. We recommend making sure it's set to 'UTC'."
|
|
313
313
|
)
|
|
314
|
+
return db
|
|
315
|
+
db.query(set_utc)
|
|
316
|
+
if isinstance(db, ThreadedDatabase):
|
|
317
|
+
# The time zone is a session setting, and that query set it on one connection of the pool: the
|
|
318
|
+
# others would render timestamps in the server's time zone, and identical rows would differ.
|
|
319
|
+
db.session_setup += (set_utc,)
|
|
314
320
|
return db
|
|
315
321
|
|
|
316
322
|
|
|
@@ -1155,7 +1155,9 @@ class Database(abc.ABC):
|
|
|
1155
1155
|
)
|
|
1156
1156
|
else:
|
|
1157
1157
|
assert col_name in col_dict
|
|
1158
|
-
col_dict[col_name] = String_VaryingAlphanum(
|
|
1158
|
+
col_dict[col_name] = String_VaryingAlphanum(
|
|
1159
|
+
collation=col_dict[col_name].collation, padded=col_dict[col_name].padded
|
|
1160
|
+
)
|
|
1159
1161
|
|
|
1160
1162
|
return col_dict
|
|
1161
1163
|
|
|
@@ -1227,6 +1229,8 @@ class ThreadedDatabase(Database):
|
|
|
1227
1229
|
"""
|
|
1228
1230
|
|
|
1229
1231
|
thread_count: int = 1
|
|
1232
|
+
# Run on each new connection before its first query: session settings, such as the time zone (see Connect_SetUTC)
|
|
1233
|
+
session_setup: Tuple[str, ...] = ()
|
|
1230
1234
|
|
|
1231
1235
|
_init_error: Optional[Exception] = None
|
|
1232
1236
|
_queue: Optional[ThreadPoolExecutor] = None
|
|
@@ -1240,6 +1244,8 @@ class ThreadedDatabase(Database):
|
|
|
1240
1244
|
assert not hasattr(self.thread_local, "conn")
|
|
1241
1245
|
try:
|
|
1242
1246
|
self.thread_local.conn = self.create_connection()
|
|
1247
|
+
for sql_code in self.session_setup:
|
|
1248
|
+
self._query_conn(self.thread_local.conn, sql_code)
|
|
1243
1249
|
except Exception as e:
|
|
1244
1250
|
self._init_error = e
|
|
1245
1251
|
|
|
@@ -6,6 +6,7 @@ from packaging.version import parse as parse_version
|
|
|
6
6
|
from data_diff.schema import RawColumnInfo
|
|
7
7
|
from data_diff.utils import match_regexps
|
|
8
8
|
from data_diff.abcs.database_types import (
|
|
9
|
+
Date,
|
|
9
10
|
Timestamp,
|
|
10
11
|
TimestampTZ,
|
|
11
12
|
DbPath,
|
|
@@ -54,6 +55,7 @@ class Dialect(BaseDialect):
|
|
|
54
55
|
# Timestamps
|
|
55
56
|
"TIMESTAMP WITH TIME ZONE": TimestampTZ,
|
|
56
57
|
"TIMESTAMP": Timestamp,
|
|
58
|
+
"DATE": Date,
|
|
57
59
|
# Numbers
|
|
58
60
|
"DOUBLE": Float,
|
|
59
61
|
"FLOAT": Float,
|
|
@@ -103,6 +105,9 @@ class Dialect(BaseDialect):
|
|
|
103
105
|
return f"md5({s})"
|
|
104
106
|
|
|
105
107
|
def normalize_timestamp(self, value: str, coltype: TemporalType) -> str:
|
|
108
|
+
if isinstance(coltype, Date):
|
|
109
|
+
return f"strftime({value}, '%Y-%m-%d')" # As the other dialects render dates
|
|
110
|
+
|
|
106
111
|
# It's precision 6 by default. If precision is less than 6 -> we remove the trailing numbers.
|
|
107
112
|
if coltype.rounds and coltype.precision > 0:
|
|
108
113
|
return f"CONCAT(SUBSTRING(STRFTIME({value}::TIMESTAMP, '%Y-%m-%d %H:%M:%S.'),1,23), LPAD(((ROUND(strftime({value}::timestamp, '%f')::DECIMAL(15,7)/100000,{coltype.precision - 1})*100000)::INT)::VARCHAR,6,'0'))"
|
|
@@ -5,6 +5,7 @@ import attrs
|
|
|
5
5
|
from data_diff.databases.base import (
|
|
6
6
|
CHECKSUM_HEXDIGITS,
|
|
7
7
|
CHECKSUM_OFFSET,
|
|
8
|
+
TIMESTAMP_PRECISION_POS,
|
|
8
9
|
QueryError,
|
|
9
10
|
ThreadedDatabase,
|
|
10
11
|
import_helper,
|
|
@@ -13,7 +14,10 @@ from data_diff.databases.base import (
|
|
|
13
14
|
)
|
|
14
15
|
from data_diff.abcs.database_types import (
|
|
15
16
|
JSON,
|
|
17
|
+
ColType,
|
|
18
|
+
ColType_UUID,
|
|
16
19
|
NumericType,
|
|
20
|
+
StringType,
|
|
17
21
|
Timestamp,
|
|
18
22
|
TimestampTZ,
|
|
19
23
|
DbPath,
|
|
@@ -27,8 +31,11 @@ from data_diff.abcs.database_types import (
|
|
|
27
31
|
Date,
|
|
28
32
|
Time,
|
|
29
33
|
)
|
|
34
|
+
from data_diff.schema import RawColumnInfo
|
|
30
35
|
|
|
31
36
|
DEFAULT_ODBC_DRIVER = "ODBC Driver 18 for SQL Server"
|
|
37
|
+
# Binary, so it never compares or sorts anything differently; text is only converted with it
|
|
38
|
+
UTF8_COLLATION = "Latin1_General_100_BIN2_UTF8"
|
|
32
39
|
|
|
33
40
|
|
|
34
41
|
@import_helper("mssql")
|
|
@@ -42,6 +49,8 @@ def import_mssql():
|
|
|
42
49
|
class Dialect(BaseDialect):
|
|
43
50
|
name = "MsSQL"
|
|
44
51
|
ROUNDS_ON_PREC_LOSS = True
|
|
52
|
+
# Set on connecting: the UTF-8 collation to hash text in, or "" where the server has none (before 2019)
|
|
53
|
+
utf8_collation: Optional[str] = None
|
|
45
54
|
SUPPORTS_PRIMARY_KEY: ClassVar[bool] = True
|
|
46
55
|
SUPPORTS_INDEXES = True
|
|
47
56
|
TYPE_CLASSES = {
|
|
@@ -56,6 +65,7 @@ class Dialect(BaseDialect):
|
|
|
56
65
|
"float": Float,
|
|
57
66
|
"real": Float,
|
|
58
67
|
"decimal": Decimal,
|
|
68
|
+
"numeric": Decimal,
|
|
59
69
|
"money": Decimal,
|
|
60
70
|
"smallmoney": Decimal,
|
|
61
71
|
# int
|
|
@@ -84,6 +94,29 @@ class Dialect(BaseDialect):
|
|
|
84
94
|
def quote(self, s: str) -> str:
|
|
85
95
|
return f"[{s}]"
|
|
86
96
|
|
|
97
|
+
def parse_type(self, table_path: DbPath, info: RawColumnInfo) -> ColType:
|
|
98
|
+
coltype = super().parse_type(table_path, info)
|
|
99
|
+
if info.data_type in ("char", "nchar"):
|
|
100
|
+
return attrs.evolve(coltype, padded=True)
|
|
101
|
+
return coltype
|
|
102
|
+
|
|
103
|
+
def normalize_value_by_type(self, value: str, coltype: ColType) -> str:
|
|
104
|
+
if isinstance(coltype, StringType) and not isinstance(coltype, ColType_UUID):
|
|
105
|
+
if coltype.padded:
|
|
106
|
+
# Other databases' char types drop the padding (Postgres, MySQL) or have none (Snowflake)
|
|
107
|
+
value = f"RTRIM({value})"
|
|
108
|
+
if self.utf8_collation:
|
|
109
|
+
# Text is hashed as UTF-8 elsewhere. A plain varchar takes the database's code page, which can't
|
|
110
|
+
# hold every nvarchar character ('Łukasz' became 'Lukasz') and hashes non-ASCII differently.
|
|
111
|
+
return f"CONVERT(VARCHAR(MAX), CONVERT(NVARCHAR(MAX), {value}) COLLATE {self.utf8_collation})"
|
|
112
|
+
return self.to_string(value)
|
|
113
|
+
return super().normalize_value_by_type(value, coltype)
|
|
114
|
+
|
|
115
|
+
def _convert_db_precision_to_digits(self, p: int) -> int:
|
|
116
|
+
# normalize_number casts a float to decimal from its exact binary value, so like PostgreSQL and DuckDB,
|
|
117
|
+
# compare two fewer digits than the binary precision nominally holds: REAL 100.98 is 100.9800034.
|
|
118
|
+
return max(super()._convert_db_precision_to_digits(p) - 2, 0)
|
|
119
|
+
|
|
87
120
|
def set_timezone_to_utc(self) -> str:
|
|
88
121
|
raise NotImplementedError("MsSQL does not support a session timezone setting.")
|
|
89
122
|
|
|
@@ -140,21 +173,32 @@ class Dialect(BaseDialect):
|
|
|
140
173
|
return f"VALUES {values}"
|
|
141
174
|
|
|
142
175
|
def normalize_timestamp(self, value: str, coltype: TemporalType) -> str:
|
|
143
|
-
if coltype
|
|
144
|
-
|
|
145
|
-
f"FORMAT({value}, 'yyyy-MM-dd HH:mm:ss') + '.' + "
|
|
146
|
-
f"SUBSTRING(FORMAT({value}, 'fffffff'), 1, {coltype.precision})"
|
|
147
|
-
)
|
|
148
|
-
else:
|
|
149
|
-
formatted_value = f"FORMAT({value}, 'yyyy-MM-dd HH:mm:ss')"
|
|
176
|
+
if isinstance(coltype, Date):
|
|
177
|
+
return f"CONVERT(VARCHAR(10), {value}, 23)" # yyyy-mm-dd, as the other dialects render dates
|
|
150
178
|
|
|
151
|
-
|
|
179
|
+
if isinstance(coltype, Time):
|
|
180
|
+
# hh:mi:ss.ffffff, rounded to the column's precision like the other dialects' times
|
|
181
|
+
return f"CONVERT(VARCHAR(15), CAST(CAST({value} AS TIME({min(coltype.precision, 6)})) AS TIME(6)))"
|
|
152
182
|
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
183
|
+
if isinstance(coltype, TimestampTZ):
|
|
184
|
+
# There is no session time zone to set to UTC (see set_timezone_to_utc), so convert each value
|
|
185
|
+
value = f"SWITCHOFFSET({value}, '+00:00')"
|
|
156
186
|
|
|
157
|
-
|
|
187
|
+
# Style 121 is yyyy-mm-dd hh:mi:ss.fffffff. CONVERT rather than FORMAT(), which runs on the CLR and is
|
|
188
|
+
# far slower, and which left the fraction unpadded (".123" where the other dialects give ".123000").
|
|
189
|
+
if coltype.rounds and coltype.precision <= 6:
|
|
190
|
+
# Casting down to the precision rounds; casting back up to 6 digits pads with zeros
|
|
191
|
+
timestamp6 = f"CAST(CAST({value} AS DATETIME2({coltype.precision})) AS DATETIME2(6))"
|
|
192
|
+
return f"CONVERT(VARCHAR(26), {timestamp6}, 121)"
|
|
193
|
+
|
|
194
|
+
precision = min(coltype.precision, 6)
|
|
195
|
+
timestamp7 = f"CONVERT(VARCHAR(27), CAST({value} AS DATETIME2(7)), 121)"
|
|
196
|
+
return f"LEFT({timestamp7}, {TIMESTAMP_PRECISION_POS + precision}) + REPLICATE('0', {6 - precision})"
|
|
197
|
+
|
|
198
|
+
def normalize_number(self, value: str, coltype: NumericType) -> str:
|
|
199
|
+
# CAST rounds half away from zero, like the other dialects. FORMAT(value, 'N2') added grouping separators
|
|
200
|
+
# in the session's culture ("1,234.50"), and precision 0 went through FLOOR, which truncates.
|
|
201
|
+
return f"CONVERT(VARCHAR(40), CAST({value} AS DECIMAL(38, {coltype.precision})))"
|
|
158
202
|
|
|
159
203
|
def md5_as_int(self, s: str) -> str:
|
|
160
204
|
return f"convert(bigint, convert(varbinary, '0x' + RIGHT(CONVERT(NVARCHAR(32), HashBytes('MD5', {s}), 2), {CHECKSUM_HEXDIGITS}), 1)) - {CHECKSUM_OFFSET}"
|
|
@@ -206,9 +250,22 @@ class MsSQL(ThreadedDatabase):
|
|
|
206
250
|
self._mssql = import_mssql()
|
|
207
251
|
try:
|
|
208
252
|
connection = self._mssql.connect(**self._args)
|
|
209
|
-
return connection
|
|
210
253
|
except self._mssql.Error as error:
|
|
211
254
|
raise ConnectError(*error.args) from error
|
|
255
|
+
if self.dialect.utf8_collation is None:
|
|
256
|
+
self.dialect.utf8_collation = self._find_utf8_collation(connection)
|
|
257
|
+
return connection
|
|
258
|
+
|
|
259
|
+
def _find_utf8_collation(self, connection) -> str:
|
|
260
|
+
# UTF-8 collations came with SQL Server 2019 (and Azure SQL); before that, text keeps the code page
|
|
261
|
+
cursor = connection.cursor()
|
|
262
|
+
try:
|
|
263
|
+
cursor.execute(f"SELECT CONVERT(VARCHAR(1), N'x' COLLATE {UTF8_COLLATION})").fetchall()
|
|
264
|
+
except self._mssql.Error:
|
|
265
|
+
return ""
|
|
266
|
+
finally:
|
|
267
|
+
cursor.close()
|
|
268
|
+
return UTF8_COLLATION
|
|
212
269
|
|
|
213
270
|
def select_table_schema(self, path: DbPath) -> str:
|
|
214
271
|
"""Provide SQL for selecting the table schema as (name, type, date_prec, num_prec)"""
|
|
@@ -14,6 +14,7 @@ from data_diff.abcs.database_types import (
|
|
|
14
14
|
ColType_UUID,
|
|
15
15
|
Boolean,
|
|
16
16
|
Date,
|
|
17
|
+
Time,
|
|
17
18
|
)
|
|
18
19
|
from data_diff.databases.base import (
|
|
19
20
|
ThreadedDatabase,
|
|
@@ -48,6 +49,7 @@ class Dialect(BaseDialect):
|
|
|
48
49
|
"datetime": Datetime,
|
|
49
50
|
"timestamp": Timestamp,
|
|
50
51
|
"date": Date,
|
|
52
|
+
"time": Time,
|
|
51
53
|
# Numbers
|
|
52
54
|
"double": Float,
|
|
53
55
|
"float": Float,
|
|
@@ -106,6 +108,13 @@ class Dialect(BaseDialect):
|
|
|
106
108
|
return f"md5({s})"
|
|
107
109
|
|
|
108
110
|
def normalize_timestamp(self, value: str, coltype: TemporalType) -> str:
|
|
111
|
+
if isinstance(coltype, Date):
|
|
112
|
+
return f"DATE_FORMAT({value}, '%Y-%m-%d')" # As the other dialects render dates, not as a timestamp
|
|
113
|
+
|
|
114
|
+
if isinstance(coltype, Time):
|
|
115
|
+
# hh:mm:ss.ffffff, rounded to the column's precision like the other dialects' times
|
|
116
|
+
return f"TIME_FORMAT(CAST({value} AS TIME({min(coltype.precision, 6)})), '%H:%i:%s.%f')"
|
|
117
|
+
|
|
109
118
|
if coltype.rounds:
|
|
110
119
|
return self.to_string(f"cast( cast({value} as datetime({coltype.precision})) as datetime(6))")
|
|
111
120
|
|
|
@@ -152,7 +152,8 @@ class Dialect(
|
|
|
152
152
|
format_str = "FM" + "9" * (38 - coltype.precision)
|
|
153
153
|
if coltype.precision:
|
|
154
154
|
format_str += "0." + "9" * (coltype.precision - 1) + "0"
|
|
155
|
-
|
|
155
|
+
# Round first: to_char keeps the sign of a value that rounds to zero ("-0.00" for -0.004)
|
|
156
|
+
return f"to_char(round({value}, {coltype.precision}), '{format_str}')"
|
|
156
157
|
|
|
157
158
|
|
|
158
159
|
@attrs.define(frozen=False, init=False, kw_only=True)
|
|
@@ -122,7 +122,8 @@ class PostgresqlDialect(BaseDialect):
|
|
|
122
122
|
is_time = False
|
|
123
123
|
|
|
124
124
|
if isinstance(coltype, Date) or is_date:
|
|
125
|
-
|
|
125
|
+
# Explicit, since a cast follows the session's DateStyle ('15/01/2024' under "SQL, DMY")
|
|
126
|
+
return f"to_char({value}, 'YYYY-MM-DD')"
|
|
126
127
|
|
|
127
128
|
if isinstance(coltype, Time) or is_time:
|
|
128
129
|
seconds = f"EXTRACT( epoch from {value})"
|
|
@@ -195,12 +196,12 @@ class PostgreSQL(ThreadedDatabase):
|
|
|
195
196
|
pg = import_postgresql()
|
|
196
197
|
try:
|
|
197
198
|
self._args["password"] = unquote(self._args["password"])
|
|
198
|
-
|
|
199
|
-
**self._args, keepalives=1, keepalives_idle=5, keepalives_interval=2, keepalives_count=2
|
|
200
|
-
)
|
|
199
|
+
conn = pg.connect(**self._args, keepalives=1, keepalives_idle=5, keepalives_interval=2, keepalives_count=2)
|
|
201
200
|
if SESSION_TIME_ZONE:
|
|
202
|
-
|
|
203
|
-
|
|
201
|
+
conn.cursor().execute(f"SET TIME ZONE '{SESSION_TIME_ZONE}'")
|
|
202
|
+
# Each thread of the pool opens a connection of its own: by now self._conn may be another thread's
|
|
203
|
+
self._conn = conn
|
|
204
|
+
return conn
|
|
204
205
|
except pg.OperationalError as e:
|
|
205
206
|
raise ConnectError(*e.args) from e
|
|
206
207
|
|
|
@@ -8,6 +8,7 @@ from data_diff.schema import RawColumnInfo
|
|
|
8
8
|
from data_diff.utils import match_regexps
|
|
9
9
|
|
|
10
10
|
from data_diff.abcs.database_types import (
|
|
11
|
+
Date,
|
|
11
12
|
Timestamp,
|
|
12
13
|
TimestampTZ,
|
|
13
14
|
Integer,
|
|
@@ -60,6 +61,7 @@ class Dialect(BaseDialect):
|
|
|
60
61
|
"timestamp with time zone": TimestampTZ,
|
|
61
62
|
"timestamp without time zone": Timestamp,
|
|
62
63
|
"timestamp": Timestamp,
|
|
64
|
+
"date": Date,
|
|
63
65
|
# Numbers
|
|
64
66
|
"integer": Integer,
|
|
65
67
|
"bigint": Integer,
|
|
@@ -134,6 +136,9 @@ class Dialect(BaseDialect):
|
|
|
134
136
|
return f"TRIM(CAST({value} AS VARCHAR))"
|
|
135
137
|
|
|
136
138
|
def normalize_timestamp(self, value: str, coltype: TemporalType) -> str:
|
|
139
|
+
if isinstance(coltype, Date):
|
|
140
|
+
return f"CAST({value} AS VARCHAR)" # yyyy-mm-dd, as the other dialects render dates
|
|
141
|
+
|
|
137
142
|
# PrestoDB has no parameterized TIMESTAMP(p) type, only plain TIMESTAMP
|
|
138
143
|
# which is fixed at millisecond precision. Casting to TIMESTAMP(6) here
|
|
139
144
|
# fails with "Unknown type: timestamp(6)" on real PrestoDB even though
|
|
@@ -90,7 +90,8 @@ class Dialect(BaseDialect):
|
|
|
90
90
|
is_date = False
|
|
91
91
|
is_time = False
|
|
92
92
|
if isinstance(coltype, Date) or is_date:
|
|
93
|
-
|
|
93
|
+
# Explicit, since a cast follows the session's DATE_OUTPUT_FORMAT
|
|
94
|
+
return f"TO_CHAR({value}, 'YYYY-MM-DD')"
|
|
94
95
|
elif isinstance(coltype, Time) or is_time:
|
|
95
96
|
microseconds = f"TIMEDIFF(microsecond, cast('00:00:00' as time), {value})"
|
|
96
97
|
rounded = f"round({microseconds}, -6 + {coltype.precision})"
|
|
@@ -3,10 +3,14 @@ from typing import Any, ClassVar, Type
|
|
|
3
3
|
import attrs
|
|
4
4
|
from requests import Session
|
|
5
5
|
|
|
6
|
-
from data_diff.abcs.database_types import TemporalType,
|
|
6
|
+
from data_diff.abcs.database_types import ColType, ColType_UUID, Date, DbPath, TemporalType, Time, TimestampTZ
|
|
7
7
|
from data_diff.databases import presto
|
|
8
8
|
from data_diff.databases.base import import_helper
|
|
9
9
|
from data_diff.databases.base import TIMESTAMP_PRECISION_POS, BaseDialect
|
|
10
|
+
from data_diff.schema import RawColumnInfo
|
|
11
|
+
from data_diff.utils import match_regexps
|
|
12
|
+
|
|
13
|
+
TIME_PRECISION_POS = 9 # len("12:24:35.") == 9
|
|
10
14
|
|
|
11
15
|
|
|
12
16
|
@import_helper("trino")
|
|
@@ -31,7 +35,21 @@ class Dialect(presto.Dialect):
|
|
|
31
35
|
return f"TIMESTAMP({t.precision}) WITH TIME ZONE"
|
|
32
36
|
return super().type_repr(t)
|
|
33
37
|
|
|
38
|
+
def parse_type(self, table_path: DbPath, info: RawColumnInfo) -> ColType:
|
|
39
|
+
for m, t_cls in match_regexps({r"time\((\d)\)": Time}, info.data_type):
|
|
40
|
+
return t_cls(precision=int(m.group(1)), rounds=self.ROUNDS_ON_PREC_LOSS)
|
|
41
|
+
return super().parse_type(table_path, info)
|
|
42
|
+
|
|
34
43
|
def normalize_timestamp(self, value: str, coltype: TemporalType) -> str:
|
|
44
|
+
if isinstance(coltype, Date):
|
|
45
|
+
return f"CAST({value} AS VARCHAR)" # yyyy-mm-dd, as the other dialects render dates
|
|
46
|
+
|
|
47
|
+
if isinstance(coltype, Time):
|
|
48
|
+
# hh:mm:ss.ffffff: the cast rounds to the column's precision, like the other dialects' times
|
|
49
|
+
precision = min(coltype.precision, 6)
|
|
50
|
+
s = f"CAST(CAST({value} AS TIME({precision})) AS VARCHAR)"
|
|
51
|
+
return f"RPAD(RPAD({s}, {TIME_PRECISION_POS + precision}, '.'), {TIME_PRECISION_POS + 6}, '0')"
|
|
52
|
+
|
|
35
53
|
# Trino's date_format truncates fractional seconds to millisecond
|
|
36
54
|
# precision and loses any trailing microseconds. Casting the value to
|
|
37
55
|
# TIMESTAMP(p) and then to VARCHAR keeps the full precision. For
|
|
@@ -94,6 +94,7 @@ class DiffResultWrapper:
|
|
|
94
94
|
info_tree: InfoTree
|
|
95
95
|
stats: dict
|
|
96
96
|
result_list: list = attrs.field(factory=list)
|
|
97
|
+
sample: list = attrs.field(factory=list) # The first rows counted by the last stats call, see _get_stats
|
|
97
98
|
|
|
98
99
|
def __iter__(self) -> Iterator[Any]:
|
|
99
100
|
yield from self.result_list
|
|
@@ -101,13 +102,17 @@ class DiffResultWrapper:
|
|
|
101
102
|
self.result_list.append(i)
|
|
102
103
|
yield i
|
|
103
104
|
|
|
104
|
-
def _get_stats(self, is_dbt: bool = False, retain_rows: bool = True) -> DiffStats:
|
|
105
|
+
def _get_stats(self, is_dbt: bool = False, retain_rows: bool = True, sample_size: int = 0) -> DiffStats:
|
|
105
106
|
"""Count the diff by key, consuming the rest of the diff.
|
|
106
107
|
|
|
107
108
|
With `retain_rows`, every row is kept in `result_list` so the diff can be iterated again afterwards.
|
|
108
109
|
Without it, rows not consumed yet are counted and dropped, so memory grows with the number of
|
|
109
110
|
differing keys rather than with every differing row and all of its columns.
|
|
111
|
+
|
|
112
|
+
Either way, the first `sample_size` rows, in the order the diff yields them, are kept in `sample`, so a
|
|
113
|
+
caller that wants a few differing rows to show doesn't have to diff the tables a second time.
|
|
110
114
|
"""
|
|
115
|
+
self.sample.clear()
|
|
111
116
|
if retain_rows:
|
|
112
117
|
list(self) # Consume the iterator into result_list, if we haven't already
|
|
113
118
|
rows = self.result_list
|
|
@@ -124,6 +129,8 @@ class DiffResultWrapper:
|
|
|
124
129
|
extra_column_diffs = {k: 0 for k in extra_columns}
|
|
125
130
|
|
|
126
131
|
for sign, values in rows:
|
|
132
|
+
if len(self.sample) < sample_size:
|
|
133
|
+
self.sample.append((sign, values))
|
|
127
134
|
k = values[:len_key_columns]
|
|
128
135
|
if is_dbt:
|
|
129
136
|
extra_column_values = values[len_key_columns:]
|
|
@@ -157,8 +164,8 @@ class DiffResultWrapper:
|
|
|
157
164
|
|
|
158
165
|
return DiffStats(diff_by_sign, table1_count, table2_count, unchanged, diff_percent, extra_column_diffs)
|
|
159
166
|
|
|
160
|
-
def get_stats_string(self, is_dbt: bool = False, retain_rows: bool = True):
|
|
161
|
-
diff_stats = self._get_stats(is_dbt, retain_rows)
|
|
167
|
+
def get_stats_string(self, is_dbt: bool = False, retain_rows: bool = True, sample_size: int = 0):
|
|
168
|
+
diff_stats = self._get_stats(is_dbt, retain_rows, sample_size)
|
|
162
169
|
|
|
163
170
|
total_rows_diff = diff_stats.table2_count - diff_stats.table1_count
|
|
164
171
|
|
|
@@ -192,8 +199,8 @@ class DiffResultWrapper:
|
|
|
192
199
|
|
|
193
200
|
return string_output
|
|
194
201
|
|
|
195
|
-
def get_stats_dict(self, is_dbt: bool = False, retain_rows: bool = True):
|
|
196
|
-
diff_stats = self._get_stats(is_dbt, retain_rows)
|
|
202
|
+
def get_stats_dict(self, is_dbt: bool = False, retain_rows: bool = True, sample_size: int = 0):
|
|
203
|
+
diff_stats = self._get_stats(is_dbt, retain_rows, sample_size)
|
|
197
204
|
json_output = {
|
|
198
205
|
"rows_A": diff_stats.table1_count,
|
|
199
206
|
"rows_B": diff_stats.table2_count,
|
|
@@ -208,6 +215,11 @@ class DiffResultWrapper:
|
|
|
208
215
|
return json_output
|
|
209
216
|
|
|
210
217
|
|
|
218
|
+
# Segments are diffed on a thread pool, so merging their counts into a differ's stats takes this lock. It's not a
|
|
219
|
+
# field of the differ because those are sent as JSON with the tracking event.
|
|
220
|
+
_stats_lock = threading.Lock()
|
|
221
|
+
|
|
222
|
+
|
|
211
223
|
@attrs.define(frozen=False)
|
|
212
224
|
class TableDiffer(ThreadBase, ABC):
|
|
213
225
|
INFO_TREE_CLASS = InfoTree
|
|
@@ -240,9 +252,8 @@ class TableDiffer(ThreadBase, ABC):
|
|
|
240
252
|
|
|
241
253
|
def _diff_tables_wrapper(self, table1: TableSegment, table2: TableSegment, info_tree: InfoTree) -> DiffResult:
|
|
242
254
|
if is_tracking_enabled():
|
|
243
|
-
options
|
|
244
|
-
|
|
245
|
-
options.pop("_ignored_columns_lock")
|
|
255
|
+
# Only the options the differ was created with: its locks and progress aren't useful event attributes
|
|
256
|
+
options = attrs.asdict(self, recurse=False, filter=lambda attribute, _value: attribute.init)
|
|
246
257
|
options["differ_name"] = type(self).__name__
|
|
247
258
|
event_json = create_start_event_json(options)
|
|
248
259
|
run_as_daemon(send_event_json, event_json)
|
|
@@ -446,6 +457,19 @@ class TableDiffer(ThreadBase, ABC):
|
|
|
446
457
|
self._diff_segments, ti, t1, t2, info_node, max_rows, level + 1, i + 1, len(segmented1), priority=level
|
|
447
458
|
)
|
|
448
459
|
|
|
460
|
+
def _add_diff_counts(self, diff_counts: Dict[str, int]) -> None:
|
|
461
|
+
"""Add a segment's counts of differing rows per column to stats["diff_counts"].
|
|
462
|
+
|
|
463
|
+
Only rows found in both tables are counted, in the columns compared other than the keys, by table1's names.
|
|
464
|
+
Columns that never differ are left out, so with no such rows there is no "diff_counts" at all.
|
|
465
|
+
"""
|
|
466
|
+
if not diff_counts:
|
|
467
|
+
return
|
|
468
|
+
with _stats_lock:
|
|
469
|
+
totals = self.stats.setdefault("diff_counts", {})
|
|
470
|
+
for column, count in diff_counts.items():
|
|
471
|
+
totals[column] = totals.get(column, 0) + count
|
|
472
|
+
|
|
449
473
|
def ignore_column(self, column_name1: str, column_name2: str) -> None:
|
|
450
474
|
"""
|
|
451
475
|
Ignore the column (by name on sides A & B) in md5s & diffs from now on.
|
|
@@ -1,15 +1,18 @@
|
|
|
1
1
|
import os
|
|
2
|
+
import time
|
|
3
|
+
from datetime import timedelta
|
|
2
4
|
from numbers import Number
|
|
3
5
|
import logging
|
|
6
|
+
import threading
|
|
4
7
|
from collections import defaultdict
|
|
5
|
-
from typing import Any, Collection, Dict, Iterator, List, Sequence, Set, Tuple
|
|
8
|
+
from typing import Any, Collection, Dict, Iterator, List, Optional, Sequence, Set, Tuple
|
|
6
9
|
|
|
7
10
|
import attrs
|
|
8
11
|
from typing_extensions import Literal
|
|
9
12
|
|
|
10
13
|
from data_diff.abcs.database_types import ColType_UUID, NumericType, PrecisionType, StringType, Boolean, JSON
|
|
11
14
|
from data_diff.info_tree import InfoTree
|
|
12
|
-
from data_diff.utils import safezip, diffs_are_equiv_jsons
|
|
15
|
+
from data_diff.utils import safezip, diffs_are_equiv_jsons, _jsons_equiv
|
|
13
16
|
from data_diff.thread_utils import ThreadedYielder
|
|
14
17
|
from data_diff.table_segment import TableSegment
|
|
15
18
|
from data_diff.diff_tables import TableDiffer
|
|
@@ -18,8 +21,11 @@ BENCHMARK = os.environ.get("BENCHMARK", False)
|
|
|
18
21
|
|
|
19
22
|
DEFAULT_BISECTION_THRESHOLD = 1024 * 16
|
|
20
23
|
DEFAULT_BISECTION_FACTOR = 32
|
|
24
|
+
PROGRESS_LOG_INTERVAL = 60 # Seconds between a diff's progress lines
|
|
21
25
|
|
|
22
26
|
logger = logging.getLogger("hashdiff_tables")
|
|
27
|
+
# Progress has a logger of its own, so it can be shown without data-diff's other INFO logs
|
|
28
|
+
progress_logger = logging.getLogger("data_diff.progress")
|
|
23
29
|
|
|
24
30
|
# Just for local readability: TODO: later switch to real type declarations of these.
|
|
25
31
|
_Op = Literal["+", "-"]
|
|
@@ -38,6 +44,7 @@ def diff_sets(
|
|
|
38
44
|
key_columns2: Sequence[str],
|
|
39
45
|
ignored_columns1: Collection[str],
|
|
40
46
|
ignored_columns2: Collection[str],
|
|
47
|
+
diff_counts: dict = None,
|
|
41
48
|
) -> Iterator:
|
|
42
49
|
# Group full rows by PKs on each side. The first items are the PK: TableSegment.relevant_columns
|
|
43
50
|
rows_by_pks1: Dict[_PK, List[_Row]] = defaultdict(list)
|
|
@@ -68,6 +75,14 @@ def diff_sets(
|
|
|
68
75
|
for row2 in rows_by_pks2[pk]:
|
|
69
76
|
diffs_by_pks[pk].append(("+", row2))
|
|
70
77
|
|
|
78
|
+
# For diff_counts: how many reported keys differ in each compared column (not a key, not ignored), by table1's names
|
|
79
|
+
counted_columns = [
|
|
80
|
+
(i, col1)
|
|
81
|
+
for i, (col1, col2) in enumerate(zip(columns1, columns2))
|
|
82
|
+
if i >= len(key_columns1) and col1 not in ignored_columns1 and col2 not in ignored_columns2
|
|
83
|
+
]
|
|
84
|
+
json_col_names = set(json_cols.values()) if json_cols else set()
|
|
85
|
+
|
|
71
86
|
warned_diff_cols = set()
|
|
72
87
|
for diffs in (diffs_by_pks[pk] for pk in sorted(diffs_by_pks)):
|
|
73
88
|
if json_cols:
|
|
@@ -81,9 +96,44 @@ def diff_sets(
|
|
|
81
96
|
)
|
|
82
97
|
warned_diff_cols.add(w)
|
|
83
98
|
continue
|
|
99
|
+
# Only a key with one row on each side ("-" then "+") has a pair to compare, not a duplicated or exclusive one
|
|
100
|
+
if diff_counts is not None and len(diffs) == 2 and diffs[0][0] != diffs[1][0]:
|
|
101
|
+
(_, row1), (_, row2) = diffs
|
|
102
|
+
for i, col in counted_columns:
|
|
103
|
+
# Like whole rows, values of a JSON column that parse the same aren't a difference
|
|
104
|
+
if row1[i] != row2[i] and not (col in json_col_names and _jsons_equiv(row1[i], row2[i])):
|
|
105
|
+
diff_counts[col] = diff_counts.get(col, 0) + 1
|
|
84
106
|
yield from diffs
|
|
85
107
|
|
|
86
108
|
|
|
109
|
+
class _DiffProgress:
|
|
110
|
+
"""Counts the rows a diff has compared and the differing rows it has found, and logs them"""
|
|
111
|
+
|
|
112
|
+
def __init__(self, table1: TableSegment, table2: TableSegment) -> None:
|
|
113
|
+
self.tables = f"{'.'.join(table1.table_path)} <> {'.'.join(table2.table_path)}"
|
|
114
|
+
self.rows_compared = 0
|
|
115
|
+
self.differing_rows = 0
|
|
116
|
+
self.started = self.logged = time.monotonic()
|
|
117
|
+
self.lock = threading.Lock()
|
|
118
|
+
|
|
119
|
+
def update(self, rows_compared: int, differing_rows: int = 0) -> None:
|
|
120
|
+
"Adds a finished segment. Segments are diffed on a thread pool."
|
|
121
|
+
with self.lock:
|
|
122
|
+
self.rows_compared += rows_compared
|
|
123
|
+
self.differing_rows += differing_rows
|
|
124
|
+
now = time.monotonic()
|
|
125
|
+
if now - self.logged >= PROGRESS_LOG_INTERVAL:
|
|
126
|
+
self.logged = now
|
|
127
|
+
self._log("Diffing %s: %s rows compared in %s, %s differing so far")
|
|
128
|
+
|
|
129
|
+
def log_done(self) -> None:
|
|
130
|
+
self._log("Diffed %s: %s rows compared in %s, %s differing")
|
|
131
|
+
|
|
132
|
+
def _log(self, message: str) -> None:
|
|
133
|
+
elapsed = timedelta(seconds=round(time.monotonic() - self.started))
|
|
134
|
+
progress_logger.info(message, self.tables, f"{self.rows_compared:,}", elapsed, f"{self.differing_rows:,}")
|
|
135
|
+
|
|
136
|
+
|
|
87
137
|
@attrs.define(frozen=False)
|
|
88
138
|
class HashDiffer(TableDiffer):
|
|
89
139
|
"""Finds the diff between two SQL tables
|
|
@@ -107,6 +157,7 @@ class HashDiffer(TableDiffer):
|
|
|
107
157
|
bisection_disabled: bool = False # i.e. always download the rows (used in tests)
|
|
108
158
|
|
|
109
159
|
stats: dict = attrs.field(factory=dict)
|
|
160
|
+
_progress: Optional[_DiffProgress] = attrs.field(default=None, init=False)
|
|
110
161
|
|
|
111
162
|
def __attrs_post_init__(self) -> None:
|
|
112
163
|
# Validate options
|
|
@@ -115,6 +166,12 @@ class HashDiffer(TableDiffer):
|
|
|
115
166
|
if self.bisection_factor < 2:
|
|
116
167
|
raise ValueError("Must have at least two segments per iteration (i.e. bisection_factor >= 2)")
|
|
117
168
|
|
|
169
|
+
def _diff_tables_root(self, table1: TableSegment, table2: TableSegment, info_tree: InfoTree) -> Iterator:
|
|
170
|
+
self._progress = _DiffProgress(table1, table2)
|
|
171
|
+
yield from super()._diff_tables_root(table1, table2, info_tree)
|
|
172
|
+
# Only once the diff completes: not when it fails or is closed early
|
|
173
|
+
self._progress.log_done()
|
|
174
|
+
|
|
118
175
|
def _validate_and_adjust_columns(self, table1: TableSegment, table2: TableSegment, *, strict: bool = True) -> None:
|
|
119
176
|
for c1, c2 in safezip(table1.relevant_columns, table2.relevant_columns):
|
|
120
177
|
if c1 not in table1._schema:
|
|
@@ -209,6 +266,7 @@ class HashDiffer(TableDiffer):
|
|
|
209
266
|
|
|
210
267
|
if checksum1 == checksum2:
|
|
211
268
|
info_tree.info.is_diff = False
|
|
269
|
+
self._progress.update(max(count1, count2))
|
|
212
270
|
return
|
|
213
271
|
|
|
214
272
|
info_tree.info.is_diff = True
|
|
@@ -240,6 +298,7 @@ class HashDiffer(TableDiffer):
|
|
|
240
298
|
for i, colname in enumerate(table1.extra_columns)
|
|
241
299
|
if isinstance(table1._schema[colname], JSON)
|
|
242
300
|
}
|
|
301
|
+
diff_counts = {}
|
|
243
302
|
diff = list(
|
|
244
303
|
diff_sets(
|
|
245
304
|
rows1,
|
|
@@ -251,8 +310,10 @@ class HashDiffer(TableDiffer):
|
|
|
251
310
|
key_columns2=table2.key_columns,
|
|
252
311
|
ignored_columns1=self.ignored_columns1,
|
|
253
312
|
ignored_columns2=self.ignored_columns2,
|
|
313
|
+
diff_counts=diff_counts,
|
|
254
314
|
)
|
|
255
315
|
)
|
|
316
|
+
self._add_diff_counts(diff_counts)
|
|
256
317
|
|
|
257
318
|
# Record only how many rows differ. The rows are yielded to the caller; keeping them on the
|
|
258
319
|
# info tree as well would hold every differing row in memory until the whole diff completes.
|
|
@@ -262,6 +323,8 @@ class HashDiffer(TableDiffer):
|
|
|
262
323
|
|
|
263
324
|
logger.info(". " * level + f"Diff found {len(diff)} different rows.")
|
|
264
325
|
self.stats["rows_downloaded"] = self.stats.get("rows_downloaded", 0) + max(len(rows1), len(rows2))
|
|
326
|
+
differing_keys = {row[: len(table1.key_columns)] for _sign, row in diff}
|
|
327
|
+
self._progress.update(max(len(rows1), len(rows2)), len(differing_keys))
|
|
265
328
|
return diff
|
|
266
329
|
|
|
267
330
|
return super()._bisect_and_diff_segments(ti, table1, table2, info_tree, level, max_rows)
|
|
@@ -95,6 +95,13 @@ def _outerjoin(db: Database, a: ITable, b: ITable, keys1: List[str], keys2: List
|
|
|
95
95
|
return outerjoin(a, b).on(*on).select(is_exclusive_a=is_exclusive_a, is_exclusive_b=is_exclusive_b, **select_fields)
|
|
96
96
|
|
|
97
97
|
|
|
98
|
+
def _is_exclusive(db: Database) -> Expr:
|
|
99
|
+
"Whether a row of the outer join is found in only one of the tables"
|
|
100
|
+
if isinstance(db, (Oracle, MsSQL)):
|
|
101
|
+
return (this.is_exclusive_a == 1) | (this.is_exclusive_b == 1)
|
|
102
|
+
return this.is_exclusive_a | this.is_exclusive_b
|
|
103
|
+
|
|
104
|
+
|
|
98
105
|
def _slice_tuple(t, *sizes):
|
|
99
106
|
i = 0
|
|
100
107
|
for size in sizes:
|
|
@@ -193,7 +200,7 @@ class JoinDiffer(TableDiffer):
|
|
|
193
200
|
partial(self._collect_stats, 2, table2, info_tree),
|
|
194
201
|
partial(self._test_null_keys, table1, table2),
|
|
195
202
|
partial(self._sample_and_count_exclusive, db, diff_rows, a_cols, b_cols, table1, table2),
|
|
196
|
-
partial(self._count_diff_per_column, db, diff_rows,
|
|
203
|
+
partial(self._count_diff_per_column, db, diff_rows, table1, table2),
|
|
197
204
|
partial(
|
|
198
205
|
self._materialize_diff,
|
|
199
206
|
db,
|
|
@@ -333,25 +340,19 @@ class JoinDiffer(TableDiffer):
|
|
|
333
340
|
diff_rows = all_rows.where(or_(this[c] == 1 for c in is_diff_cols))
|
|
334
341
|
return diff_rows, a_cols, b_cols, is_diff_cols, all_rows
|
|
335
342
|
|
|
336
|
-
def _count_diff_per_column(
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
is_diff_cols,
|
|
342
|
-
table1: Optional[TableSegment] = None,
|
|
343
|
-
table2: Optional[TableSegment] = None,
|
|
344
|
-
):
|
|
343
|
+
def _count_diff_per_column(self, db, diff_rows, table1: TableSegment, table2: TableSegment):
|
|
344
|
+
# Counted as HashDiffer counts them: in the rows found in both tables, every column but the keys
|
|
345
|
+
columns = [c for c in table1.relevant_columns if c not in table1.key_columns]
|
|
346
|
+
if not columns:
|
|
347
|
+
return
|
|
345
348
|
logger.debug(f"Counting differences per column: {table1.table_path} <> {table2.table_path}")
|
|
346
|
-
|
|
347
|
-
diff_rows.select(sum_(this[c]) for c in
|
|
349
|
+
counts = db.query(
|
|
350
|
+
diff_rows.select(sum_(if_(_is_exclusive(db), 0, this[f"is_diff_{c}"])) for c in columns),
|
|
348
351
|
tuple,
|
|
349
352
|
log_message=f"{table1.table_path} <> {table2.table_path}",
|
|
350
353
|
)
|
|
351
|
-
|
|
352
|
-
for
|
|
353
|
-
diff_counts[name] = diff_counts.get(name, 0) + (count or 0)
|
|
354
|
-
self.stats["diff_counts"] = diff_counts
|
|
354
|
+
# A sum is NULL when no rows differ, and some databases return it as a Decimal
|
|
355
|
+
self._add_diff_counts({column: int(count) for column, count in safezip(columns, counts) if count})
|
|
355
356
|
|
|
356
357
|
def _sample_and_count_exclusive(
|
|
357
358
|
self,
|
|
@@ -362,10 +363,7 @@ class JoinDiffer(TableDiffer):
|
|
|
362
363
|
table1: Optional[TableSegment] = None,
|
|
363
364
|
table2: Optional[TableSegment] = None,
|
|
364
365
|
):
|
|
365
|
-
|
|
366
|
-
exclusive_rows_query = diff_rows.where((this.is_exclusive_a == 1) | (this.is_exclusive_b == 1))
|
|
367
|
-
else:
|
|
368
|
-
exclusive_rows_query = diff_rows.where(this.is_exclusive_a | this.is_exclusive_b)
|
|
366
|
+
exclusive_rows_query = diff_rows.where(_is_exclusive(db))
|
|
369
367
|
|
|
370
368
|
if not self.sample_exclusive_rows:
|
|
371
369
|
logger.debug(f"Counting exclusive rows: {table1.table_path} <> {table2.table_path}")
|
|
@@ -248,8 +248,14 @@ class TableSegment:
|
|
|
248
248
|
duration = time.monotonic() - start
|
|
249
249
|
if duration > RECOMMENDED_CHECKSUM_DURATION:
|
|
250
250
|
logger.warning(
|
|
251
|
-
"Checksum
|
|
252
|
-
"
|
|
251
|
+
"Checksum of %s rows in %s table %s at key-range %s..%s took %.2fs. Slow checksums usually mean "
|
|
252
|
+
"the key columns aren't indexed, many columns are compared, or the database is busy; "
|
|
253
|
+
"CLI users can also use a larger --bisection-factor or fewer --threads.",
|
|
254
|
+
count,
|
|
255
|
+
self.database.name,
|
|
256
|
+
".".join(self.table_path),
|
|
257
|
+
self.min_key,
|
|
258
|
+
self.max_key,
|
|
253
259
|
duration,
|
|
254
260
|
)
|
|
255
261
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[tool.poetry]
|
|
2
2
|
name = "collate-data-diff"
|
|
3
|
-
version = "0.11.
|
|
3
|
+
version = "0.11.19"
|
|
4
4
|
description = "Command-line tool and Python library to efficiently diff rows across two different databases."
|
|
5
5
|
authors = ["Collate <info@getcollate.io>", "Datafold <data-diff@datafold.com>"]
|
|
6
6
|
license = "MIT"
|
|
@@ -40,7 +40,7 @@ trino = {version=">=0.314.0", optional=true}
|
|
|
40
40
|
presto-python-client = {version="*", optional=true}
|
|
41
41
|
clickhouse-driver = {version="*", optional=true}
|
|
42
42
|
duckdb = {version="*", optional=true}
|
|
43
|
-
dbt-core = {version="^1.0.0"}
|
|
43
|
+
dbt-core = {version="^1.0.0", optional=true}
|
|
44
44
|
deepdiff = ">=8.6.1,<9.0" # Security fix for CVE-2025-58367
|
|
45
45
|
keyring = "*"
|
|
46
46
|
tabulate = ">=0.9.0"
|
|
@@ -50,8 +50,10 @@ urllib3 = ">=2.7.0,<3" # Security fix for CVE-2026-44431, CVE-2026-21441, CVE-20
|
|
|
50
50
|
oracledb = {version = "*", optional=true}
|
|
51
51
|
pyodbc = {version=">=4.0.39", optional=true}
|
|
52
52
|
typing-extensions = ">=4.0.1"
|
|
53
|
+
# Imported at module level by data_diff.utils; previously only arrived through dbt-core.
|
|
54
|
+
requests = ">=2.32.4" # CVE-2024-47081
|
|
55
|
+
packaging = "*"
|
|
53
56
|
attrs = ">=23.1.0"
|
|
54
|
-
mashumaro = {version = ">=2.9,<3.11.0", extras = ["msgpack"]}
|
|
55
57
|
|
|
56
58
|
[tool.poetry.extras]
|
|
57
59
|
# When adding, update also: README + dev deps just above
|
|
@@ -68,6 +70,9 @@ trino = ["trino"]
|
|
|
68
70
|
clickhouse = ["clickhouse-driver"]
|
|
69
71
|
vertica = ["vertica-python"]
|
|
70
72
|
duckdb = ["duckdb"]
|
|
73
|
+
# Only the CLI --dbt mode imports dbt; dbt 2 ships as `dbt`/`dbt-oss`, so a hard dbt-core pin
|
|
74
|
+
# would install two distributions into site-packages/dbt next to it.
|
|
75
|
+
dbt = ["dbt-core"]
|
|
71
76
|
all-dbs = [
|
|
72
77
|
"preql", "mysql-connector-python", "psycopg2", "snowflake-connector-python", "cryptography", "presto-python-client",
|
|
73
78
|
"oracledb", "pyodbc", "trino", "clickhouse-driver", "vertica-python", "duckdb"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|