collate-data-diff 0.11.16__tar.gz → 0.11.18__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. {collate_data_diff-0.11.16 → collate_data_diff-0.11.18}/PKG-INFO +2 -1
  2. {collate_data_diff-0.11.16 → collate_data_diff-0.11.18}/data_diff/abcs/database_types.py +2 -0
  3. {collate_data_diff-0.11.16 → collate_data_diff-0.11.18}/data_diff/databases/_connect.py +7 -1
  4. {collate_data_diff-0.11.16 → collate_data_diff-0.11.18}/data_diff/databases/base.py +7 -1
  5. {collate_data_diff-0.11.16 → collate_data_diff-0.11.18}/data_diff/databases/duckdb.py +5 -0
  6. {collate_data_diff-0.11.16 → collate_data_diff-0.11.18}/data_diff/databases/mssql.py +70 -13
  7. {collate_data_diff-0.11.16 → collate_data_diff-0.11.18}/data_diff/databases/mysql.py +9 -0
  8. {collate_data_diff-0.11.16 → collate_data_diff-0.11.18}/data_diff/databases/oracle.py +2 -1
  9. {collate_data_diff-0.11.16 → collate_data_diff-0.11.18}/data_diff/databases/postgresql.py +7 -6
  10. {collate_data_diff-0.11.16 → collate_data_diff-0.11.18}/data_diff/databases/presto.py +5 -0
  11. {collate_data_diff-0.11.16 → collate_data_diff-0.11.18}/data_diff/databases/snowflake.py +2 -1
  12. {collate_data_diff-0.11.16 → collate_data_diff-0.11.18}/data_diff/databases/trino.py +19 -1
  13. {collate_data_diff-0.11.16 → collate_data_diff-0.11.18}/data_diff/diff_tables.py +58 -12
  14. {collate_data_diff-0.11.16 → collate_data_diff-0.11.18}/data_diff/errors.py +10 -0
  15. {collate_data_diff-0.11.16 → collate_data_diff-0.11.18}/data_diff/hashdiff_tables.py +69 -3
  16. {collate_data_diff-0.11.16 → collate_data_diff-0.11.18}/data_diff/info_tree.py +7 -3
  17. {collate_data_diff-0.11.16 → collate_data_diff-0.11.18}/data_diff/joindiff_tables.py +18 -20
  18. {collate_data_diff-0.11.16 → collate_data_diff-0.11.18}/data_diff/table_segment.py +8 -2
  19. {collate_data_diff-0.11.16 → collate_data_diff-0.11.18}/data_diff/thread_utils.py +17 -12
  20. {collate_data_diff-0.11.16 → collate_data_diff-0.11.18}/pyproject.toml +1 -1
  21. {collate_data_diff-0.11.16 → collate_data_diff-0.11.18}/LICENSE +0 -0
  22. {collate_data_diff-0.11.16 → collate_data_diff-0.11.18}/README.md +0 -0
  23. {collate_data_diff-0.11.16 → collate_data_diff-0.11.18}/data_diff/__init__.py +0 -0
  24. {collate_data_diff-0.11.16 → collate_data_diff-0.11.18}/data_diff/__main__.py +0 -0
  25. {collate_data_diff-0.11.16 → collate_data_diff-0.11.18}/data_diff/abcs/__init__.py +0 -0
  26. {collate_data_diff-0.11.16 → collate_data_diff-0.11.18}/data_diff/abcs/compiler.py +0 -0
  27. {collate_data_diff-0.11.16 → collate_data_diff-0.11.18}/data_diff/cloud/__init__.py +0 -0
  28. {collate_data_diff-0.11.16 → collate_data_diff-0.11.18}/data_diff/cloud/data_source.py +0 -0
  29. {collate_data_diff-0.11.16 → collate_data_diff-0.11.18}/data_diff/cloud/datafold_api.py +0 -0
  30. {collate_data_diff-0.11.16 → collate_data_diff-0.11.18}/data_diff/config.py +0 -0
  31. {collate_data_diff-0.11.16 → collate_data_diff-0.11.18}/data_diff/databases/__init__.py +0 -0
  32. {collate_data_diff-0.11.16 → collate_data_diff-0.11.18}/data_diff/databases/bigquery.py +0 -0
  33. {collate_data_diff-0.11.16 → collate_data_diff-0.11.18}/data_diff/databases/clickhouse.py +0 -0
  34. {collate_data_diff-0.11.16 → collate_data_diff-0.11.18}/data_diff/databases/databricks.py +0 -0
  35. {collate_data_diff-0.11.16 → collate_data_diff-0.11.18}/data_diff/databases/redshift.py +0 -0
  36. {collate_data_diff-0.11.16 → collate_data_diff-0.11.18}/data_diff/databases/vertica.py +0 -0
  37. {collate_data_diff-0.11.16 → collate_data_diff-0.11.18}/data_diff/dbt.py +0 -0
  38. {collate_data_diff-0.11.16 → collate_data_diff-0.11.18}/data_diff/dbt_config_validators.py +0 -0
  39. {collate_data_diff-0.11.16 → collate_data_diff-0.11.18}/data_diff/dbt_parser.py +0 -0
  40. {collate_data_diff-0.11.16 → collate_data_diff-0.11.18}/data_diff/format.py +0 -0
  41. {collate_data_diff-0.11.16 → collate_data_diff-0.11.18}/data_diff/lexicographic_space.py +0 -0
  42. {collate_data_diff-0.11.16 → collate_data_diff-0.11.18}/data_diff/parse_time.py +0 -0
  43. {collate_data_diff-0.11.16 → collate_data_diff-0.11.18}/data_diff/py.typed +0 -0
  44. {collate_data_diff-0.11.16 → collate_data_diff-0.11.18}/data_diff/queries/__init__.py +0 -0
  45. {collate_data_diff-0.11.16 → collate_data_diff-0.11.18}/data_diff/queries/api.py +0 -0
  46. {collate_data_diff-0.11.16 → collate_data_diff-0.11.18}/data_diff/queries/ast_classes.py +0 -0
  47. {collate_data_diff-0.11.16 → collate_data_diff-0.11.18}/data_diff/queries/base.py +0 -0
  48. {collate_data_diff-0.11.16 → collate_data_diff-0.11.18}/data_diff/queries/extras.py +0 -0
  49. {collate_data_diff-0.11.16 → collate_data_diff-0.11.18}/data_diff/query_utils.py +0 -0
  50. {collate_data_diff-0.11.16 → collate_data_diff-0.11.18}/data_diff/schema.py +0 -0
  51. {collate_data_diff-0.11.16 → collate_data_diff-0.11.18}/data_diff/tracking.py +0 -0
  52. {collate_data_diff-0.11.16 → collate_data_diff-0.11.18}/data_diff/utils.py +0 -0
  53. {collate_data_diff-0.11.16 → collate_data_diff-0.11.18}/data_diff/version.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: collate-data-diff
3
- Version: 0.11.16
3
+ Version: 0.11.18
4
4
  Summary: Command-line tool and Python library to efficiently diff rows across two different databases.
5
5
  License: MIT
6
6
  License-File: LICENSE
@@ -19,6 +19,7 @@ Classifier: Programming Language :: Python :: 3.11
19
19
  Classifier: Programming Language :: Python :: 3.12
20
20
  Classifier: Programming Language :: Python :: 3.13
21
21
  Classifier: Programming Language :: Python :: 3.14
22
+ Classifier: Programming Language :: Python :: 3.15
22
23
  Classifier: Programming Language :: Python :: 3.8
23
24
  Classifier: Programming Language :: Python :: 3.9
24
25
  Classifier: Topic :: Database :: Database Engines/Servers
@@ -205,6 +205,8 @@ class Decimal(FractionalType, IKey): # Snowflake may use Decimal as a key
205
205
  class StringType(ColType):
206
206
  python_type = str
207
207
  collation: Optional[Collation] = attrs.field(default=None, kw_only=True)
208
+ # A fixed-width column (CHAR) the database pads with trailing spaces
209
+ padded: bool = attrs.field(default=False, kw_only=True)
208
210
 
209
211
 
210
212
  @attrs.define(frozen=True)
@@ -306,11 +306,17 @@ class Connect_SetUTC(Connect):
306
306
  def _connection_created(self, db):
307
307
  db = super()._connection_created(db)
308
308
  try:
309
- db.query(db.dialect.set_timezone_to_utc())
309
+ set_utc = db.dialect.set_timezone_to_utc()
310
310
  except NotImplementedError:
311
311
  logging.debug(
312
312
  f"Database '{db}' does not allow setting timezone. We recommend making sure it's set to 'UTC'."
313
313
  )
314
+ return db
315
+ db.query(set_utc)
316
+ if isinstance(db, ThreadedDatabase):
317
+ # The time zone is a session setting, and that query set it on one connection of the pool: the
318
+ # others would render timestamps in the server's time zone, and identical rows would differ.
319
+ db.session_setup += (set_utc,)
314
320
  return db
315
321
 
316
322
 
@@ -1155,7 +1155,9 @@ class Database(abc.ABC):
1155
1155
  )
1156
1156
  else:
1157
1157
  assert col_name in col_dict
1158
- col_dict[col_name] = String_VaryingAlphanum(collation=col_dict[col_name].collation)
1158
+ col_dict[col_name] = String_VaryingAlphanum(
1159
+ collation=col_dict[col_name].collation, padded=col_dict[col_name].padded
1160
+ )
1159
1161
 
1160
1162
  return col_dict
1161
1163
 
@@ -1227,6 +1229,8 @@ class ThreadedDatabase(Database):
1227
1229
  """
1228
1230
 
1229
1231
  thread_count: int = 1
1232
+ # Run on each new connection before its first query: session settings, such as the time zone (see Connect_SetUTC)
1233
+ session_setup: Tuple[str, ...] = ()
1230
1234
 
1231
1235
  _init_error: Optional[Exception] = None
1232
1236
  _queue: Optional[ThreadPoolExecutor] = None
@@ -1240,6 +1244,8 @@ class ThreadedDatabase(Database):
1240
1244
  assert not hasattr(self.thread_local, "conn")
1241
1245
  try:
1242
1246
  self.thread_local.conn = self.create_connection()
1247
+ for sql_code in self.session_setup:
1248
+ self._query_conn(self.thread_local.conn, sql_code)
1243
1249
  except Exception as e:
1244
1250
  self._init_error = e
1245
1251
 
@@ -6,6 +6,7 @@ from packaging.version import parse as parse_version
6
6
  from data_diff.schema import RawColumnInfo
7
7
  from data_diff.utils import match_regexps
8
8
  from data_diff.abcs.database_types import (
9
+ Date,
9
10
  Timestamp,
10
11
  TimestampTZ,
11
12
  DbPath,
@@ -54,6 +55,7 @@ class Dialect(BaseDialect):
54
55
  # Timestamps
55
56
  "TIMESTAMP WITH TIME ZONE": TimestampTZ,
56
57
  "TIMESTAMP": Timestamp,
58
+ "DATE": Date,
57
59
  # Numbers
58
60
  "DOUBLE": Float,
59
61
  "FLOAT": Float,
@@ -103,6 +105,9 @@ class Dialect(BaseDialect):
103
105
  return f"md5({s})"
104
106
 
105
107
  def normalize_timestamp(self, value: str, coltype: TemporalType) -> str:
108
+ if isinstance(coltype, Date):
109
+ return f"strftime({value}, '%Y-%m-%d')" # As the other dialects render dates
110
+
106
111
  # It's precision 6 by default. If precision is less than 6 -> we remove the trailing numbers.
107
112
  if coltype.rounds and coltype.precision > 0:
108
113
  return f"CONCAT(SUBSTRING(STRFTIME({value}::TIMESTAMP, '%Y-%m-%d %H:%M:%S.'),1,23), LPAD(((ROUND(strftime({value}::timestamp, '%f')::DECIMAL(15,7)/100000,{coltype.precision - 1})*100000)::INT)::VARCHAR,6,'0'))"
@@ -5,6 +5,7 @@ import attrs
5
5
  from data_diff.databases.base import (
6
6
  CHECKSUM_HEXDIGITS,
7
7
  CHECKSUM_OFFSET,
8
+ TIMESTAMP_PRECISION_POS,
8
9
  QueryError,
9
10
  ThreadedDatabase,
10
11
  import_helper,
@@ -13,7 +14,10 @@ from data_diff.databases.base import (
13
14
  )
14
15
  from data_diff.abcs.database_types import (
15
16
  JSON,
17
+ ColType,
18
+ ColType_UUID,
16
19
  NumericType,
20
+ StringType,
17
21
  Timestamp,
18
22
  TimestampTZ,
19
23
  DbPath,
@@ -27,8 +31,11 @@ from data_diff.abcs.database_types import (
27
31
  Date,
28
32
  Time,
29
33
  )
34
+ from data_diff.schema import RawColumnInfo
30
35
 
31
36
  DEFAULT_ODBC_DRIVER = "ODBC Driver 18 for SQL Server"
37
+ # Binary, so it never compares or sorts anything differently; text is only converted with it
38
+ UTF8_COLLATION = "Latin1_General_100_BIN2_UTF8"
32
39
 
33
40
 
34
41
  @import_helper("mssql")
@@ -42,6 +49,8 @@ def import_mssql():
42
49
  class Dialect(BaseDialect):
43
50
  name = "MsSQL"
44
51
  ROUNDS_ON_PREC_LOSS = True
52
+ # Set on connecting: the UTF-8 collation to hash text in, or "" where the server has none (before 2019)
53
+ utf8_collation: Optional[str] = None
45
54
  SUPPORTS_PRIMARY_KEY: ClassVar[bool] = True
46
55
  SUPPORTS_INDEXES = True
47
56
  TYPE_CLASSES = {
@@ -56,6 +65,7 @@ class Dialect(BaseDialect):
56
65
  "float": Float,
57
66
  "real": Float,
58
67
  "decimal": Decimal,
68
+ "numeric": Decimal,
59
69
  "money": Decimal,
60
70
  "smallmoney": Decimal,
61
71
  # int
@@ -84,6 +94,29 @@ class Dialect(BaseDialect):
84
94
  def quote(self, s: str) -> str:
85
95
  return f"[{s}]"
86
96
 
97
+ def parse_type(self, table_path: DbPath, info: RawColumnInfo) -> ColType:
98
+ coltype = super().parse_type(table_path, info)
99
+ if info.data_type in ("char", "nchar"):
100
+ return attrs.evolve(coltype, padded=True)
101
+ return coltype
102
+
103
+ def normalize_value_by_type(self, value: str, coltype: ColType) -> str:
104
+ if isinstance(coltype, StringType) and not isinstance(coltype, ColType_UUID):
105
+ if coltype.padded:
106
+ # Other databases' char types drop the padding (Postgres, MySQL) or have none (Snowflake)
107
+ value = f"RTRIM({value})"
108
+ if self.utf8_collation:
109
+ # Text is hashed as UTF-8 elsewhere. A plain varchar takes the database's code page, which can't
110
+ # hold every nvarchar character ('Łukasz' became 'Lukasz') and hashes non-ASCII differently.
111
+ return f"CONVERT(VARCHAR(MAX), CONVERT(NVARCHAR(MAX), {value}) COLLATE {self.utf8_collation})"
112
+ return self.to_string(value)
113
+ return super().normalize_value_by_type(value, coltype)
114
+
115
+ def _convert_db_precision_to_digits(self, p: int) -> int:
116
+ # normalize_number casts a float to decimal from its exact binary value, so like PostgreSQL and DuckDB,
117
+ # compare two fewer digits than the binary precision nominally holds: REAL 100.98 is 100.9800034.
118
+ return max(super()._convert_db_precision_to_digits(p) - 2, 0)
119
+
87
120
  def set_timezone_to_utc(self) -> str:
88
121
  raise NotImplementedError("MsSQL does not support a session timezone setting.")
89
122
 
@@ -140,21 +173,32 @@ class Dialect(BaseDialect):
140
173
  return f"VALUES {values}"
141
174
 
142
175
  def normalize_timestamp(self, value: str, coltype: TemporalType) -> str:
143
- if coltype.precision > 0:
144
- formatted_value = (
145
- f"FORMAT({value}, 'yyyy-MM-dd HH:mm:ss') + '.' + "
146
- f"SUBSTRING(FORMAT({value}, 'fffffff'), 1, {coltype.precision})"
147
- )
148
- else:
149
- formatted_value = f"FORMAT({value}, 'yyyy-MM-dd HH:mm:ss')"
176
+ if isinstance(coltype, Date):
177
+ return f"CONVERT(VARCHAR(10), {value}, 23)" # yyyy-mm-dd, as the other dialects render dates
150
178
 
151
- return formatted_value
179
+ if isinstance(coltype, Time):
180
+ # hh:mi:ss.ffffff, rounded to the column's precision like the other dialects' times
181
+ return f"CONVERT(VARCHAR(15), CAST(CAST({value} AS TIME({min(coltype.precision, 6)})) AS TIME(6)))"
152
182
 
153
- def normalize_number(self, value: str, coltype: NumericType) -> str:
154
- if coltype.precision == 0:
155
- return f"CAST(FLOOR({value}) AS VARCHAR)"
183
+ if isinstance(coltype, TimestampTZ):
184
+ # There is no session time zone to set to UTC (see set_timezone_to_utc), so convert each value
185
+ value = f"SWITCHOFFSET({value}, '+00:00')"
156
186
 
157
- return f"FORMAT({value}, 'N{coltype.precision}')"
187
+ # Style 121 is yyyy-mm-dd hh:mi:ss.fffffff. CONVERT rather than FORMAT(), which runs on the CLR and is
188
+ # far slower, and which left the fraction unpadded (".123" where the other dialects give ".123000").
189
+ if coltype.rounds and coltype.precision <= 6:
190
+ # Casting down to the precision rounds; casting back up to 6 digits pads with zeros
191
+ timestamp6 = f"CAST(CAST({value} AS DATETIME2({coltype.precision})) AS DATETIME2(6))"
192
+ return f"CONVERT(VARCHAR(26), {timestamp6}, 121)"
193
+
194
+ precision = min(coltype.precision, 6)
195
+ timestamp7 = f"CONVERT(VARCHAR(27), CAST({value} AS DATETIME2(7)), 121)"
196
+ return f"LEFT({timestamp7}, {TIMESTAMP_PRECISION_POS + precision}) + REPLICATE('0', {6 - precision})"
197
+
198
+ def normalize_number(self, value: str, coltype: NumericType) -> str:
199
+ # CAST rounds half away from zero, like the other dialects. FORMAT(value, 'N2') added grouping separators
200
+ # in the session's culture ("1,234.50"), and precision 0 went through FLOOR, which truncates.
201
+ return f"CONVERT(VARCHAR(40), CAST({value} AS DECIMAL(38, {coltype.precision})))"
158
202
 
159
203
  def md5_as_int(self, s: str) -> str:
160
204
  return f"convert(bigint, convert(varbinary, '0x' + RIGHT(CONVERT(NVARCHAR(32), HashBytes('MD5', {s}), 2), {CHECKSUM_HEXDIGITS}), 1)) - {CHECKSUM_OFFSET}"
@@ -206,9 +250,22 @@ class MsSQL(ThreadedDatabase):
206
250
  self._mssql = import_mssql()
207
251
  try:
208
252
  connection = self._mssql.connect(**self._args)
209
- return connection
210
253
  except self._mssql.Error as error:
211
254
  raise ConnectError(*error.args) from error
255
+ if self.dialect.utf8_collation is None:
256
+ self.dialect.utf8_collation = self._find_utf8_collation(connection)
257
+ return connection
258
+
259
+ def _find_utf8_collation(self, connection) -> str:
260
+ # UTF-8 collations came with SQL Server 2019 (and Azure SQL); before that, text keeps the code page
261
+ cursor = connection.cursor()
262
+ try:
263
+ cursor.execute(f"SELECT CONVERT(VARCHAR(1), N'x' COLLATE {UTF8_COLLATION})").fetchall()
264
+ except self._mssql.Error:
265
+ return ""
266
+ finally:
267
+ cursor.close()
268
+ return UTF8_COLLATION
212
269
 
213
270
  def select_table_schema(self, path: DbPath) -> str:
214
271
  """Provide SQL for selecting the table schema as (name, type, date_prec, num_prec)"""
@@ -14,6 +14,7 @@ from data_diff.abcs.database_types import (
14
14
  ColType_UUID,
15
15
  Boolean,
16
16
  Date,
17
+ Time,
17
18
  )
18
19
  from data_diff.databases.base import (
19
20
  ThreadedDatabase,
@@ -48,6 +49,7 @@ class Dialect(BaseDialect):
48
49
  "datetime": Datetime,
49
50
  "timestamp": Timestamp,
50
51
  "date": Date,
52
+ "time": Time,
51
53
  # Numbers
52
54
  "double": Float,
53
55
  "float": Float,
@@ -106,6 +108,13 @@ class Dialect(BaseDialect):
106
108
  return f"md5({s})"
107
109
 
108
110
  def normalize_timestamp(self, value: str, coltype: TemporalType) -> str:
111
+ if isinstance(coltype, Date):
112
+ return f"DATE_FORMAT({value}, '%Y-%m-%d')" # As the other dialects render dates, not as a timestamp
113
+
114
+ if isinstance(coltype, Time):
115
+ # hh:mm:ss.ffffff, rounded to the column's precision like the other dialects' times
116
+ return f"TIME_FORMAT(CAST({value} AS TIME({min(coltype.precision, 6)})), '%H:%i:%s.%f')"
117
+
109
118
  if coltype.rounds:
110
119
  return self.to_string(f"cast( cast({value} as datetime({coltype.precision})) as datetime(6))")
111
120
 
@@ -152,7 +152,8 @@ class Dialect(
152
152
  format_str = "FM" + "9" * (38 - coltype.precision)
153
153
  if coltype.precision:
154
154
  format_str += "0." + "9" * (coltype.precision - 1) + "0"
155
- return f"to_char({value}, '{format_str}')"
155
+ # Round first: to_char keeps the sign of a value that rounds to zero ("-0.00" for -0.004)
156
+ return f"to_char(round({value}, {coltype.precision}), '{format_str}')"
156
157
 
157
158
 
158
159
  @attrs.define(frozen=False, init=False, kw_only=True)
@@ -122,7 +122,8 @@ class PostgresqlDialect(BaseDialect):
122
122
  is_time = False
123
123
 
124
124
  if isinstance(coltype, Date) or is_date:
125
- return f"cast({value} as varchar)"
125
+ # Explicit, since a cast follows the session's DateStyle ('15/01/2024' under "SQL, DMY")
126
+ return f"to_char({value}, 'YYYY-MM-DD')"
126
127
 
127
128
  if isinstance(coltype, Time) or is_time:
128
129
  seconds = f"EXTRACT( epoch from {value})"
@@ -195,12 +196,12 @@ class PostgreSQL(ThreadedDatabase):
195
196
  pg = import_postgresql()
196
197
  try:
197
198
  self._args["password"] = unquote(self._args["password"])
198
- self._conn = pg.connect(
199
- **self._args, keepalives=1, keepalives_idle=5, keepalives_interval=2, keepalives_count=2
200
- )
199
+ conn = pg.connect(**self._args, keepalives=1, keepalives_idle=5, keepalives_interval=2, keepalives_count=2)
201
200
  if SESSION_TIME_ZONE:
202
- self._conn.cursor().execute(f"SET TIME ZONE '{SESSION_TIME_ZONE}'")
203
- return self._conn
201
+ conn.cursor().execute(f"SET TIME ZONE '{SESSION_TIME_ZONE}'")
202
+ # Each thread of the pool opens a connection of its own: by now self._conn may be another thread's
203
+ self._conn = conn
204
+ return conn
204
205
  except pg.OperationalError as e:
205
206
  raise ConnectError(*e.args) from e
206
207
 
@@ -8,6 +8,7 @@ from data_diff.schema import RawColumnInfo
8
8
  from data_diff.utils import match_regexps
9
9
 
10
10
  from data_diff.abcs.database_types import (
11
+ Date,
11
12
  Timestamp,
12
13
  TimestampTZ,
13
14
  Integer,
@@ -60,6 +61,7 @@ class Dialect(BaseDialect):
60
61
  "timestamp with time zone": TimestampTZ,
61
62
  "timestamp without time zone": Timestamp,
62
63
  "timestamp": Timestamp,
64
+ "date": Date,
63
65
  # Numbers
64
66
  "integer": Integer,
65
67
  "bigint": Integer,
@@ -134,6 +136,9 @@ class Dialect(BaseDialect):
134
136
  return f"TRIM(CAST({value} AS VARCHAR))"
135
137
 
136
138
  def normalize_timestamp(self, value: str, coltype: TemporalType) -> str:
139
+ if isinstance(coltype, Date):
140
+ return f"CAST({value} AS VARCHAR)" # yyyy-mm-dd, as the other dialects render dates
141
+
137
142
  # PrestoDB has no parameterized TIMESTAMP(p) type, only plain TIMESTAMP
138
143
  # which is fixed at millisecond precision. Casting to TIMESTAMP(6) here
139
144
  # fails with "Unknown type: timestamp(6)" on real PrestoDB even though
@@ -90,7 +90,8 @@ class Dialect(BaseDialect):
90
90
  is_date = False
91
91
  is_time = False
92
92
  if isinstance(coltype, Date) or is_date:
93
- return f"({value}::varchar)"
93
+ # Explicit, since a cast follows the session's DATE_OUTPUT_FORMAT
94
+ return f"TO_CHAR({value}, 'YYYY-MM-DD')"
94
95
  elif isinstance(coltype, Time) or is_time:
95
96
  microseconds = f"TIMEDIFF(microsecond, cast('00:00:00' as time), {value})"
96
97
  rounded = f"round({microseconds}, -6 + {coltype.precision})"
@@ -3,10 +3,14 @@ from typing import Any, ClassVar, Type
3
3
  import attrs
4
4
  from requests import Session
5
5
 
6
- from data_diff.abcs.database_types import TemporalType, TimestampTZ, ColType_UUID
6
+ from data_diff.abcs.database_types import ColType, ColType_UUID, Date, DbPath, TemporalType, Time, TimestampTZ
7
7
  from data_diff.databases import presto
8
8
  from data_diff.databases.base import import_helper
9
9
  from data_diff.databases.base import TIMESTAMP_PRECISION_POS, BaseDialect
10
+ from data_diff.schema import RawColumnInfo
11
+ from data_diff.utils import match_regexps
12
+
13
+ TIME_PRECISION_POS = 9 # len("12:24:35.") == 9
10
14
 
11
15
 
12
16
  @import_helper("trino")
@@ -31,7 +35,21 @@ class Dialect(presto.Dialect):
31
35
  return f"TIMESTAMP({t.precision}) WITH TIME ZONE"
32
36
  return super().type_repr(t)
33
37
 
38
+ def parse_type(self, table_path: DbPath, info: RawColumnInfo) -> ColType:
39
+ for m, t_cls in match_regexps({r"time\((\d)\)": Time}, info.data_type):
40
+ return t_cls(precision=int(m.group(1)), rounds=self.ROUNDS_ON_PREC_LOSS)
41
+ return super().parse_type(table_path, info)
42
+
34
43
  def normalize_timestamp(self, value: str, coltype: TemporalType) -> str:
44
+ if isinstance(coltype, Date):
45
+ return f"CAST({value} AS VARCHAR)" # yyyy-mm-dd, as the other dialects render dates
46
+
47
+ if isinstance(coltype, Time):
48
+ # hh:mm:ss.ffffff: the cast rounds to the column's precision, like the other dialects' times
49
+ precision = min(coltype.precision, 6)
50
+ s = f"CAST(CAST({value} AS TIME({precision})) AS VARCHAR)"
51
+ return f"RPAD(RPAD({s}, {TIME_PRECISION_POS + precision}, '.'), {TIME_PRECISION_POS + 6}, '0')"
52
+
35
53
  # Trino's date_format truncates fractional seconds to millisecond
36
54
  # precision and loses any trailing microseconds. Casting the value to
37
55
  # TIMESTAMP(p) and then to VARCHAR keeps the full precision. For
@@ -5,13 +5,18 @@ import time
5
5
  from abc import ABC, abstractmethod
6
6
  from enum import Enum
7
7
  from contextlib import contextmanager
8
+ from itertools import chain
8
9
  from operator import methodcaller
9
10
  from typing import Any, Dict, Set, List, Tuple, Iterator, Optional, Union
10
11
  from concurrent.futures import ThreadPoolExecutor, as_completed
11
12
 
12
13
  import attrs
13
14
 
14
- from data_diff.errors import DataDiffMismatchingKeyTypesError, DataDiffUnsupportedKeyValueError
15
+ from data_diff.errors import (
16
+ DataDiffDuplicateKeyError,
17
+ DataDiffMismatchingKeyTypesError,
18
+ DataDiffUnsupportedKeyValueError,
19
+ )
15
20
  from data_diff.info_tree import InfoTree, SegmentInfo
16
21
  from data_diff.utils import dbt_diff_string_template, run_as_daemon, safezip, getLogger, truncate_error, Vector
17
22
  from data_diff.thread_utils import ThreadedYielder
@@ -89,6 +94,7 @@ class DiffResultWrapper:
89
94
  info_tree: InfoTree
90
95
  stats: dict
91
96
  result_list: list = attrs.field(factory=list)
97
+ sample: list = attrs.field(factory=list) # The first rows counted by the last stats call, see _get_stats
92
98
 
93
99
  def __iter__(self) -> Iterator[Any]:
94
100
  yield from self.result_list
@@ -96,8 +102,22 @@ class DiffResultWrapper:
96
102
  self.result_list.append(i)
97
103
  yield i
98
104
 
99
- def _get_stats(self, is_dbt: bool = False) -> DiffStats:
100
- list(self) # Consume the iterator into result_list, if we haven't already
105
+ def _get_stats(self, is_dbt: bool = False, retain_rows: bool = True, sample_size: int = 0) -> DiffStats:
106
+ """Count the diff by key, consuming the rest of the diff.
107
+
108
+ With `retain_rows`, every row is kept in `result_list` so the diff can be iterated again afterwards.
109
+ Without it, rows not consumed yet are counted and dropped, so memory grows with the number of
110
+ differing keys rather than with every differing row and all of its columns.
111
+
112
+ Either way, the first `sample_size` rows, in the order the diff yields them, are kept in `sample`, so a
113
+ caller that wants a few differing rows to show doesn't have to diff the tables a second time.
114
+ """
115
+ self.sample.clear()
116
+ if retain_rows:
117
+ list(self) # Consume the iterator into result_list, if we haven't already
118
+ rows = self.result_list
119
+ else:
120
+ rows = chain(self.result_list, self.diff)
101
121
 
102
122
  key_columns = self.info_tree.info.tables[0].key_columns
103
123
  len_key_columns = len(key_columns)
@@ -108,12 +128,21 @@ class DiffResultWrapper:
108
128
  extra_columns = self.info_tree.info.tables[0].extra_columns
109
129
  extra_column_diffs = {k: 0 for k in extra_columns}
110
130
 
111
- for sign, values in self.result_list:
131
+ for sign, values in rows:
132
+ if len(self.sample) < sample_size:
133
+ self.sample.append((sign, values))
112
134
  k = values[:len_key_columns]
113
135
  if is_dbt:
114
136
  extra_column_values = values[len_key_columns:]
115
137
  if k in diff_by_key:
116
- assert sign != diff_by_key[k]
138
+ if sign == diff_by_key[k]:
139
+ table_index = 1 if sign == "-" else 2
140
+ table = self.info_tree.info.tables[table_index - 1]
141
+ raise DataDiffDuplicateKeyError(
142
+ f"Key ({', '.join(key_columns)}) is not unique in table {'.'.join(table.table_path)}. "
143
+ "A row-level diff needs a unique key.",
144
+ table_index,
145
+ )
117
146
  diff_by_key[k] = "!"
118
147
  if is_dbt:
119
148
  for i in range(0, len(extra_columns)):
@@ -135,8 +164,8 @@ class DiffResultWrapper:
135
164
 
136
165
  return DiffStats(diff_by_sign, table1_count, table2_count, unchanged, diff_percent, extra_column_diffs)
137
166
 
138
- def get_stats_string(self, is_dbt: bool = False):
139
- diff_stats = self._get_stats(is_dbt)
167
+ def get_stats_string(self, is_dbt: bool = False, retain_rows: bool = True, sample_size: int = 0):
168
+ diff_stats = self._get_stats(is_dbt, retain_rows, sample_size)
140
169
 
141
170
  total_rows_diff = diff_stats.table2_count - diff_stats.table1_count
142
171
 
@@ -170,8 +199,8 @@ class DiffResultWrapper:
170
199
 
171
200
  return string_output
172
201
 
173
- def get_stats_dict(self, is_dbt: bool = False):
174
- diff_stats = self._get_stats(is_dbt)
202
+ def get_stats_dict(self, is_dbt: bool = False, retain_rows: bool = True, sample_size: int = 0):
203
+ diff_stats = self._get_stats(is_dbt, retain_rows, sample_size)
175
204
  json_output = {
176
205
  "rows_A": diff_stats.table1_count,
177
206
  "rows_B": diff_stats.table2_count,
@@ -186,6 +215,11 @@ class DiffResultWrapper:
186
215
  return json_output
187
216
 
188
217
 
218
+ # Segments are diffed on a thread pool, so merging their counts into a differ's stats takes this lock. It's not a
219
+ # field of the differ because those are sent as JSON with the tracking event.
220
+ _stats_lock = threading.Lock()
221
+
222
+
189
223
  @attrs.define(frozen=False)
190
224
  class TableDiffer(ThreadBase, ABC):
191
225
  INFO_TREE_CLASS = InfoTree
@@ -218,9 +252,8 @@ class TableDiffer(ThreadBase, ABC):
218
252
 
219
253
  def _diff_tables_wrapper(self, table1: TableSegment, table2: TableSegment, info_tree: InfoTree) -> DiffResult:
220
254
  if is_tracking_enabled():
221
- options = attrs.asdict(self, recurse=False)
222
- # not a useful event attribute
223
- options.pop("_ignored_columns_lock")
255
+ # Only the options the differ was created with: its locks and progress aren't useful event attributes
256
+ options = attrs.asdict(self, recurse=False, filter=lambda attribute, _value: attribute.init)
224
257
  options["differ_name"] = type(self).__name__
225
258
  event_json = create_start_event_json(options)
226
259
  run_as_daemon(send_event_json, event_json)
@@ -424,6 +457,19 @@ class TableDiffer(ThreadBase, ABC):
424
457
  self._diff_segments, ti, t1, t2, info_node, max_rows, level + 1, i + 1, len(segmented1), priority=level
425
458
  )
426
459
 
460
+ def _add_diff_counts(self, diff_counts: Dict[str, int]) -> None:
461
+ """Add a segment's counts of differing rows per column to stats["diff_counts"].
462
+
463
+ Only rows found in both tables are counted, in the columns compared other than the keys, by table1's names.
464
+ Columns that never differ are left out, so with no such rows there is no "diff_counts" at all.
465
+ """
466
+ if not diff_counts:
467
+ return
468
+ with _stats_lock:
469
+ totals = self.stats.setdefault("diff_counts", {})
470
+ for column, count in diff_counts.items():
471
+ totals[column] = totals.get(column, 0) + count
472
+
427
473
  def ignore_column(self, column_name1: str, column_name2: str) -> None:
428
474
  """
429
475
  Ignore the column (by name on sides A & B) in md5s & diffs from now on.
@@ -78,3 +78,13 @@ class DataDiffMismatchingKeyTypesError(Exception):
78
78
  # _parse_key_range_result — keeps the exception contract for existing callers.
79
79
  class DataDiffUnsupportedKeyValueError(ValueError):
80
80
  "Raised when a key column holds values that cannot be mapped to a bisectable range (e.g. text keys with '.', accents or other non-ASCII characters)."
81
+
82
+
83
+ # Subclasses AssertionError because it replaces the bare assert previously used by
84
+ # DiffResultWrapper._get_stats — callers catching AssertionError keep working.
85
+ class DataDiffDuplicateKeyError(AssertionError):
86
+ "Raised when computing diff stats finds the same key twice in one table. `table_index` is 1 or 2."
87
+
88
+ def __init__(self, message: str, table_index: int) -> None:
89
+ super().__init__(message)
90
+ self.table_index = table_index
@@ -1,15 +1,18 @@
1
1
  import os
2
+ import time
3
+ from datetime import timedelta
2
4
  from numbers import Number
3
5
  import logging
6
+ import threading
4
7
  from collections import defaultdict
5
- from typing import Any, Collection, Dict, Iterator, List, Sequence, Set, Tuple
8
+ from typing import Any, Collection, Dict, Iterator, List, Optional, Sequence, Set, Tuple
6
9
 
7
10
  import attrs
8
11
  from typing_extensions import Literal
9
12
 
10
13
  from data_diff.abcs.database_types import ColType_UUID, NumericType, PrecisionType, StringType, Boolean, JSON
11
14
  from data_diff.info_tree import InfoTree
12
- from data_diff.utils import safezip, diffs_are_equiv_jsons
15
+ from data_diff.utils import safezip, diffs_are_equiv_jsons, _jsons_equiv
13
16
  from data_diff.thread_utils import ThreadedYielder
14
17
  from data_diff.table_segment import TableSegment
15
18
  from data_diff.diff_tables import TableDiffer
@@ -18,8 +21,11 @@ BENCHMARK = os.environ.get("BENCHMARK", False)
18
21
 
19
22
  DEFAULT_BISECTION_THRESHOLD = 1024 * 16
20
23
  DEFAULT_BISECTION_FACTOR = 32
24
+ PROGRESS_LOG_INTERVAL = 60 # Seconds between a diff's progress lines
21
25
 
22
26
  logger = logging.getLogger("hashdiff_tables")
27
+ # Progress has a logger of its own, so it can be shown without data-diff's other INFO logs
28
+ progress_logger = logging.getLogger("data_diff.progress")
23
29
 
24
30
  # Just for local readability: TODO: later switch to real type declarations of these.
25
31
  _Op = Literal["+", "-"]
@@ -38,6 +44,7 @@ def diff_sets(
38
44
  key_columns2: Sequence[str],
39
45
  ignored_columns1: Collection[str],
40
46
  ignored_columns2: Collection[str],
47
+ diff_counts: dict = None,
41
48
  ) -> Iterator:
42
49
  # Group full rows by PKs on each side. The first items are the PK: TableSegment.relevant_columns
43
50
  rows_by_pks1: Dict[_PK, List[_Row]] = defaultdict(list)
@@ -68,6 +75,14 @@ def diff_sets(
68
75
  for row2 in rows_by_pks2[pk]:
69
76
  diffs_by_pks[pk].append(("+", row2))
70
77
 
78
+ # For diff_counts: how many reported keys differ in each compared column (not a key, not ignored), by table1's names
79
+ counted_columns = [
80
+ (i, col1)
81
+ for i, (col1, col2) in enumerate(zip(columns1, columns2))
82
+ if i >= len(key_columns1) and col1 not in ignored_columns1 and col2 not in ignored_columns2
83
+ ]
84
+ json_col_names = set(json_cols.values()) if json_cols else set()
85
+
71
86
  warned_diff_cols = set()
72
87
  for diffs in (diffs_by_pks[pk] for pk in sorted(diffs_by_pks)):
73
88
  if json_cols:
@@ -81,9 +96,44 @@ def diff_sets(
81
96
  )
82
97
  warned_diff_cols.add(w)
83
98
  continue
99
+ # Only a key with one row on each side ("-" then "+") has a pair to compare, not a duplicated or exclusive one
100
+ if diff_counts is not None and len(diffs) == 2 and diffs[0][0] != diffs[1][0]:
101
+ (_, row1), (_, row2) = diffs
102
+ for i, col in counted_columns:
103
+ # Like whole rows, values of a JSON column that parse the same aren't a difference
104
+ if row1[i] != row2[i] and not (col in json_col_names and _jsons_equiv(row1[i], row2[i])):
105
+ diff_counts[col] = diff_counts.get(col, 0) + 1
84
106
  yield from diffs
85
107
 
86
108
 
109
+ class _DiffProgress:
110
+ """Counts the rows a diff has compared and the differing rows it has found, and logs them"""
111
+
112
+ def __init__(self, table1: TableSegment, table2: TableSegment) -> None:
113
+ self.tables = f"{'.'.join(table1.table_path)} <> {'.'.join(table2.table_path)}"
114
+ self.rows_compared = 0
115
+ self.differing_rows = 0
116
+ self.started = self.logged = time.monotonic()
117
+ self.lock = threading.Lock()
118
+
119
+ def update(self, rows_compared: int, differing_rows: int = 0) -> None:
120
+ "Adds a finished segment. Segments are diffed on a thread pool."
121
+ with self.lock:
122
+ self.rows_compared += rows_compared
123
+ self.differing_rows += differing_rows
124
+ now = time.monotonic()
125
+ if now - self.logged >= PROGRESS_LOG_INTERVAL:
126
+ self.logged = now
127
+ self._log("Diffing %s: %s rows compared in %s, %s differing so far")
128
+
129
+ def log_done(self) -> None:
130
+ self._log("Diffed %s: %s rows compared in %s, %s differing")
131
+
132
+ def _log(self, message: str) -> None:
133
+ elapsed = timedelta(seconds=round(time.monotonic() - self.started))
134
+ progress_logger.info(message, self.tables, f"{self.rows_compared:,}", elapsed, f"{self.differing_rows:,}")
135
+
136
+
87
137
  @attrs.define(frozen=False)
88
138
  class HashDiffer(TableDiffer):
89
139
  """Finds the diff between two SQL tables
@@ -107,6 +157,7 @@ class HashDiffer(TableDiffer):
107
157
  bisection_disabled: bool = False # i.e. always download the rows (used in tests)
108
158
 
109
159
  stats: dict = attrs.field(factory=dict)
160
+ _progress: Optional[_DiffProgress] = attrs.field(default=None, init=False)
110
161
 
111
162
  def __attrs_post_init__(self) -> None:
112
163
  # Validate options
@@ -115,6 +166,12 @@ class HashDiffer(TableDiffer):
115
166
  if self.bisection_factor < 2:
116
167
  raise ValueError("Must have at least two segments per iteration (i.e. bisection_factor >= 2)")
117
168
 
169
+ def _diff_tables_root(self, table1: TableSegment, table2: TableSegment, info_tree: InfoTree) -> Iterator:
170
+ self._progress = _DiffProgress(table1, table2)
171
+ yield from super()._diff_tables_root(table1, table2, info_tree)
172
+ # Only once the diff completes: not when it fails or is closed early
173
+ self._progress.log_done()
174
+
118
175
  def _validate_and_adjust_columns(self, table1: TableSegment, table2: TableSegment, *, strict: bool = True) -> None:
119
176
  for c1, c2 in safezip(table1.relevant_columns, table2.relevant_columns):
120
177
  if c1 not in table1._schema:
@@ -209,6 +266,7 @@ class HashDiffer(TableDiffer):
209
266
 
210
267
  if checksum1 == checksum2:
211
268
  info_tree.info.is_diff = False
269
+ self._progress.update(max(count1, count2))
212
270
  return
213
271
 
214
272
  info_tree.info.is_diff = True
@@ -240,6 +298,7 @@ class HashDiffer(TableDiffer):
240
298
  for i, colname in enumerate(table1.extra_columns)
241
299
  if isinstance(table1._schema[colname], JSON)
242
300
  }
301
+ diff_counts = {}
243
302
  diff = list(
244
303
  diff_sets(
245
304
  rows1,
@@ -251,14 +310,21 @@ class HashDiffer(TableDiffer):
251
310
  key_columns2=table2.key_columns,
252
311
  ignored_columns1=self.ignored_columns1,
253
312
  ignored_columns2=self.ignored_columns2,
313
+ diff_counts=diff_counts,
254
314
  )
255
315
  )
316
+ self._add_diff_counts(diff_counts)
256
317
 
257
- info_tree.info.set_diff(diff)
318
+ # Record only how many rows differ. The rows are yielded to the caller; keeping them on the
319
+ # info tree as well would hold every differing row in memory until the whole diff completes.
320
+ info_tree.info.diff_count = len(diff)
321
+ info_tree.info.is_diff = bool(diff)
258
322
  info_tree.info.rowcounts = {1: len(rows1), 2: len(rows2)}
259
323
 
260
324
  logger.info(". " * level + f"Diff found {len(diff)} different rows.")
261
325
  self.stats["rows_downloaded"] = self.stats.get("rows_downloaded", 0) + max(len(rows1), len(rows2))
326
+ differing_keys = {row[: len(table1.key_columns)] for _sign, row in diff}
327
+ self._progress.update(max(len(rows1), len(rows2)), len(differing_keys))
262
328
  return diff
263
329
 
264
330
  return super()._bisect_and_diff_segments(ti, table1, table2, info_tree, level, max_rows)
@@ -1,3 +1,4 @@
1
+ from itertools import chain
1
2
  from typing import List, Dict, Optional, Any, Tuple, Union
2
3
 
3
4
  import attrs
@@ -34,11 +35,14 @@ class SegmentInfo:
34
35
  self.diff_count = sum(c.diff_count for c in child_infos if c.diff_count is not None)
35
36
  self.is_diff = any(c.is_diff for c in child_infos)
36
37
  self.diff_schema = next((child.diff_schema for child in child_infos if child.diff_schema is not None), None)
37
- self.diff = sum((c.diff for c in child_infos if c.diff is not None), [])
38
+ # None when no child kept its rows (hashdiff keeps only counts), not an empty list that reads as "no diff"
39
+ child_diffs = [c.diff for c in child_infos if c.diff is not None]
40
+ self.diff = list(chain.from_iterable(child_diffs)) if child_diffs else None
38
41
 
42
+ # A diff closed early can leave a joindiff segment that has counted only one of the tables so far
39
43
  self.rowcounts = {
40
- 1: sum(c.rowcounts[1] for c in child_infos if c.rowcounts),
41
- 2: sum(c.rowcounts[2] for c in child_infos if c.rowcounts),
44
+ 1: sum(c.rowcounts.get(1, 0) for c in child_infos),
45
+ 2: sum(c.rowcounts.get(2, 0) for c in child_infos),
42
46
  }
43
47
 
44
48
 
@@ -95,6 +95,13 @@ def _outerjoin(db: Database, a: ITable, b: ITable, keys1: List[str], keys2: List
95
95
  return outerjoin(a, b).on(*on).select(is_exclusive_a=is_exclusive_a, is_exclusive_b=is_exclusive_b, **select_fields)
96
96
 
97
97
 
98
+ def _is_exclusive(db: Database) -> Expr:
99
+ "Whether a row of the outer join is found in only one of the tables"
100
+ if isinstance(db, (Oracle, MsSQL)):
101
+ return (this.is_exclusive_a == 1) | (this.is_exclusive_b == 1)
102
+ return this.is_exclusive_a | this.is_exclusive_b
103
+
104
+
98
105
  def _slice_tuple(t, *sizes):
99
106
  i = 0
100
107
  for size in sizes:
@@ -193,7 +200,7 @@ class JoinDiffer(TableDiffer):
193
200
  partial(self._collect_stats, 2, table2, info_tree),
194
201
  partial(self._test_null_keys, table1, table2),
195
202
  partial(self._sample_and_count_exclusive, db, diff_rows, a_cols, b_cols, table1, table2),
196
- partial(self._count_diff_per_column, db, diff_rows, list(a_cols), is_diff_cols, table1, table2),
203
+ partial(self._count_diff_per_column, db, diff_rows, table1, table2),
197
204
  partial(
198
205
  self._materialize_diff,
199
206
  db,
@@ -333,25 +340,19 @@ class JoinDiffer(TableDiffer):
333
340
  diff_rows = all_rows.where(or_(this[c] == 1 for c in is_diff_cols))
334
341
  return diff_rows, a_cols, b_cols, is_diff_cols, all_rows
335
342
 
336
- def _count_diff_per_column(
337
- self,
338
- db,
339
- diff_rows,
340
- cols,
341
- is_diff_cols,
342
- table1: Optional[TableSegment] = None,
343
- table2: Optional[TableSegment] = None,
344
- ):
343
+ def _count_diff_per_column(self, db, diff_rows, table1: TableSegment, table2: TableSegment):
344
+ # Counted as HashDiffer counts them: in the rows found in both tables, every column but the keys
345
+ columns = [c for c in table1.relevant_columns if c not in table1.key_columns]
346
+ if not columns:
347
+ return
345
348
  logger.debug(f"Counting differences per column: {table1.table_path} <> {table2.table_path}")
346
- is_diff_cols_counts = db.query(
347
- diff_rows.select(sum_(this[c]) for c in is_diff_cols),
349
+ counts = db.query(
350
+ diff_rows.select(sum_(if_(_is_exclusive(db), 0, this[f"is_diff_{c}"])) for c in columns),
348
351
  tuple,
349
352
  log_message=f"{table1.table_path} <> {table2.table_path}",
350
353
  )
351
- diff_counts = {}
352
- for name, count in safezip(cols, is_diff_cols_counts):
353
- diff_counts[name] = diff_counts.get(name, 0) + (count or 0)
354
- self.stats["diff_counts"] = diff_counts
354
+ # A sum is NULL when no rows differ, and some databases return it as a Decimal
355
+ self._add_diff_counts({column: int(count) for column, count in safezip(columns, counts) if count})
355
356
 
356
357
  def _sample_and_count_exclusive(
357
358
  self,
@@ -362,10 +363,7 @@ class JoinDiffer(TableDiffer):
362
363
  table1: Optional[TableSegment] = None,
363
364
  table2: Optional[TableSegment] = None,
364
365
  ):
365
- if isinstance(db, (Oracle, MsSQL)):
366
- exclusive_rows_query = diff_rows.where((this.is_exclusive_a == 1) | (this.is_exclusive_b == 1))
367
- else:
368
- exclusive_rows_query = diff_rows.where(this.is_exclusive_a | this.is_exclusive_b)
366
+ exclusive_rows_query = diff_rows.where(_is_exclusive(db))
369
367
 
370
368
  if not self.sample_exclusive_rows:
371
369
  logger.debug(f"Counting exclusive rows: {table1.table_path} <> {table2.table_path}")
@@ -248,8 +248,14 @@ class TableSegment:
248
248
  duration = time.monotonic() - start
249
249
  if duration > RECOMMENDED_CHECKSUM_DURATION:
250
250
  logger.warning(
251
- "Checksum is taking longer than expected (%.2f). "
252
- "We recommend increasing --bisection-factor or decreasing --threads.",
251
+ "Checksum of %s rows in %s table %s at key-range %s..%s took %.2fs. Slow checksums usually mean "
252
+ "the key columns aren't indexed, many columns are compared, or the database is busy; "
253
+ "CLI users can also use a larger --bisection-factor or fewer --threads.",
254
+ count,
255
+ self.database.name,
256
+ ".".join(self.table_path),
257
+ self.min_key,
258
+ self.max_key,
253
259
  duration,
254
260
  )
255
261
 
@@ -81,18 +81,23 @@ class ThreadedYielder(Iterable):
81
81
  self._futures.append(self._pool.submit(self._worker, fn, *args, priority=priority, **kwargs))
82
82
 
83
83
  def __iter__(self) -> Iterator[Any]:
84
- while True:
85
- if self._exception:
86
- raise self._exception
84
+ try:
85
+ while True:
86
+ if self._exception:
87
+ raise self._exception
87
88
 
88
- while self._yield:
89
- yield self._yield.popleft()
89
+ while self._yield:
90
+ yield self._yield.popleft()
90
91
 
91
- if not self._futures:
92
- # No more tasks
93
- return
92
+ if not self._futures:
93
+ # No more tasks
94
+ return
94
95
 
95
- if self._futures[0].done():
96
- self._futures.popleft()
97
- else:
98
- sleep(0.001)
96
+ if self._futures[0].done():
97
+ self._futures.popleft()
98
+ else:
99
+ sleep(0.001)
100
+ finally:
101
+ # When the caller stops early (takes a sample, or hits an error), drop the queued work. Otherwise the
102
+ # pool finishes the whole diff in the background, piling up results in _yield that nobody will read.
103
+ self._pool.shutdown(wait=False, cancel_futures=True)
@@ -1,6 +1,6 @@
1
1
  [tool.poetry]
2
2
  name = "collate-data-diff"
3
- version = "0.11.16"
3
+ version = "0.11.18"
4
4
  description = "Command-line tool and Python library to efficiently diff rows across two different databases."
5
5
  authors = ["Collate <info@getcollate.io>", "Datafold <data-diff@datafold.com>"]
6
6
  license = "MIT"