collate-data-diff 0.11.16__tar.gz → 0.11.17__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/PKG-INFO +2 -1
  2. {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/diff_tables.py +31 -9
  3. {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/errors.py +10 -0
  4. {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/hashdiff_tables.py +4 -1
  5. {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/info_tree.py +7 -3
  6. {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/thread_utils.py +17 -12
  7. {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/pyproject.toml +1 -1
  8. {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/LICENSE +0 -0
  9. {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/README.md +0 -0
  10. {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/__init__.py +0 -0
  11. {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/__main__.py +0 -0
  12. {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/abcs/__init__.py +0 -0
  13. {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/abcs/compiler.py +0 -0
  14. {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/abcs/database_types.py +0 -0
  15. {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/cloud/__init__.py +0 -0
  16. {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/cloud/data_source.py +0 -0
  17. {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/cloud/datafold_api.py +0 -0
  18. {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/config.py +0 -0
  19. {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/databases/__init__.py +0 -0
  20. {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/databases/_connect.py +0 -0
  21. {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/databases/base.py +0 -0
  22. {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/databases/bigquery.py +0 -0
  23. {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/databases/clickhouse.py +0 -0
  24. {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/databases/databricks.py +0 -0
  25. {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/databases/duckdb.py +0 -0
  26. {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/databases/mssql.py +0 -0
  27. {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/databases/mysql.py +0 -0
  28. {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/databases/oracle.py +0 -0
  29. {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/databases/postgresql.py +0 -0
  30. {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/databases/presto.py +0 -0
  31. {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/databases/redshift.py +0 -0
  32. {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/databases/snowflake.py +0 -0
  33. {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/databases/trino.py +0 -0
  34. {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/databases/vertica.py +0 -0
  35. {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/dbt.py +0 -0
  36. {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/dbt_config_validators.py +0 -0
  37. {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/dbt_parser.py +0 -0
  38. {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/format.py +0 -0
  39. {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/joindiff_tables.py +0 -0
  40. {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/lexicographic_space.py +0 -0
  41. {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/parse_time.py +0 -0
  42. {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/py.typed +0 -0
  43. {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/queries/__init__.py +0 -0
  44. {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/queries/api.py +0 -0
  45. {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/queries/ast_classes.py +0 -0
  46. {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/queries/base.py +0 -0
  47. {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/queries/extras.py +0 -0
  48. {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/query_utils.py +0 -0
  49. {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/schema.py +0 -0
  50. {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/table_segment.py +0 -0
  51. {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/tracking.py +0 -0
  52. {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/utils.py +0 -0
  53. {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/version.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: collate-data-diff
3
- Version: 0.11.16
3
+ Version: 0.11.17
4
4
  Summary: Command-line tool and Python library to efficiently diff rows across two different databases.
5
5
  License: MIT
6
6
  License-File: LICENSE
@@ -19,6 +19,7 @@ Classifier: Programming Language :: Python :: 3.11
19
19
  Classifier: Programming Language :: Python :: 3.12
20
20
  Classifier: Programming Language :: Python :: 3.13
21
21
  Classifier: Programming Language :: Python :: 3.14
22
+ Classifier: Programming Language :: Python :: 3.15
22
23
  Classifier: Programming Language :: Python :: 3.8
23
24
  Classifier: Programming Language :: Python :: 3.9
24
25
  Classifier: Topic :: Database :: Database Engines/Servers
@@ -5,13 +5,18 @@ import time
5
5
  from abc import ABC, abstractmethod
6
6
  from enum import Enum
7
7
  from contextlib import contextmanager
8
+ from itertools import chain
8
9
  from operator import methodcaller
9
10
  from typing import Any, Dict, Set, List, Tuple, Iterator, Optional, Union
10
11
  from concurrent.futures import ThreadPoolExecutor, as_completed
11
12
 
12
13
  import attrs
13
14
 
14
- from data_diff.errors import DataDiffMismatchingKeyTypesError, DataDiffUnsupportedKeyValueError
15
+ from data_diff.errors import (
16
+ DataDiffDuplicateKeyError,
17
+ DataDiffMismatchingKeyTypesError,
18
+ DataDiffUnsupportedKeyValueError,
19
+ )
15
20
  from data_diff.info_tree import InfoTree, SegmentInfo
16
21
  from data_diff.utils import dbt_diff_string_template, run_as_daemon, safezip, getLogger, truncate_error, Vector
17
22
  from data_diff.thread_utils import ThreadedYielder
@@ -96,8 +101,18 @@ class DiffResultWrapper:
96
101
  self.result_list.append(i)
97
102
  yield i
98
103
 
99
- def _get_stats(self, is_dbt: bool = False) -> DiffStats:
100
- list(self) # Consume the iterator into result_list, if we haven't already
104
+ def _get_stats(self, is_dbt: bool = False, retain_rows: bool = True) -> DiffStats:
105
+ """Count the diff by key, consuming the rest of the diff.
106
+
107
+ With `retain_rows`, every row is kept in `result_list` so the diff can be iterated again afterwards.
108
+ Without it, rows not consumed yet are counted and dropped, so memory grows with the number of
109
+ differing keys rather than with every differing row and all of its columns.
110
+ """
111
+ if retain_rows:
112
+ list(self) # Consume the iterator into result_list, if we haven't already
113
+ rows = self.result_list
114
+ else:
115
+ rows = chain(self.result_list, self.diff)
101
116
 
102
117
  key_columns = self.info_tree.info.tables[0].key_columns
103
118
  len_key_columns = len(key_columns)
@@ -108,12 +123,19 @@ class DiffResultWrapper:
108
123
  extra_columns = self.info_tree.info.tables[0].extra_columns
109
124
  extra_column_diffs = {k: 0 for k in extra_columns}
110
125
 
111
- for sign, values in self.result_list:
126
+ for sign, values in rows:
112
127
  k = values[:len_key_columns]
113
128
  if is_dbt:
114
129
  extra_column_values = values[len_key_columns:]
115
130
  if k in diff_by_key:
116
- assert sign != diff_by_key[k]
131
+ if sign == diff_by_key[k]:
132
+ table_index = 1 if sign == "-" else 2
133
+ table = self.info_tree.info.tables[table_index - 1]
134
+ raise DataDiffDuplicateKeyError(
135
+ f"Key ({', '.join(key_columns)}) is not unique in table {'.'.join(table.table_path)}. "
136
+ "A row-level diff needs a unique key.",
137
+ table_index,
138
+ )
117
139
  diff_by_key[k] = "!"
118
140
  if is_dbt:
119
141
  for i in range(0, len(extra_columns)):
@@ -135,8 +157,8 @@ class DiffResultWrapper:
135
157
 
136
158
  return DiffStats(diff_by_sign, table1_count, table2_count, unchanged, diff_percent, extra_column_diffs)
137
159
 
138
- def get_stats_string(self, is_dbt: bool = False):
139
- diff_stats = self._get_stats(is_dbt)
160
+ def get_stats_string(self, is_dbt: bool = False, retain_rows: bool = True):
161
+ diff_stats = self._get_stats(is_dbt, retain_rows)
140
162
 
141
163
  total_rows_diff = diff_stats.table2_count - diff_stats.table1_count
142
164
 
@@ -170,8 +192,8 @@ class DiffResultWrapper:
170
192
 
171
193
  return string_output
172
194
 
173
- def get_stats_dict(self, is_dbt: bool = False):
174
- diff_stats = self._get_stats(is_dbt)
195
+ def get_stats_dict(self, is_dbt: bool = False, retain_rows: bool = True):
196
+ diff_stats = self._get_stats(is_dbt, retain_rows)
175
197
  json_output = {
176
198
  "rows_A": diff_stats.table1_count,
177
199
  "rows_B": diff_stats.table2_count,
@@ -78,3 +78,13 @@ class DataDiffMismatchingKeyTypesError(Exception):
78
78
  # _parse_key_range_result — keeps the exception contract for existing callers.
79
79
  class DataDiffUnsupportedKeyValueError(ValueError):
80
80
  "Raised when a key column holds values that cannot be mapped to a bisectable range (e.g. text keys with '.', accents or other non-ASCII characters)."
81
+
82
+
83
+ # Subclasses AssertionError because it replaces the bare assert previously used by
84
+ # DiffResultWrapper._get_stats — callers catching AssertionError keep working.
85
+ class DataDiffDuplicateKeyError(AssertionError):
86
+ "Raised when computing diff stats finds the same key twice in one table. `table_index` is 1 or 2."
87
+
88
+ def __init__(self, message: str, table_index: int) -> None:
89
+ super().__init__(message)
90
+ self.table_index = table_index
@@ -254,7 +254,10 @@ class HashDiffer(TableDiffer):
254
254
  )
255
255
  )
256
256
 
257
- info_tree.info.set_diff(diff)
257
+ # Record only how many rows differ. The rows are yielded to the caller; keeping them on the
258
+ # info tree as well would hold every differing row in memory until the whole diff completes.
259
+ info_tree.info.diff_count = len(diff)
260
+ info_tree.info.is_diff = bool(diff)
258
261
  info_tree.info.rowcounts = {1: len(rows1), 2: len(rows2)}
259
262
 
260
263
  logger.info(". " * level + f"Diff found {len(diff)} different rows.")
@@ -1,3 +1,4 @@
1
+ from itertools import chain
1
2
  from typing import List, Dict, Optional, Any, Tuple, Union
2
3
 
3
4
  import attrs
@@ -34,11 +35,14 @@ class SegmentInfo:
34
35
  self.diff_count = sum(c.diff_count for c in child_infos if c.diff_count is not None)
35
36
  self.is_diff = any(c.is_diff for c in child_infos)
36
37
  self.diff_schema = next((child.diff_schema for child in child_infos if child.diff_schema is not None), None)
37
- self.diff = sum((c.diff for c in child_infos if c.diff is not None), [])
38
+ # None when no child kept its rows (hashdiff keeps only counts), not an empty list that reads as "no diff"
39
+ child_diffs = [c.diff for c in child_infos if c.diff is not None]
40
+ self.diff = list(chain.from_iterable(child_diffs)) if child_diffs else None
38
41
 
42
+ # A diff closed early can leave a joindiff segment that has counted only one of the tables so far
39
43
  self.rowcounts = {
40
- 1: sum(c.rowcounts[1] for c in child_infos if c.rowcounts),
41
- 2: sum(c.rowcounts[2] for c in child_infos if c.rowcounts),
44
+ 1: sum(c.rowcounts.get(1, 0) for c in child_infos),
45
+ 2: sum(c.rowcounts.get(2, 0) for c in child_infos),
42
46
  }
43
47
 
44
48
 
@@ -81,18 +81,23 @@ class ThreadedYielder(Iterable):
81
81
  self._futures.append(self._pool.submit(self._worker, fn, *args, priority=priority, **kwargs))
82
82
 
83
83
  def __iter__(self) -> Iterator[Any]:
84
- while True:
85
- if self._exception:
86
- raise self._exception
84
+ try:
85
+ while True:
86
+ if self._exception:
87
+ raise self._exception
87
88
 
88
- while self._yield:
89
- yield self._yield.popleft()
89
+ while self._yield:
90
+ yield self._yield.popleft()
90
91
 
91
- if not self._futures:
92
- # No more tasks
93
- return
92
+ if not self._futures:
93
+ # No more tasks
94
+ return
94
95
 
95
- if self._futures[0].done():
96
- self._futures.popleft()
97
- else:
98
- sleep(0.001)
96
+ if self._futures[0].done():
97
+ self._futures.popleft()
98
+ else:
99
+ sleep(0.001)
100
+ finally:
101
+ # When the caller stops early (takes a sample, or hits an error), drop the queued work. Otherwise the
102
+ # pool finishes the whole diff in the background, piling up results in _yield that nobody will read.
103
+ self._pool.shutdown(wait=False, cancel_futures=True)
@@ -1,6 +1,6 @@
1
1
  [tool.poetry]
2
2
  name = "collate-data-diff"
3
- version = "0.11.16"
3
+ version = "0.11.17"
4
4
  description = "Command-line tool and Python library to efficiently diff rows across two different databases."
5
5
  authors = ["Collate <info@getcollate.io>", "Datafold <data-diff@datafold.com>"]
6
6
  license = "MIT"