collate-data-diff 0.11.16__tar.gz → 0.11.17__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/PKG-INFO +2 -1
- {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/diff_tables.py +31 -9
- {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/errors.py +10 -0
- {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/hashdiff_tables.py +4 -1
- {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/info_tree.py +7 -3
- {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/thread_utils.py +17 -12
- {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/pyproject.toml +1 -1
- {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/LICENSE +0 -0
- {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/README.md +0 -0
- {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/__init__.py +0 -0
- {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/__main__.py +0 -0
- {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/abcs/__init__.py +0 -0
- {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/abcs/compiler.py +0 -0
- {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/abcs/database_types.py +0 -0
- {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/cloud/__init__.py +0 -0
- {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/cloud/data_source.py +0 -0
- {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/cloud/datafold_api.py +0 -0
- {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/config.py +0 -0
- {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/databases/__init__.py +0 -0
- {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/databases/_connect.py +0 -0
- {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/databases/base.py +0 -0
- {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/databases/bigquery.py +0 -0
- {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/databases/clickhouse.py +0 -0
- {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/databases/databricks.py +0 -0
- {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/databases/duckdb.py +0 -0
- {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/databases/mssql.py +0 -0
- {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/databases/mysql.py +0 -0
- {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/databases/oracle.py +0 -0
- {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/databases/postgresql.py +0 -0
- {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/databases/presto.py +0 -0
- {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/databases/redshift.py +0 -0
- {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/databases/snowflake.py +0 -0
- {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/databases/trino.py +0 -0
- {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/databases/vertica.py +0 -0
- {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/dbt.py +0 -0
- {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/dbt_config_validators.py +0 -0
- {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/dbt_parser.py +0 -0
- {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/format.py +0 -0
- {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/joindiff_tables.py +0 -0
- {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/lexicographic_space.py +0 -0
- {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/parse_time.py +0 -0
- {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/py.typed +0 -0
- {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/queries/__init__.py +0 -0
- {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/queries/api.py +0 -0
- {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/queries/ast_classes.py +0 -0
- {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/queries/base.py +0 -0
- {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/queries/extras.py +0 -0
- {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/query_utils.py +0 -0
- {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/schema.py +0 -0
- {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/table_segment.py +0 -0
- {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/tracking.py +0 -0
- {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/utils.py +0 -0
- {collate_data_diff-0.11.16 → collate_data_diff-0.11.17}/data_diff/version.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: collate-data-diff
|
|
3
|
-
Version: 0.11.
|
|
3
|
+
Version: 0.11.17
|
|
4
4
|
Summary: Command-line tool and Python library to efficiently diff rows across two different databases.
|
|
5
5
|
License: MIT
|
|
6
6
|
License-File: LICENSE
|
|
@@ -19,6 +19,7 @@ Classifier: Programming Language :: Python :: 3.11
|
|
|
19
19
|
Classifier: Programming Language :: Python :: 3.12
|
|
20
20
|
Classifier: Programming Language :: Python :: 3.13
|
|
21
21
|
Classifier: Programming Language :: Python :: 3.14
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.15
|
|
22
23
|
Classifier: Programming Language :: Python :: 3.8
|
|
23
24
|
Classifier: Programming Language :: Python :: 3.9
|
|
24
25
|
Classifier: Topic :: Database :: Database Engines/Servers
|
|
@@ -5,13 +5,18 @@ import time
|
|
|
5
5
|
from abc import ABC, abstractmethod
|
|
6
6
|
from enum import Enum
|
|
7
7
|
from contextlib import contextmanager
|
|
8
|
+
from itertools import chain
|
|
8
9
|
from operator import methodcaller
|
|
9
10
|
from typing import Any, Dict, Set, List, Tuple, Iterator, Optional, Union
|
|
10
11
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
11
12
|
|
|
12
13
|
import attrs
|
|
13
14
|
|
|
14
|
-
from data_diff.errors import
|
|
15
|
+
from data_diff.errors import (
|
|
16
|
+
DataDiffDuplicateKeyError,
|
|
17
|
+
DataDiffMismatchingKeyTypesError,
|
|
18
|
+
DataDiffUnsupportedKeyValueError,
|
|
19
|
+
)
|
|
15
20
|
from data_diff.info_tree import InfoTree, SegmentInfo
|
|
16
21
|
from data_diff.utils import dbt_diff_string_template, run_as_daemon, safezip, getLogger, truncate_error, Vector
|
|
17
22
|
from data_diff.thread_utils import ThreadedYielder
|
|
@@ -96,8 +101,18 @@ class DiffResultWrapper:
|
|
|
96
101
|
self.result_list.append(i)
|
|
97
102
|
yield i
|
|
98
103
|
|
|
99
|
-
def _get_stats(self, is_dbt: bool = False) -> DiffStats:
|
|
100
|
-
|
|
104
|
+
def _get_stats(self, is_dbt: bool = False, retain_rows: bool = True) -> DiffStats:
|
|
105
|
+
"""Count the diff by key, consuming the rest of the diff.
|
|
106
|
+
|
|
107
|
+
With `retain_rows`, every row is kept in `result_list` so the diff can be iterated again afterwards.
|
|
108
|
+
Without it, rows not consumed yet are counted and dropped, so memory grows with the number of
|
|
109
|
+
differing keys rather than with every differing row and all of its columns.
|
|
110
|
+
"""
|
|
111
|
+
if retain_rows:
|
|
112
|
+
list(self) # Consume the iterator into result_list, if we haven't already
|
|
113
|
+
rows = self.result_list
|
|
114
|
+
else:
|
|
115
|
+
rows = chain(self.result_list, self.diff)
|
|
101
116
|
|
|
102
117
|
key_columns = self.info_tree.info.tables[0].key_columns
|
|
103
118
|
len_key_columns = len(key_columns)
|
|
@@ -108,12 +123,19 @@ class DiffResultWrapper:
|
|
|
108
123
|
extra_columns = self.info_tree.info.tables[0].extra_columns
|
|
109
124
|
extra_column_diffs = {k: 0 for k in extra_columns}
|
|
110
125
|
|
|
111
|
-
for sign, values in
|
|
126
|
+
for sign, values in rows:
|
|
112
127
|
k = values[:len_key_columns]
|
|
113
128
|
if is_dbt:
|
|
114
129
|
extra_column_values = values[len_key_columns:]
|
|
115
130
|
if k in diff_by_key:
|
|
116
|
-
|
|
131
|
+
if sign == diff_by_key[k]:
|
|
132
|
+
table_index = 1 if sign == "-" else 2
|
|
133
|
+
table = self.info_tree.info.tables[table_index - 1]
|
|
134
|
+
raise DataDiffDuplicateKeyError(
|
|
135
|
+
f"Key ({', '.join(key_columns)}) is not unique in table {'.'.join(table.table_path)}. "
|
|
136
|
+
"A row-level diff needs a unique key.",
|
|
137
|
+
table_index,
|
|
138
|
+
)
|
|
117
139
|
diff_by_key[k] = "!"
|
|
118
140
|
if is_dbt:
|
|
119
141
|
for i in range(0, len(extra_columns)):
|
|
@@ -135,8 +157,8 @@ class DiffResultWrapper:
|
|
|
135
157
|
|
|
136
158
|
return DiffStats(diff_by_sign, table1_count, table2_count, unchanged, diff_percent, extra_column_diffs)
|
|
137
159
|
|
|
138
|
-
def get_stats_string(self, is_dbt: bool = False):
|
|
139
|
-
diff_stats = self._get_stats(is_dbt)
|
|
160
|
+
def get_stats_string(self, is_dbt: bool = False, retain_rows: bool = True):
|
|
161
|
+
diff_stats = self._get_stats(is_dbt, retain_rows)
|
|
140
162
|
|
|
141
163
|
total_rows_diff = diff_stats.table2_count - diff_stats.table1_count
|
|
142
164
|
|
|
@@ -170,8 +192,8 @@ class DiffResultWrapper:
|
|
|
170
192
|
|
|
171
193
|
return string_output
|
|
172
194
|
|
|
173
|
-
def get_stats_dict(self, is_dbt: bool = False):
|
|
174
|
-
diff_stats = self._get_stats(is_dbt)
|
|
195
|
+
def get_stats_dict(self, is_dbt: bool = False, retain_rows: bool = True):
|
|
196
|
+
diff_stats = self._get_stats(is_dbt, retain_rows)
|
|
175
197
|
json_output = {
|
|
176
198
|
"rows_A": diff_stats.table1_count,
|
|
177
199
|
"rows_B": diff_stats.table2_count,
|
|
@@ -78,3 +78,13 @@ class DataDiffMismatchingKeyTypesError(Exception):
|
|
|
78
78
|
# _parse_key_range_result — keeps the exception contract for existing callers.
|
|
79
79
|
class DataDiffUnsupportedKeyValueError(ValueError):
|
|
80
80
|
"Raised when a key column holds values that cannot be mapped to a bisectable range (e.g. text keys with '.', accents or other non-ASCII characters)."
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
# Subclasses AssertionError because it replaces the bare assert previously used by
|
|
84
|
+
# DiffResultWrapper._get_stats — callers catching AssertionError keep working.
|
|
85
|
+
class DataDiffDuplicateKeyError(AssertionError):
|
|
86
|
+
"Raised when computing diff stats finds the same key twice in one table. `table_index` is 1 or 2."
|
|
87
|
+
|
|
88
|
+
def __init__(self, message: str, table_index: int) -> None:
|
|
89
|
+
super().__init__(message)
|
|
90
|
+
self.table_index = table_index
|
|
@@ -254,7 +254,10 @@ class HashDiffer(TableDiffer):
|
|
|
254
254
|
)
|
|
255
255
|
)
|
|
256
256
|
|
|
257
|
-
|
|
257
|
+
# Record only how many rows differ. The rows are yielded to the caller; keeping them on the
|
|
258
|
+
# info tree as well would hold every differing row in memory until the whole diff completes.
|
|
259
|
+
info_tree.info.diff_count = len(diff)
|
|
260
|
+
info_tree.info.is_diff = bool(diff)
|
|
258
261
|
info_tree.info.rowcounts = {1: len(rows1), 2: len(rows2)}
|
|
259
262
|
|
|
260
263
|
logger.info(". " * level + f"Diff found {len(diff)} different rows.")
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
from itertools import chain
|
|
1
2
|
from typing import List, Dict, Optional, Any, Tuple, Union
|
|
2
3
|
|
|
3
4
|
import attrs
|
|
@@ -34,11 +35,14 @@ class SegmentInfo:
|
|
|
34
35
|
self.diff_count = sum(c.diff_count for c in child_infos if c.diff_count is not None)
|
|
35
36
|
self.is_diff = any(c.is_diff for c in child_infos)
|
|
36
37
|
self.diff_schema = next((child.diff_schema for child in child_infos if child.diff_schema is not None), None)
|
|
37
|
-
|
|
38
|
+
# None when no child kept its rows (hashdiff keeps only counts), not an empty list that reads as "no diff"
|
|
39
|
+
child_diffs = [c.diff for c in child_infos if c.diff is not None]
|
|
40
|
+
self.diff = list(chain.from_iterable(child_diffs)) if child_diffs else None
|
|
38
41
|
|
|
42
|
+
# A diff closed early can leave a joindiff segment that has counted only one of the tables so far
|
|
39
43
|
self.rowcounts = {
|
|
40
|
-
1: sum(c.rowcounts
|
|
41
|
-
2: sum(c.rowcounts
|
|
44
|
+
1: sum(c.rowcounts.get(1, 0) for c in child_infos),
|
|
45
|
+
2: sum(c.rowcounts.get(2, 0) for c in child_infos),
|
|
42
46
|
}
|
|
43
47
|
|
|
44
48
|
|
|
@@ -81,18 +81,23 @@ class ThreadedYielder(Iterable):
|
|
|
81
81
|
self._futures.append(self._pool.submit(self._worker, fn, *args, priority=priority, **kwargs))
|
|
82
82
|
|
|
83
83
|
def __iter__(self) -> Iterator[Any]:
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
84
|
+
try:
|
|
85
|
+
while True:
|
|
86
|
+
if self._exception:
|
|
87
|
+
raise self._exception
|
|
87
88
|
|
|
88
|
-
|
|
89
|
-
|
|
89
|
+
while self._yield:
|
|
90
|
+
yield self._yield.popleft()
|
|
90
91
|
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
92
|
+
if not self._futures:
|
|
93
|
+
# No more tasks
|
|
94
|
+
return
|
|
94
95
|
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
96
|
+
if self._futures[0].done():
|
|
97
|
+
self._futures.popleft()
|
|
98
|
+
else:
|
|
99
|
+
sleep(0.001)
|
|
100
|
+
finally:
|
|
101
|
+
# When the caller stops early (takes a sample, or hits an error), drop the queued work. Otherwise the
|
|
102
|
+
# pool finishes the whole diff in the background, piling up results in _yield that nobody will read.
|
|
103
|
+
self._pool.shutdown(wait=False, cancel_futures=True)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[tool.poetry]
|
|
2
2
|
name = "collate-data-diff"
|
|
3
|
-
version = "0.11.
|
|
3
|
+
version = "0.11.17"
|
|
4
4
|
description = "Command-line tool and Python library to efficiently diff rows across two different databases."
|
|
5
5
|
authors = ["Collate <info@getcollate.io>", "Datafold <data-diff@datafold.com>"]
|
|
6
6
|
license = "MIT"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|