recce-nightly 1.3.0.20250507__py3-none-any.whl → 1.4.0.20250515__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of recce-nightly might be problematic. Click here for more details.
- recce/VERSION +1 -1
- recce/__init__.py +22 -22
- recce/adapter/base.py +11 -14
- recce/adapter/dbt_adapter/__init__.py +355 -316
- recce/adapter/dbt_adapter/dbt_version.py +3 -0
- recce/adapter/sqlmesh_adapter.py +24 -35
- recce/apis/check_api.py +39 -28
- recce/apis/check_func.py +33 -27
- recce/apis/run_api.py +25 -19
- recce/apis/run_func.py +29 -23
- recce/artifact.py +44 -49
- recce/cli.py +484 -285
- recce/config.py +42 -33
- recce/core.py +52 -44
- recce/data/404.html +1 -1
- recce/data/_next/static/chunks/{368-7587b306577df275.js → 778-aef312bffb4c0312.js} +15 -15
- recce/data/_next/static/chunks/8d700b6a.ed11a130057c7a47.js +1 -0
- recce/data/_next/static/chunks/app/layout-c713a2829d3279e4.js +1 -0
- recce/data/_next/static/chunks/app/page-7086764277331fcb.js +1 -0
- recce/data/_next/static/chunks/{cd9f8d63-cf0d5a7b0f7a92e8.js → cd9f8d63-e020f408095ed77c.js} +3 -3
- recce/data/_next/static/chunks/webpack-b787cb1a4f2293de.js +1 -0
- recce/data/_next/static/css/88b8abc134cfd59a.css +3 -0
- recce/data/index.html +2 -2
- recce/data/index.txt +2 -2
- recce/diff.py +6 -12
- recce/event/__init__.py +74 -72
- recce/event/collector.py +27 -20
- recce/event/track.py +39 -27
- recce/exceptions.py +1 -1
- recce/git.py +7 -7
- recce/github.py +57 -53
- recce/models/__init__.py +1 -1
- recce/models/check.py +6 -7
- recce/models/run.py +1 -0
- recce/models/types.py +27 -27
- recce/pull_request.py +26 -24
- recce/run.py +148 -111
- recce/server.py +103 -89
- recce/state.py +209 -177
- recce/summary.py +168 -143
- recce/tasks/__init__.py +3 -3
- recce/tasks/core.py +11 -13
- recce/tasks/dataframe.py +19 -17
- recce/tasks/histogram.py +69 -34
- recce/tasks/lineage.py +2 -2
- recce/tasks/profile.py +147 -86
- recce/tasks/query.py +139 -87
- recce/tasks/rowcount.py +33 -30
- recce/tasks/schema.py +14 -14
- recce/tasks/top_k.py +35 -35
- recce/tasks/valuediff.py +216 -152
- recce/util/breaking.py +77 -84
- recce/util/cll.py +55 -51
- recce/util/io.py +19 -17
- recce/util/logger.py +1 -1
- recce/util/recce_cloud.py +70 -72
- recce/util/singleton.py +4 -4
- recce/yaml/__init__.py +7 -10
- {recce_nightly-1.3.0.20250507.dist-info → recce_nightly-1.4.0.20250515.dist-info}/METADATA +5 -2
- recce_nightly-1.4.0.20250515.dist-info/RECORD +143 -0
- {recce_nightly-1.3.0.20250507.dist-info → recce_nightly-1.4.0.20250515.dist-info}/WHEEL +1 -1
- tests/adapter/dbt_adapter/conftest.py +1 -0
- tests/adapter/dbt_adapter/dbt_test_helper.py +28 -18
- tests/adapter/dbt_adapter/test_dbt_adapter.py +0 -15
- tests/adapter/dbt_adapter/test_dbt_cll.py +39 -32
- tests/adapter/dbt_adapter/test_selector.py +22 -21
- tests/tasks/test_histogram.py +58 -66
- tests/tasks/test_lineage.py +36 -23
- tests/tasks/test_preset_checks.py +45 -31
- tests/tasks/test_profile.py +340 -15
- tests/tasks/test_query.py +40 -40
- tests/tasks/test_row_count.py +65 -46
- tests/tasks/test_schema.py +65 -42
- tests/tasks/test_top_k.py +22 -18
- tests/tasks/test_valuediff.py +43 -32
- tests/test_cli.py +71 -58
- tests/test_config.py +7 -9
- tests/test_core.py +5 -3
- tests/test_dbt.py +7 -7
- tests/test_pull_request.py +1 -1
- tests/test_server.py +19 -13
- tests/test_state.py +40 -27
- tests/test_summary.py +18 -14
- recce/data/_next/static/chunks/8d700b6a-f0b1f6b9e0d97ce2.js +0 -1
- recce/data/_next/static/chunks/app/layout-9102e22cb73f74d6.js +0 -1
- recce/data/_next/static/chunks/app/page-92f13c8fad9fae3d.js +0 -1
- recce/data/_next/static/chunks/webpack-567d72f0bc0820d5.js +0 -1
- recce_nightly-1.3.0.20250507.dist-info/RECORD +0 -142
- /recce/data/_next/static/{K5iKlCYhdcpq8Ea6ck9J_ → q0Xsc9Sd6PDuo1lshYpLu}/_buildManifest.js +0 -0
- /recce/data/_next/static/{K5iKlCYhdcpq8Ea6ck9J_ → q0Xsc9Sd6PDuo1lshYpLu}/_ssgManifest.js +0 -0
- {recce_nightly-1.3.0.20250507.dist-info → recce_nightly-1.4.0.20250515.dist-info}/entry_points.txt +0 -0
- {recce_nightly-1.3.0.20250507.dist-info → recce_nightly-1.4.0.20250515.dist-info}/licenses/LICENSE +0 -0
- {recce_nightly-1.3.0.20250507.dist-info → recce_nightly-1.4.0.20250515.dist-info}/top_level.txt +0 -0
|
@@ -4,8 +4,18 @@ import uuid
|
|
|
4
4
|
from datetime import datetime
|
|
5
5
|
from io import StringIO
|
|
6
6
|
|
|
7
|
-
from dbt.contracts.graph.nodes import
|
|
8
|
-
|
|
7
|
+
from dbt.contracts.graph.nodes import (
|
|
8
|
+
ModelNode,
|
|
9
|
+
SeedNode,
|
|
10
|
+
SnapshotNode,
|
|
11
|
+
SourceDefinition,
|
|
12
|
+
)
|
|
13
|
+
from dbt.contracts.results import (
|
|
14
|
+
CatalogArtifact,
|
|
15
|
+
CatalogTable,
|
|
16
|
+
ColumnMetadata,
|
|
17
|
+
TableMetadata,
|
|
18
|
+
)
|
|
9
19
|
|
|
10
20
|
from recce.adapter.dbt_adapter import DbtAdapter, as_manifest, load_manifest
|
|
11
21
|
from recce.core import RecceContext
|
|
@@ -19,9 +29,9 @@ class DbtTestHelper:
|
|
|
19
29
|
self.curr_schema = f"{schema_prefix}_curr"
|
|
20
30
|
|
|
21
31
|
current_dir = os.path.dirname(os.path.abspath(__file__))
|
|
22
|
-
project_dir = os.path.join(current_dir,
|
|
32
|
+
project_dir = os.path.join(current_dir, "test_proj")
|
|
23
33
|
profiles_dir = project_dir
|
|
24
|
-
manifest_path = os.path.join(project_dir,
|
|
34
|
+
manifest_path = os.path.join(project_dir, "manifest.json")
|
|
25
35
|
|
|
26
36
|
dbt_adapter = DbtAdapter.load(
|
|
27
37
|
no_artifacts=True,
|
|
@@ -30,7 +40,7 @@ class DbtTestHelper:
|
|
|
30
40
|
)
|
|
31
41
|
|
|
32
42
|
context = RecceContext()
|
|
33
|
-
context.adapter_type =
|
|
43
|
+
context.adapter_type = "dbt"
|
|
34
44
|
context.adapter = dbt_adapter
|
|
35
45
|
context.schema_prefix = schema_prefix
|
|
36
46
|
self.adapter = dbt_adapter
|
|
@@ -108,16 +118,16 @@ class DbtTestHelper:
|
|
|
108
118
|
if csv:
|
|
109
119
|
dbt_adapter = self.adapter
|
|
110
120
|
csv = textwrap.dedent(csv)
|
|
111
|
-
with dbt_adapter.connection_named(
|
|
121
|
+
with dbt_adapter.connection_named("create model"):
|
|
112
122
|
import pandas as pd
|
|
123
|
+
|
|
113
124
|
df = pd.read_csv(StringIO(csv))
|
|
114
125
|
dbt_adapter.execute(f"CREATE TABLE {schema}.{model_name} AS SELECT * FROM df")
|
|
115
126
|
raw_code = sql if sql else csv
|
|
116
127
|
|
|
117
128
|
if columns:
|
|
118
129
|
index = 1
|
|
119
|
-
table = CatalogTable(
|
|
120
|
-
TableMetadata(type="BASE TABLE", schema=schema, name=model_name), {}, {})
|
|
130
|
+
table = CatalogTable(TableMetadata(type="BASE TABLE", schema=schema, name=model_name), {}, {})
|
|
121
131
|
catalog.nodes[unique_id] = table
|
|
122
132
|
for column, column_type in columns.items():
|
|
123
133
|
col_data = ColumnMetadata(type=column_type, index=index, name=column)
|
|
@@ -147,9 +157,7 @@ class DbtTestHelper:
|
|
|
147
157
|
"tags": ["test_tag"],
|
|
148
158
|
},
|
|
149
159
|
"tags": ["test_tag"],
|
|
150
|
-
"depends_on": {
|
|
151
|
-
"nodes": depends_on
|
|
152
|
-
},
|
|
160
|
+
"depends_on": {"nodes": depends_on},
|
|
153
161
|
}
|
|
154
162
|
if patch_func:
|
|
155
163
|
patch_func(node_dict)
|
|
@@ -179,11 +187,12 @@ class DbtTestHelper:
|
|
|
179
187
|
self.curr_manifest,
|
|
180
188
|
self.base_manifest,
|
|
181
189
|
self.base_catalog,
|
|
182
|
-
self.curr_catalog
|
|
190
|
+
self.curr_catalog,
|
|
191
|
+
)
|
|
183
192
|
|
|
184
193
|
def remove_model(self, model_name):
|
|
185
194
|
dbt_adapter = self.adapter
|
|
186
|
-
with dbt_adapter.connection_named(
|
|
195
|
+
with dbt_adapter.connection_named("cleanup"):
|
|
187
196
|
dbt_adapter.execute(f"DROP TABLE IF EXISTS {self.base_schema}.{model_name}")
|
|
188
197
|
dbt_adapter.execute(f"DROP TABLE IF EXISTS {self.curr_schema}.{model_name} ")
|
|
189
198
|
|
|
@@ -219,15 +228,15 @@ class DbtTestHelper:
|
|
|
219
228
|
if csv:
|
|
220
229
|
dbt_adapter = self.adapter
|
|
221
230
|
csv = textwrap.dedent(csv)
|
|
222
|
-
with dbt_adapter.connection_named(
|
|
231
|
+
with dbt_adapter.connection_named("create source"):
|
|
223
232
|
import pandas as pd
|
|
233
|
+
|
|
224
234
|
df = pd.read_csv(StringIO(csv))
|
|
225
235
|
dbt_adapter.execute(f"CREATE TABLE {schema}.{table_name} AS SELECT * FROM df")
|
|
226
236
|
|
|
227
237
|
if columns:
|
|
228
238
|
index = 1
|
|
229
|
-
table = CatalogTable(
|
|
230
|
-
TableMetadata(type="BASE TABLE", schema=schema, name=table_name), {}, {})
|
|
239
|
+
table = CatalogTable(TableMetadata(type="BASE TABLE", schema=schema, name=table_name), {}, {})
|
|
231
240
|
catalog.sources[unique_id] = table
|
|
232
241
|
for column, column_type in columns.items():
|
|
233
242
|
col_data = ColumnMetadata(type=column_type, index=index, name=column)
|
|
@@ -271,11 +280,12 @@ class DbtTestHelper:
|
|
|
271
280
|
self.curr_manifest,
|
|
272
281
|
self.base_manifest,
|
|
273
282
|
self.base_catalog,
|
|
274
|
-
self.curr_catalog
|
|
283
|
+
self.curr_catalog,
|
|
284
|
+
)
|
|
275
285
|
|
|
276
286
|
def cleanup(self):
|
|
277
287
|
dbt_adapter = self.adapter
|
|
278
|
-
with dbt_adapter.connection_named(
|
|
288
|
+
with dbt_adapter.connection_named("cleanup"):
|
|
279
289
|
dbt_adapter.execute(f"DROP SCHEMA IF EXISTS {self.base_schema} CASCADE")
|
|
280
290
|
dbt_adapter.execute(f"DROP SCHEMA IF EXISTS {self.curr_schema} CASCADE")
|
|
281
291
|
|
|
@@ -1,15 +1,9 @@
|
|
|
1
1
|
from recce.adapter.dbt_adapter import DbtAdapter, dbt_supported_registry
|
|
2
|
-
from recce.models import RunType
|
|
3
2
|
|
|
4
3
|
|
|
5
4
|
def test_dbt_adapter_support_tasks(dbt_test_helper):
|
|
6
5
|
adapter: DbtAdapter = dbt_test_helper.context.adapter
|
|
7
6
|
|
|
8
|
-
# Test dbt package name
|
|
9
|
-
supported_dbt_packages = set([package.package_name for package in adapter.manifest.macros.values()])
|
|
10
|
-
assert 'audit_helper' in supported_dbt_packages
|
|
11
|
-
assert 'dbt_profiler' in supported_dbt_packages
|
|
12
|
-
|
|
13
7
|
# Test dbt task support
|
|
14
8
|
support_tasks = adapter.support_tasks()
|
|
15
9
|
|
|
@@ -29,12 +23,3 @@ def test_dbt_adapter_support_tasks_without_required_dbt_package(dbt_test_helper)
|
|
|
29
23
|
for task_type in dbt_supported_registry:
|
|
30
24
|
task = task_type.value
|
|
31
25
|
assert task in support_tasks
|
|
32
|
-
if task == RunType.PROFILE_DIFF.value:
|
|
33
|
-
assert support_tasks[task] is False
|
|
34
|
-
elif task == RunType.VALUE_DIFF.value:
|
|
35
|
-
assert support_tasks[task] is False
|
|
36
|
-
elif task == RunType.VALUE_DIFF_DETAIL.value:
|
|
37
|
-
assert support_tasks[task] is False
|
|
38
|
-
|
|
39
|
-
# Check the query_diff_with_primary_key is not supported
|
|
40
|
-
assert support_tasks["query_diff_with_primary_key"] is False
|
|
@@ -3,25 +3,27 @@ from recce.adapter.dbt_adapter import DbtAdapter
|
|
|
3
3
|
|
|
4
4
|
def test_cll_basic(dbt_test_helper):
|
|
5
5
|
dbt_test_helper.create_model("model1", curr_sql="select 1 as c", curr_columns={"c": "int"})
|
|
6
|
-
dbt_test_helper.create_model(
|
|
7
|
-
|
|
6
|
+
dbt_test_helper.create_model(
|
|
7
|
+
"model2", curr_sql='select c from {{ ref("model1") }}', curr_columns={"c": "int"}, depends_on=["model1"]
|
|
8
|
+
)
|
|
8
9
|
adapter: DbtAdapter = dbt_test_helper.context.adapter
|
|
9
10
|
result = adapter.get_cll_by_node_id("model1")
|
|
10
|
-
assert result[
|
|
11
|
-
assert result[
|
|
11
|
+
assert result["nodes"]["model2"]["columns"]["c"]["depends_on"][0].column == "c"
|
|
12
|
+
assert result["nodes"]["model2"]["columns"]["c"]["depends_on"][0].node == "model1"
|
|
12
13
|
|
|
13
14
|
|
|
14
15
|
def test_cll_table_alisa(dbt_test_helper):
|
|
15
16
|
def patch_node(node):
|
|
16
|
-
node[
|
|
17
|
+
node["alias"] = "model1_alias"
|
|
17
18
|
|
|
18
19
|
dbt_test_helper.create_model("model1", curr_sql="select 1 as c", curr_columns={"c": "int"}, patch_func=patch_node)
|
|
19
|
-
dbt_test_helper.create_model(
|
|
20
|
-
|
|
20
|
+
dbt_test_helper.create_model(
|
|
21
|
+
"model2", curr_sql='select c from {{ ref("model1") }}', curr_columns={"c": "int"}, depends_on=["model1"]
|
|
22
|
+
)
|
|
21
23
|
adapter: DbtAdapter = dbt_test_helper.context.adapter
|
|
22
24
|
result = adapter.get_cll_by_node_id("model1")
|
|
23
|
-
assert result[
|
|
24
|
-
assert result[
|
|
25
|
+
assert result["nodes"]["model2"]["columns"]["c"]["depends_on"][0].column == "c"
|
|
26
|
+
assert result["nodes"]["model2"]["columns"]["c"]["depends_on"][0].node == "model1"
|
|
25
27
|
|
|
26
28
|
|
|
27
29
|
def test_seed(dbt_test_helper):
|
|
@@ -32,20 +34,22 @@ def test_seed(dbt_test_helper):
|
|
|
32
34
|
3,Charlie,35
|
|
33
35
|
"""
|
|
34
36
|
|
|
35
|
-
dbt_test_helper.create_model(
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
37
|
+
dbt_test_helper.create_model(
|
|
38
|
+
"seed1",
|
|
39
|
+
curr_csv=csv_data_curr,
|
|
40
|
+
curr_columns={"customer_id": "varchar", "name": "varchar", "age": "int"},
|
|
41
|
+
resource_type="seed",
|
|
42
|
+
)
|
|
39
43
|
adapter: DbtAdapter = dbt_test_helper.context.adapter
|
|
40
44
|
result = adapter.get_cll_by_node_id("seed1")
|
|
41
45
|
|
|
42
|
-
assert result[
|
|
43
|
-
assert len(result[
|
|
46
|
+
assert result["nodes"]["seed1"]["columns"]["customer_id"]["transformation_type"] == "source"
|
|
47
|
+
assert len(result["nodes"]["seed1"]["columns"]["customer_id"]["depends_on"]) == 0
|
|
44
48
|
|
|
45
49
|
|
|
46
50
|
def test_python_model(dbt_test_helper):
|
|
47
51
|
def python_node(node):
|
|
48
|
-
node[
|
|
52
|
+
node["language"] = "python"
|
|
49
53
|
|
|
50
54
|
csv_data_curr = """
|
|
51
55
|
customer_id,name,age
|
|
@@ -53,20 +57,22 @@ def test_python_model(dbt_test_helper):
|
|
|
53
57
|
2,Bob,25
|
|
54
58
|
3,Charlie,35
|
|
55
59
|
"""
|
|
56
|
-
dbt_test_helper.create_model(
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
dbt_test_helper.create_model(
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
60
|
+
dbt_test_helper.create_model(
|
|
61
|
+
"model1", curr_csv=csv_data_curr, curr_columns={"customer_id": "varchar", "name": "varchar", "age": "int"}
|
|
62
|
+
)
|
|
63
|
+
dbt_test_helper.create_model(
|
|
64
|
+
"model2",
|
|
65
|
+
curr_csv=csv_data_curr,
|
|
66
|
+
curr_columns={"customer_id": "varchar", "name": "varchar", "age": "int"},
|
|
67
|
+
depends_on=["model1"],
|
|
68
|
+
patch_func=python_node,
|
|
69
|
+
)
|
|
64
70
|
adapter: DbtAdapter = dbt_test_helper.context.adapter
|
|
65
|
-
assert not adapter.is_python_model(
|
|
66
|
-
assert adapter.is_python_model(
|
|
71
|
+
assert not adapter.is_python_model("model1")
|
|
72
|
+
assert adapter.is_python_model("model2")
|
|
67
73
|
|
|
68
74
|
result = adapter.get_cll_by_node_id("model1")
|
|
69
|
-
assert result[
|
|
75
|
+
assert result["nodes"]["model2"]["columns"]["customer_id"]["transformation_type"] == "unknown"
|
|
70
76
|
|
|
71
77
|
|
|
72
78
|
def test_source(dbt_test_helper):
|
|
@@ -81,22 +87,23 @@ def test_source(dbt_test_helper):
|
|
|
81
87
|
"source1",
|
|
82
88
|
"table1",
|
|
83
89
|
curr_csv=csv_data_curr,
|
|
84
|
-
curr_columns={"customer_id": "varchar", "name": "varchar", "age": "int"}
|
|
90
|
+
curr_columns={"customer_id": "varchar", "name": "varchar", "age": "int"},
|
|
91
|
+
)
|
|
85
92
|
adapter: DbtAdapter = dbt_test_helper.context.adapter
|
|
86
93
|
result = adapter.get_cll_by_node_id("source1.table1")
|
|
87
|
-
assert result[
|
|
94
|
+
assert result["nodes"]["source1.table1"]["columns"]["customer_id"]["transformation_type"] == "source"
|
|
88
95
|
|
|
89
96
|
|
|
90
97
|
def test_parse_error(dbt_test_helper):
|
|
91
98
|
dbt_test_helper.create_model("model1", curr_sql="select 1 as c", curr_columns={"c": "int"})
|
|
92
|
-
dbt_test_helper.create_model("model2", curr_sql=
|
|
99
|
+
dbt_test_helper.create_model("model2", curr_sql="this is not a valid sql", curr_columns={"c": "int"})
|
|
93
100
|
adapter: DbtAdapter = dbt_test_helper.context.adapter
|
|
94
101
|
result = adapter.get_cll_by_node_id("model2")
|
|
95
|
-
assert result[
|
|
102
|
+
assert result["nodes"]["model2"]["columns"]["c"]["transformation_type"] == "unknown"
|
|
96
103
|
|
|
97
104
|
|
|
98
105
|
def test_model_without_catalog(dbt_test_helper):
|
|
99
106
|
dbt_test_helper.create_model("model1", curr_sql="select 1 as c")
|
|
100
107
|
adapter: DbtAdapter = dbt_test_helper.context.adapter
|
|
101
108
|
result = adapter.get_cll_by_node_id("model1")
|
|
102
|
-
assert not hasattr(result[
|
|
109
|
+
assert not hasattr(result["nodes"]["model1"], "columns")
|
|
@@ -21,13 +21,13 @@ def test_select(dbt_test_helper):
|
|
|
21
21
|
adapter: DbtAdapter = dbt_test_helper.context.adapter
|
|
22
22
|
|
|
23
23
|
# Test methods
|
|
24
|
-
node_ids = adapter.select_nodes(
|
|
24
|
+
node_ids = adapter.select_nodes("customers_1")
|
|
25
25
|
assert len(node_ids) == 1
|
|
26
|
-
node_ids = adapter.select_nodes(
|
|
26
|
+
node_ids = adapter.select_nodes("resource_type:model")
|
|
27
27
|
assert len(node_ids) == 2
|
|
28
|
-
node_ids = adapter.select_nodes(
|
|
28
|
+
node_ids = adapter.select_nodes("tag:test_tag")
|
|
29
29
|
assert len(node_ids) == 2
|
|
30
|
-
node_ids = adapter.select_nodes(
|
|
30
|
+
node_ids = adapter.select_nodes("tag:test_tag2")
|
|
31
31
|
assert len(node_ids) == 0
|
|
32
32
|
node_ids = adapter.select_nodes("config.materialized:incremental")
|
|
33
33
|
assert len(node_ids) == 0
|
|
@@ -47,9 +47,9 @@ def test_select(dbt_test_helper):
|
|
|
47
47
|
assert len(node_ids) == 2
|
|
48
48
|
node_ids = adapter.select_nodes("config.materialized:table,tag:test_tag2")
|
|
49
49
|
assert len(node_ids) == 0
|
|
50
|
-
node_ids = adapter.select_nodes(exclude=
|
|
50
|
+
node_ids = adapter.select_nodes(exclude="customers_1")
|
|
51
51
|
assert len(node_ids) == 1
|
|
52
|
-
node_ids = adapter.select_nodes(
|
|
52
|
+
node_ids = adapter.select_nodes("customers_1", exclude="customers_2")
|
|
53
53
|
assert len(node_ids) == 1
|
|
54
54
|
|
|
55
55
|
# Test graph operation
|
|
@@ -62,10 +62,10 @@ def test_select(dbt_test_helper):
|
|
|
62
62
|
dbt_test_helper.create_snapshot("snapshot_1", csv_data_base, csv_data_curr)
|
|
63
63
|
dbt_test_helper.create_model("use_snapshot", csv_data_base, csv_data_base, depends_on=["snapshot_1"])
|
|
64
64
|
|
|
65
|
-
node_ids = adapter.select_nodes(
|
|
65
|
+
node_ids = adapter.select_nodes("resource_type:snapshot")
|
|
66
66
|
assert len(node_ids) == 1
|
|
67
67
|
|
|
68
|
-
node_ids = adapter.select_nodes(
|
|
68
|
+
node_ids = adapter.select_nodes("resource_type:snapshot+")
|
|
69
69
|
assert len(node_ids) == 2
|
|
70
70
|
|
|
71
71
|
node_ids = adapter.select_nodes("state:modified,resource_type:snapshot")
|
|
@@ -172,30 +172,31 @@ def test_select_with_pacakage_mode_include_exclude(dbt_test_helper):
|
|
|
172
172
|
Only customers_3 is changed
|
|
173
173
|
"""
|
|
174
174
|
dbt_test_helper.create_model("customers_1", csv_data_base, csv_data_base)
|
|
175
|
-
dbt_test_helper.create_model("customers_2", csv_data_base, csv_data_base, depends_on=[
|
|
176
|
-
dbt_test_helper.create_model("customers_3", csv_data_base, csv_data_curr, depends_on=[
|
|
177
|
-
dbt_test_helper.create_model(
|
|
178
|
-
|
|
179
|
-
|
|
175
|
+
dbt_test_helper.create_model("customers_2", csv_data_base, csv_data_base, depends_on=["customers_1"])
|
|
176
|
+
dbt_test_helper.create_model("customers_3", csv_data_base, csv_data_curr, depends_on=["customers_2"])
|
|
177
|
+
dbt_test_helper.create_model(
|
|
178
|
+
"customers_4", csv_data_base, csv_data_base, depends_on=["customers_3"], package_name="other_package"
|
|
179
|
+
)
|
|
180
|
+
dbt_test_helper.create_model("customers_5", csv_data_base, csv_data_base, depends_on=["customers_3"])
|
|
180
181
|
|
|
181
182
|
adapter: DbtAdapter = dbt_test_helper.context.adapter
|
|
182
183
|
|
|
183
|
-
node_ids = adapter.select_nodes(packages=[
|
|
184
|
+
node_ids = adapter.select_nodes(packages=["other_package"])
|
|
184
185
|
assert len(node_ids) == 1
|
|
185
186
|
|
|
186
|
-
node_ids = adapter.select_nodes(view_mode=
|
|
187
|
+
node_ids = adapter.select_nodes(view_mode="changed_models")
|
|
187
188
|
assert len(node_ids) == 4
|
|
188
189
|
|
|
189
|
-
node_ids = adapter.select_nodes(view_mode=
|
|
190
|
+
node_ids = adapter.select_nodes(view_mode="changed_models", packages=["other_package"])
|
|
190
191
|
assert len(node_ids) == 1
|
|
191
192
|
|
|
192
|
-
node_ids = adapter.select_nodes(view_mode=
|
|
193
|
+
node_ids = adapter.select_nodes(view_mode="changed_models", packages=["other_package"], exclude="customers_4")
|
|
193
194
|
assert len(node_ids) == 0
|
|
194
|
-
node_ids = adapter.select_nodes(view_mode=
|
|
195
|
+
node_ids = adapter.select_nodes(view_mode="changed_models", packages=["other_package"], select="customers_1+")
|
|
195
196
|
assert len(node_ids) == 1
|
|
196
|
-
node_ids = adapter.select_nodes(view_mode=
|
|
197
|
+
node_ids = adapter.select_nodes(view_mode="changed_models", select="+customers_5")
|
|
197
198
|
assert len(node_ids) == 3
|
|
198
|
-
node_ids = adapter.select_nodes(view_mode=
|
|
199
|
+
node_ids = adapter.select_nodes(view_mode="all", select="+customers_5")
|
|
199
200
|
assert len(node_ids) == 4
|
|
200
|
-
node_ids = adapter.select_nodes(select=
|
|
201
|
+
node_ids = adapter.select_nodes(select="+customers_5")
|
|
201
202
|
assert len(node_ids) == 4
|
tests/tasks/test_histogram.py
CHANGED
|
@@ -1,6 +1,10 @@
|
|
|
1
1
|
import pytest
|
|
2
2
|
|
|
3
|
-
from recce.tasks.histogram import
|
|
3
|
+
from recce.tasks.histogram import (
|
|
4
|
+
HistogramDiffCheckValidator,
|
|
5
|
+
HistogramDiffTask,
|
|
6
|
+
_is_histogram_supported,
|
|
7
|
+
)
|
|
4
8
|
|
|
5
9
|
|
|
6
10
|
def test_histogram(dbt_test_helper):
|
|
@@ -14,11 +18,7 @@ def test_histogram(dbt_test_helper):
|
|
|
14
18
|
|
|
15
19
|
dbt_test_helper.create_model("customers", csv_data, csv_data)
|
|
16
20
|
|
|
17
|
-
params = {
|
|
18
|
-
"model": "customers",
|
|
19
|
-
"column_name": "age",
|
|
20
|
-
"column_type": "int"
|
|
21
|
-
}
|
|
21
|
+
params = {"model": "customers", "column_name": "age", "column_type": "int"}
|
|
22
22
|
|
|
23
23
|
task = HistogramDiffTask(params)
|
|
24
24
|
run_result = task.execute()
|
|
@@ -30,13 +30,13 @@ def test_histogram(dbt_test_helper):
|
|
|
30
30
|
# 'bin_edges': [25, 26, ..., 51],
|
|
31
31
|
# 'labels': ['25-26', ..., '51-52']
|
|
32
32
|
# }
|
|
33
|
-
assert run_result[
|
|
34
|
-
assert run_result[
|
|
35
|
-
assert run_result[
|
|
36
|
-
assert run_result[
|
|
37
|
-
assert run_result[
|
|
38
|
-
assert run_result[
|
|
39
|
-
assert run_result[
|
|
33
|
+
assert run_result["current"]["counts"][0] == 1
|
|
34
|
+
assert run_result["current"]["counts"][-1] == 1
|
|
35
|
+
assert run_result["current"]["total"] == 4
|
|
36
|
+
assert run_result["min"] == 25
|
|
37
|
+
assert run_result["max"] == 50
|
|
38
|
+
assert run_result["bin_edges"][0] == 25
|
|
39
|
+
assert run_result["bin_edges"][-1] == 51
|
|
40
40
|
|
|
41
41
|
|
|
42
42
|
def test_histogram_emtpy(dbt_test_helper):
|
|
@@ -56,74 +56,66 @@ def test_histogram_emtpy(dbt_test_helper):
|
|
|
56
56
|
dbt_test_helper.create_model("customers2", csv_data, csv_data_zero)
|
|
57
57
|
dbt_test_helper.create_model("customers3", csv_data_zero, csv_data)
|
|
58
58
|
|
|
59
|
-
params = {
|
|
60
|
-
"model": "customers",
|
|
61
|
-
"column_name": "age",
|
|
62
|
-
"column_type": "int"
|
|
63
|
-
}
|
|
59
|
+
params = {"model": "customers", "column_name": "age", "column_type": "int"}
|
|
64
60
|
|
|
65
61
|
task = HistogramDiffTask(params)
|
|
66
62
|
run_result = task.execute()
|
|
67
63
|
|
|
68
|
-
assert len(run_result[
|
|
69
|
-
assert run_result[
|
|
70
|
-
assert run_result[
|
|
71
|
-
assert run_result[
|
|
72
|
-
assert len(run_result[
|
|
64
|
+
assert len(run_result["current"]["counts"]) == 0
|
|
65
|
+
assert run_result["current"]["total"] == 0
|
|
66
|
+
assert run_result["min"] is None
|
|
67
|
+
assert run_result["max"] is None
|
|
68
|
+
assert len(run_result["bin_edges"]) == 0
|
|
73
69
|
|
|
74
|
-
params = {
|
|
75
|
-
"model": "customers2",
|
|
76
|
-
"column_name": "age",
|
|
77
|
-
"column_type": "int"
|
|
78
|
-
}
|
|
70
|
+
params = {"model": "customers2", "column_name": "age", "column_type": "int"}
|
|
79
71
|
|
|
80
72
|
task = HistogramDiffTask(params)
|
|
81
73
|
run_result = task.execute()
|
|
82
|
-
assert run_result[
|
|
83
|
-
assert run_result[
|
|
84
|
-
assert run_result[
|
|
85
|
-
assert run_result[
|
|
86
|
-
assert run_result[
|
|
87
|
-
assert run_result[
|
|
88
|
-
assert run_result[
|
|
89
|
-
assert run_result[
|
|
90
|
-
assert run_result[
|
|
91
|
-
assert run_result[
|
|
92
|
-
|
|
93
|
-
params = {
|
|
94
|
-
"model": "customers3",
|
|
95
|
-
"column_name": "age",
|
|
96
|
-
"column_type": "int"
|
|
97
|
-
}
|
|
74
|
+
assert run_result["base"]["counts"][0] == 1
|
|
75
|
+
assert run_result["base"]["counts"][-1] == 1
|
|
76
|
+
assert run_result["base"]["total"] == 4
|
|
77
|
+
assert run_result["current"]["counts"][0] == 0
|
|
78
|
+
assert run_result["current"]["counts"][-1] == 0
|
|
79
|
+
assert run_result["current"]["total"] == 0
|
|
80
|
+
assert run_result["min"] == 25
|
|
81
|
+
assert run_result["max"] == 50
|
|
82
|
+
assert run_result["bin_edges"][0] == 25
|
|
83
|
+
assert run_result["bin_edges"][-1] == 51
|
|
84
|
+
|
|
85
|
+
params = {"model": "customers3", "column_name": "age", "column_type": "int"}
|
|
98
86
|
|
|
99
87
|
task = HistogramDiffTask(params)
|
|
100
88
|
run_result = task.execute()
|
|
101
|
-
assert run_result[
|
|
102
|
-
assert run_result[
|
|
103
|
-
assert run_result[
|
|
104
|
-
assert run_result[
|
|
105
|
-
assert run_result[
|
|
106
|
-
assert run_result[
|
|
107
|
-
assert run_result[
|
|
108
|
-
assert run_result[
|
|
109
|
-
assert run_result[
|
|
110
|
-
assert run_result[
|
|
89
|
+
assert run_result["base"]["counts"][0] == 0
|
|
90
|
+
assert run_result["base"]["counts"][-1] == 0
|
|
91
|
+
assert run_result["base"]["total"] == 0
|
|
92
|
+
assert run_result["current"]["counts"][0] == 1
|
|
93
|
+
assert run_result["current"]["counts"][-1] == 1
|
|
94
|
+
assert run_result["current"]["total"] == 4
|
|
95
|
+
assert run_result["min"] == 25
|
|
96
|
+
assert run_result["max"] == 50
|
|
97
|
+
assert run_result["bin_edges"][0] == 25
|
|
98
|
+
assert run_result["bin_edges"][-1] == 51
|
|
111
99
|
|
|
112
100
|
|
|
113
101
|
def test_validator():
|
|
114
102
|
def validate(params: dict = {}, view_options: dict = {}):
|
|
115
|
-
HistogramDiffCheckValidator().validate(
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
103
|
+
HistogramDiffCheckValidator().validate(
|
|
104
|
+
{
|
|
105
|
+
"name": "test",
|
|
106
|
+
"type": "histogram_diff",
|
|
107
|
+
"params": params,
|
|
108
|
+
"view_options": view_options,
|
|
109
|
+
}
|
|
110
|
+
)
|
|
111
|
+
|
|
112
|
+
validate(
|
|
113
|
+
{
|
|
114
|
+
"model": "customers",
|
|
115
|
+
"column_name": "age",
|
|
116
|
+
"column_type": "int",
|
|
117
|
+
}
|
|
118
|
+
)
|
|
127
119
|
|
|
128
120
|
with pytest.raises(ValueError):
|
|
129
121
|
validate({})
|
tests/tasks/test_lineage.py
CHANGED
|
@@ -3,40 +3,53 @@ import pytest
|
|
|
3
3
|
|
|
4
4
|
def test_validator():
|
|
5
5
|
from recce.tasks.lineage import LineageDiffCheckValidator
|
|
6
|
+
|
|
6
7
|
validator = LineageDiffCheckValidator()
|
|
7
8
|
|
|
8
9
|
def validate(params: dict):
|
|
9
|
-
validator.validate(
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
10
|
+
validator.validate(
|
|
11
|
+
{
|
|
12
|
+
"name": "test",
|
|
13
|
+
"type": "schema_diff",
|
|
14
|
+
"params": params,
|
|
15
|
+
}
|
|
16
|
+
)
|
|
14
17
|
|
|
15
18
|
# Select all models
|
|
16
19
|
validate({})
|
|
17
20
|
|
|
18
21
|
# Select by selector
|
|
19
|
-
validate(
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
22
|
+
validate(
|
|
23
|
+
{
|
|
24
|
+
"select": "customers",
|
|
25
|
+
"exclude": "customers",
|
|
26
|
+
"packages": ["jaffle_shop"],
|
|
27
|
+
"view_mode": "all",
|
|
28
|
+
}
|
|
29
|
+
)
|
|
25
30
|
|
|
26
31
|
# packages should be an array
|
|
27
32
|
with pytest.raises(ValueError):
|
|
28
|
-
validate(
|
|
29
|
-
|
|
30
|
-
|
|
33
|
+
validate(
|
|
34
|
+
{
|
|
35
|
+
"packages": "jaffle_shop",
|
|
36
|
+
}
|
|
37
|
+
)
|
|
31
38
|
|
|
32
39
|
# view_mode should be 'all' or 'changed_models'
|
|
33
|
-
validate(
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
40
|
+
validate(
|
|
41
|
+
{
|
|
42
|
+
"view_mode": None,
|
|
43
|
+
}
|
|
44
|
+
)
|
|
45
|
+
validate(
|
|
46
|
+
{
|
|
47
|
+
"view_mode": "all",
|
|
48
|
+
}
|
|
49
|
+
)
|
|
39
50
|
with pytest.raises(ValueError):
|
|
40
|
-
validate(
|
|
41
|
-
|
|
42
|
-
|
|
51
|
+
validate(
|
|
52
|
+
{
|
|
53
|
+
"view_mode": "abc",
|
|
54
|
+
}
|
|
55
|
+
)
|