dbt-addons 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dbt_addons-0.1.0/PKG-INFO +11 -0
- dbt_addons-0.1.0/addons/__init__.py +0 -0
- dbt_addons-0.1.0/addons/cli/__init__.py +0 -0
- dbt_addons-0.1.0/addons/cli/cli.py +104 -0
- dbt_addons-0.1.0/addons/cli/logger.py +19 -0
- dbt_addons-0.1.0/addons/install.py +102 -0
- dbt_addons-0.1.0/addons/wap/__init__.py +0 -0
- dbt_addons-0.1.0/addons/wap/macros/providers/utils.sql +86 -0
- dbt_addons-0.1.0/addons/wap/macros/providers/wap_bigquery.sql +15 -0
- dbt_addons-0.1.0/addons/wap/macros/providers/wap_databricks.sql +15 -0
- dbt_addons-0.1.0/addons/wap/macros/providers/wap_duckdb.sql +17 -0
- dbt_addons-0.1.0/addons/wap/macros/providers/wap_rename.sql +68 -0
- dbt_addons-0.1.0/addons/wap/macros/providers/wap_snowflake.sql +14 -0
- dbt_addons-0.1.0/addons/wap/macros/wap_deploy.sql +41 -0
- dbt_addons-0.1.0/addons/wap/root_macros/generate_alias_name.sql +8 -0
- dbt_addons-0.1.0/addons/wap/root_macros/generate_schema_name.sql +7 -0
- dbt_addons-0.1.0/addons/wap/wap.py +104 -0
- dbt_addons-0.1.0/dbt_addons.egg-info/PKG-INFO +11 -0
- dbt_addons-0.1.0/dbt_addons.egg-info/SOURCES.txt +23 -0
- dbt_addons-0.1.0/dbt_addons.egg-info/dependency_links.txt +1 -0
- dbt_addons-0.1.0/dbt_addons.egg-info/entry_points.txt +2 -0
- dbt_addons-0.1.0/dbt_addons.egg-info/requires.txt +5 -0
- dbt_addons-0.1.0/dbt_addons.egg-info/top_level.txt +1 -0
- dbt_addons-0.1.0/setup.cfg +4 -0
- dbt_addons-0.1.0/setup.py +29 -0
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: dbt-addons
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Requires-Python: >=3.8
|
|
5
|
+
Requires-Dist: dbt-core>=1.5.0
|
|
6
|
+
Requires-Dist: pyyaml>=6.0
|
|
7
|
+
Provides-Extra: dev
|
|
8
|
+
Requires-Dist: dbt-duckdb>=1.5.0; extra == "dev"
|
|
9
|
+
Dynamic: provides-extra
|
|
10
|
+
Dynamic: requires-dist
|
|
11
|
+
Dynamic: requires-python
|
|
File without changes
|
|
File without changes
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
import sys
|
|
2
|
+
import subprocess
|
|
3
|
+
import json
|
|
4
|
+
import shutil
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from .logger import log
|
|
7
|
+
from ..wap.wap import get_executed_tables, read_wap_config
|
|
8
|
+
from ..install import install_from_project, install_addon
|
|
9
|
+
|
|
10
|
+
def get_real_dbt_path():
|
|
11
|
+
dbt_path = shutil.which('dbt')
|
|
12
|
+
|
|
13
|
+
if not dbt_path:
|
|
14
|
+
log.error("dbt not found in PATH")
|
|
15
|
+
sys.exit(1)
|
|
16
|
+
|
|
17
|
+
real_path = Path(dbt_path).resolve()
|
|
18
|
+
|
|
19
|
+
if real_path == Path(__file__).resolve():
|
|
20
|
+
import os
|
|
21
|
+
path_dirs = os.environ['PATH'].split(os.pathsep)
|
|
22
|
+
current_dir = str(Path(__file__).parent.resolve())
|
|
23
|
+
filtered_path = os.pathsep.join([d for d in path_dirs if d != current_dir])
|
|
24
|
+
|
|
25
|
+
old_path = os.environ['PATH']
|
|
26
|
+
os.environ['PATH'] = filtered_path
|
|
27
|
+
dbt_path = shutil.which('dbt')
|
|
28
|
+
os.environ['PATH'] = old_path
|
|
29
|
+
|
|
30
|
+
if not dbt_path:
|
|
31
|
+
log.error("Real dbt executable not found")
|
|
32
|
+
sys.exit(1)
|
|
33
|
+
|
|
34
|
+
return dbt_path
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _merge_wap_vars(args: list, extra: dict) -> list:
|
|
38
|
+
"""Inject extra vars into dbt args, merging with any existing --vars."""
|
|
39
|
+
import yaml
|
|
40
|
+
args = list(args)
|
|
41
|
+
if '--vars' in args:
|
|
42
|
+
idx = args.index('--vars')
|
|
43
|
+
try:
|
|
44
|
+
existing = yaml.safe_load(args[idx + 1]) or {}
|
|
45
|
+
except Exception:
|
|
46
|
+
existing = {}
|
|
47
|
+
existing.update(extra)
|
|
48
|
+
args[idx + 1] = json.dumps(existing)
|
|
49
|
+
else:
|
|
50
|
+
args += ['--vars', json.dumps(extra)]
|
|
51
|
+
return args
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def main():
|
|
55
|
+
real_dbt = get_real_dbt_path()
|
|
56
|
+
args = sys.argv[1:]
|
|
57
|
+
|
|
58
|
+
if args[:1] == ['install']:
|
|
59
|
+
return install_from_project()
|
|
60
|
+
|
|
61
|
+
if args[:2] == ['wap', 'install']:
|
|
62
|
+
return install_addon('wap')
|
|
63
|
+
|
|
64
|
+
if '--wap' not in args:
|
|
65
|
+
return subprocess.run([real_dbt] + args).returncode
|
|
66
|
+
|
|
67
|
+
args.remove('--wap')
|
|
68
|
+
dbt_command = args[0] if args else None
|
|
69
|
+
|
|
70
|
+
if dbt_command not in ['run', 'build']:
|
|
71
|
+
log.error("--wap only works with 'run' or 'build'")
|
|
72
|
+
return 1
|
|
73
|
+
|
|
74
|
+
dbt_args = list(args[1:])
|
|
75
|
+
wap_config = read_wap_config()
|
|
76
|
+
suffix = wap_config.get('wap_staging_suffix')
|
|
77
|
+
if suffix:
|
|
78
|
+
dbt_args = _merge_wap_vars(dbt_args, {'dbt_wap_staging_suffix': suffix})
|
|
79
|
+
|
|
80
|
+
log.info(f"Running dbt {' '.join([dbt_command] + dbt_args)}...")
|
|
81
|
+
result = subprocess.run([real_dbt, dbt_command] + dbt_args)
|
|
82
|
+
log.info("")
|
|
83
|
+
|
|
84
|
+
tables_to_copy, skipped = get_executed_tables()
|
|
85
|
+
|
|
86
|
+
if not tables_to_copy:
|
|
87
|
+
log.warning("No tables to copy")
|
|
88
|
+
return 0
|
|
89
|
+
|
|
90
|
+
log.info(f"[dbt-addon WAP] Publishing {len(tables_to_copy)} tables to prod...")
|
|
91
|
+
tables_json = json.dumps(tables_to_copy)
|
|
92
|
+
skipped_json = json.dumps(skipped)
|
|
93
|
+
|
|
94
|
+
deploy_result = subprocess.run([
|
|
95
|
+
real_dbt, 'run-operation', 'wap_deploy',
|
|
96
|
+
'--args', f'{{tables_to_copy: {tables_json}, skipped_tables: {skipped_json}}}'
|
|
97
|
+
])
|
|
98
|
+
log.info("")
|
|
99
|
+
|
|
100
|
+
return deploy_result.returncode
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
if __name__ == '__main__':
|
|
104
|
+
sys.exit(main())
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
|
|
3
|
+
RESET = "\033[0m"
|
|
4
|
+
YELLOW = "\033[33m"
|
|
5
|
+
CYAN = "\033[36m"
|
|
6
|
+
|
|
7
|
+
class _ColorFormatter(logging.Formatter):
|
|
8
|
+
def format(self, record):
|
|
9
|
+
prefix = f"{YELLOW}[WARN]{RESET} " if record.levelno == logging.WARNING else ""
|
|
10
|
+
record.msg = f"{prefix}{CYAN}{record.msg}{RESET}"
|
|
11
|
+
return super().format(record)
|
|
12
|
+
|
|
13
|
+
_handler = logging.StreamHandler()
|
|
14
|
+
_handler.setFormatter(_ColorFormatter(fmt="%(asctime)s %(message)s", datefmt="%H:%M:%S"))
|
|
15
|
+
|
|
16
|
+
log = logging.getLogger("dbta")
|
|
17
|
+
log.setLevel(logging.INFO)
|
|
18
|
+
log.addHandler(_handler)
|
|
19
|
+
log.propagate = False
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
import shutil
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
from typing import List
|
|
4
|
+
|
|
5
|
+
import yaml
|
|
6
|
+
|
|
7
|
+
from .cli.logger import log
|
|
8
|
+
|
|
9
|
+
ADDONS_ROOT = Path(__file__).parent
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def _install_addon(name: str) -> bool:
|
|
13
|
+
src = ADDONS_ROOT / name / 'macros'
|
|
14
|
+
if not src.exists():
|
|
15
|
+
log.warning(f"No macros found for addon '{name}' — skipping")
|
|
16
|
+
return False
|
|
17
|
+
|
|
18
|
+
dest = Path('macros/dbt_addons') / name
|
|
19
|
+
if dest.exists():
|
|
20
|
+
shutil.rmtree(dest)
|
|
21
|
+
shutil.copytree(src, dest)
|
|
22
|
+
log.info(f"Installed addon '{name}' → {dest}/")
|
|
23
|
+
return True
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _active_adapter() -> str:
|
|
27
|
+
"""Return the adapter type from profiles.yml for the active profile, or '' if unknown."""
|
|
28
|
+
project_file = Path('dbt_project.yml')
|
|
29
|
+
profiles_file = Path('profiles.yml')
|
|
30
|
+
if not project_file.exists() or not profiles_file.exists():
|
|
31
|
+
return ''
|
|
32
|
+
with open(project_file) as f:
|
|
33
|
+
profile_name = yaml.safe_load(f).get('profile', '')
|
|
34
|
+
with open(profiles_file) as f:
|
|
35
|
+
profiles = yaml.safe_load(f)
|
|
36
|
+
profile = profiles.get(profile_name, {})
|
|
37
|
+
target = profile.get('target', '')
|
|
38
|
+
return profile.get('outputs', {}).get(target, {}).get('type', '')
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _install_wap_root_macro(cfg: dict) -> None:
|
|
42
|
+
"""Install the root macros required for the configured WAP strategy."""
|
|
43
|
+
rename_mode = bool(cfg.get('wap_staging_suffix'))
|
|
44
|
+
Path('macros').mkdir(exist_ok=True)
|
|
45
|
+
|
|
46
|
+
# generate_schema_name is only needed for DuckDB — other adapters manage it themselves
|
|
47
|
+
if _active_adapter() == 'duckdb':
|
|
48
|
+
src = ADDONS_ROOT / 'wap' / 'root_macros' / 'generate_schema_name.sql'
|
|
49
|
+
shutil.copy(src, Path('macros') / 'generate_schema_name.sql')
|
|
50
|
+
log.info(" Added macros/generate_schema_name.sql (duckdb)")
|
|
51
|
+
|
|
52
|
+
if rename_mode:
|
|
53
|
+
src = ADDONS_ROOT / 'wap' / 'root_macros' / 'generate_alias_name.sql'
|
|
54
|
+
shutil.copy(src, Path('macros') / 'generate_alias_name.sql')
|
|
55
|
+
log.info(" Added macros/generate_alias_name.sql (rename mode)")
|
|
56
|
+
else:
|
|
57
|
+
dest = Path('macros') / 'generate_alias_name.sql'
|
|
58
|
+
if dest.exists():
|
|
59
|
+
dest.unlink()
|
|
60
|
+
log.info(" Removed macros/generate_alias_name.sql")
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _read_project_cfg() -> dict:
|
|
64
|
+
project_file = Path('dbt_project.yml')
|
|
65
|
+
if not project_file.exists():
|
|
66
|
+
return {}
|
|
67
|
+
with open(project_file) as f:
|
|
68
|
+
project = yaml.safe_load(f)
|
|
69
|
+
return project.get('vars', {}).get('dbt-addons', {})
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def install_from_project() -> int:
|
|
73
|
+
project_file = Path('dbt_project.yml')
|
|
74
|
+
if not project_file.exists():
|
|
75
|
+
log.error("dbt_project.yml not found — run this from your dbt project root")
|
|
76
|
+
return 1
|
|
77
|
+
|
|
78
|
+
with open(project_file) as f:
|
|
79
|
+
project = yaml.safe_load(f)
|
|
80
|
+
|
|
81
|
+
cfg = project.get('vars', {}).get('dbt-addons', {})
|
|
82
|
+
addons: List[str] = cfg.get('addons', [])
|
|
83
|
+
|
|
84
|
+
if not addons:
|
|
85
|
+
log.warning("No addons configured under vars.dbt-addons.addons in dbt_project.yml")
|
|
86
|
+
return 0
|
|
87
|
+
|
|
88
|
+
log.info(f"Installing {len(addons)} addon(s): {', '.join(addons)}")
|
|
89
|
+
for name in addons:
|
|
90
|
+
_install_addon(name)
|
|
91
|
+
if name == 'wap':
|
|
92
|
+
_install_wap_root_macro(cfg)
|
|
93
|
+
|
|
94
|
+
return 0
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def install_addon(name: str) -> int:
|
|
98
|
+
if not _install_addon(name):
|
|
99
|
+
return 1
|
|
100
|
+
if name == 'wap':
|
|
101
|
+
_install_wap_root_macro(_read_project_cfg())
|
|
102
|
+
return 0
|
|
File without changes
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
{% macro deploy_wap(tables_to_copy, skipped_tables, queries) %}
|
|
2
|
+
{% if execute %}
|
|
3
|
+
{% set total = (tables_to_copy | length) + (skipped_tables | length) %}
|
|
4
|
+
{% set width = 80 %}
|
|
5
|
+
|
|
6
|
+
{% for item in tables_to_copy %}
|
|
7
|
+
{% set start_label = loop.index ~ " of " ~ total ~ " START copying " ~ item.name %}
|
|
8
|
+
{% set start_dots = "." * ([width - start_label | length, 1] | max) %}
|
|
9
|
+
{% do log(start_label ~ " " ~ start_dots ~ " [RUN]", info=true) %}
|
|
10
|
+
|
|
11
|
+
{% do run_query(queries[loop.index - 1]) %}
|
|
12
|
+
|
|
13
|
+
{% set ok_label = loop.index ~ " of " ~ total ~ " OK " ~ item.name %}
|
|
14
|
+
{% set ok_dots = "." * ([width - ok_label | length, 1] | max) %}
|
|
15
|
+
{% do log(ok_label ~ " " ~ ok_dots ~ " [\x1b[32mOK\x1b[0m]", info=true) %}
|
|
16
|
+
{% endfor %}
|
|
17
|
+
|
|
18
|
+
{% for name in skipped_tables %}
|
|
19
|
+
{% set idx = (tables_to_copy | length) + loop.index %}
|
|
20
|
+
{% set fail_label = idx ~ " of " ~ total ~ " FAIL " ~ name %}
|
|
21
|
+
{% set fail_dots = "." * ([width - fail_label | length, 1] | max) %}
|
|
22
|
+
{% do log(fail_label ~ " " ~ fail_dots ~ " [\x1b[31mFAIL\x1b[0m]", info=true) %}
|
|
23
|
+
{% endfor %}
|
|
24
|
+
|
|
25
|
+
{% set n_copied = tables_to_copy | length %}
|
|
26
|
+
{% set n_failed = skipped_tables | length %}
|
|
27
|
+
{% set summary = "PASS=" ~ n_copied ~ " FAIL=" ~ n_failed ~ " TOTAL=" ~ total %}
|
|
28
|
+
|
|
29
|
+
{% do log("", info=true) %}
|
|
30
|
+
{% if n_copied == 0 %}
|
|
31
|
+
{% do log("[\x1b[31mNO TABLES\x1b[0m] " ~ summary, info=true) %}
|
|
32
|
+
{% elif n_copied < total %}
|
|
33
|
+
{% do log("[\x1b[33mPARTIAL\x1b[0m] " ~ summary, info=true) %}
|
|
34
|
+
{% else %}
|
|
35
|
+
{% do log("[\x1b[32mOK\x1b[0m] " ~ summary, info=true) %}
|
|
36
|
+
{% endif %}
|
|
37
|
+
{% endif %}
|
|
38
|
+
{% endmacro %}
|
|
39
|
+
|
|
40
|
+
{% macro all_required_variables_are_setup() %}
|
|
41
|
+
{% if execute %}
|
|
42
|
+
{% set adapter_type = adapter.type() %}
|
|
43
|
+
{% set cfg = var('dbt-addons', {}) %}
|
|
44
|
+
{% set required_vars = {} %}
|
|
45
|
+
|
|
46
|
+
{% set common_vars = {
|
|
47
|
+
'dbt_staging_schema': cfg.get('dbt_staging_schema'),
|
|
48
|
+
'dbt_prod_schema': cfg.get('dbt_prod_schema')
|
|
49
|
+
} %}
|
|
50
|
+
|
|
51
|
+
{% if adapter_type == 'snowflake' %}
|
|
52
|
+
{% set required_vars = common_vars %}
|
|
53
|
+
|
|
54
|
+
{% elif adapter_type == 'bigquery' %}
|
|
55
|
+
{% set required_vars = common_vars | combine({
|
|
56
|
+
'project_id': cfg.get('project_id'),
|
|
57
|
+
'prod_dataset': cfg.get('prod_dataset'),
|
|
58
|
+
'staging_dataset': cfg.get('staging_dataset')
|
|
59
|
+
}) %}
|
|
60
|
+
|
|
61
|
+
{% elif adapter_type == 'duckdb' %}
|
|
62
|
+
{% set required_vars = common_vars %}
|
|
63
|
+
|
|
64
|
+
{% else %}
|
|
65
|
+
{% do log("Unknown adapter: " ~ adapter_type, info=true) %}
|
|
66
|
+
{% set required_vars = common_vars %}
|
|
67
|
+
{% endif %}
|
|
68
|
+
|
|
69
|
+
{% set missing_vars = [] %}
|
|
70
|
+
{% for var_name, var_value in required_vars.items() %}
|
|
71
|
+
{% if var_value is none %}
|
|
72
|
+
{% do missing_vars.append(var_name) %}
|
|
73
|
+
{% endif %}
|
|
74
|
+
{% endfor %}
|
|
75
|
+
|
|
76
|
+
{% if missing_vars | length > 0 %}
|
|
77
|
+
{% do log("[" ~ adapter_type ~ "] Missing required variables: " ~ missing_vars | join(', '), info=true) %}
|
|
78
|
+
{% do exceptions.raise_compiler_error("Setup incomplete for " ~ adapter_type ~ ". Define: " ~ missing_vars | join(', ')) %}
|
|
79
|
+
{% else %}
|
|
80
|
+
{% do log("[" ~ adapter_type ~ "] All required variables configured", info=true) %}
|
|
81
|
+
{% for var_name, var_value in required_vars.items() %}
|
|
82
|
+
{% do log(" • " ~ var_name ~ ": " ~ var_value, info=true) %}
|
|
83
|
+
{% endfor %}
|
|
84
|
+
{% endif %}
|
|
85
|
+
{% endif %}
|
|
86
|
+
{% endmacro %}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
{% macro wap_deploy_bigquery(tables_to_copy, skipped_tables=[]) %}
|
|
2
|
+
{% set project_id = var('dbt-addons')['project_id'] %}
|
|
3
|
+
{% set prod_dataset = var('dbt-addons')['prod_dataset'] %}
|
|
4
|
+
|
|
5
|
+
{% set queries = [] %}
|
|
6
|
+
{% for item in tables_to_copy %}
|
|
7
|
+
{% set q %}
|
|
8
|
+
CREATE OR REPLACE TABLE `{{ project_id }}.{{ prod_dataset }}.{{ item.name }}` AS
|
|
9
|
+
SELECT * FROM {{ item.relation }}
|
|
10
|
+
{% endset %}
|
|
11
|
+
{% do queries.append(q) %}
|
|
12
|
+
{% endfor %}
|
|
13
|
+
|
|
14
|
+
{{ deploy_wap(tables_to_copy, skipped_tables, queries) }}
|
|
15
|
+
{% endmacro %}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
{% macro wap_deploy_databricks(tables_to_copy, skipped_tables=[]) %}
|
|
2
|
+
{% set catalog = var('dbt-addons')['catalog'] %}
|
|
3
|
+
{% set prod_schema = var('dbt-addons')['dbt_prod_schema'] %}
|
|
4
|
+
|
|
5
|
+
{% set queries = [] %}
|
|
6
|
+
{% for item in tables_to_copy %}
|
|
7
|
+
{% set q %}
|
|
8
|
+
CREATE OR REPLACE TABLE `{{ catalog }}`.`{{ prod_schema }}`.`{{ item.name }}`
|
|
9
|
+
SHALLOW CLONE {{ item.relation }}
|
|
10
|
+
{% endset %}
|
|
11
|
+
{% do queries.append(q) %}
|
|
12
|
+
{% endfor %}
|
|
13
|
+
|
|
14
|
+
{{ deploy_wap(tables_to_copy, skipped_tables, queries) }}
|
|
15
|
+
{% endmacro %}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
{% macro wap_deploy_duckdb(tables_to_copy, skipped_tables=[]) %}
|
|
2
|
+
{% set prod_schema = var('dbt-addons')['dbt_prod_schema'] %}
|
|
3
|
+
|
|
4
|
+
{% do run_query("CREATE SCHEMA IF NOT EXISTS " ~ prod_schema) %}
|
|
5
|
+
{% do log("", info=true) %}
|
|
6
|
+
|
|
7
|
+
{% set queries = [] %}
|
|
8
|
+
{% for item in tables_to_copy %}
|
|
9
|
+
{% set q %}
|
|
10
|
+
CREATE OR REPLACE TABLE {{ prod_schema }}.{{ item.name }} AS
|
|
11
|
+
SELECT * FROM {{ item.relation }}
|
|
12
|
+
{% endset %}
|
|
13
|
+
{% do queries.append(q) %}
|
|
14
|
+
{% endfor %}
|
|
15
|
+
|
|
16
|
+
{{ deploy_wap(tables_to_copy, skipped_tables, queries) }}
|
|
17
|
+
{% endmacro %}
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
{% macro wap_rename_snowflake(tables_to_copy, skipped_tables, suffix) %}
|
|
2
|
+
{% set queries = [] %}
|
|
3
|
+
{% for item in tables_to_copy %}
|
|
4
|
+
{% set staging_name = item.name ~ suffix %}
|
|
5
|
+
{% set target = item.relation
|
|
6
|
+
| replace(staging_name | upper, item.name | upper)
|
|
7
|
+
| replace(staging_name, item.name) %}
|
|
8
|
+
{% set q %}
|
|
9
|
+
CREATE OR REPLACE TABLE {{ target }} CLONE {{ item.relation }}
|
|
10
|
+
{% endset %}
|
|
11
|
+
{% do queries.append(q) %}
|
|
12
|
+
{% endfor %}
|
|
13
|
+
|
|
14
|
+
{{ deploy_wap(tables_to_copy, skipped_tables, queries) }}
|
|
15
|
+
{% endmacro %}
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
{% macro wap_rename_databricks(tables_to_copy, skipped_tables, suffix) %}
|
|
19
|
+
{% set queries = [] %}
|
|
20
|
+
{% for item in tables_to_copy %}
|
|
21
|
+
{% set staging_name = item.name ~ suffix %}
|
|
22
|
+
{% set target = item.relation
|
|
23
|
+
| replace(staging_name | upper, item.name | upper)
|
|
24
|
+
| replace(staging_name, item.name) %}
|
|
25
|
+
{% set q %}
|
|
26
|
+
CREATE OR REPLACE TABLE {{ target }} SHALLOW CLONE {{ item.relation }}
|
|
27
|
+
{% endset %}
|
|
28
|
+
{% do queries.append(q) %}
|
|
29
|
+
{% endfor %}
|
|
30
|
+
|
|
31
|
+
{{ deploy_wap(tables_to_copy, skipped_tables, queries) }}
|
|
32
|
+
{% endmacro %}
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
{% macro wap_rename_bigquery(tables_to_copy, skipped_tables, suffix) %}
|
|
36
|
+
{% set queries = [] %}
|
|
37
|
+
{% for item in tables_to_copy %}
|
|
38
|
+
{% set staging_name = item.name ~ suffix %}
|
|
39
|
+
{% set target = item.relation
|
|
40
|
+
| replace(staging_name | upper, item.name | upper)
|
|
41
|
+
| replace(staging_name, item.name) %}
|
|
42
|
+
{% set q %}
|
|
43
|
+
CREATE OR REPLACE TABLE {{ target }} AS
|
|
44
|
+
SELECT * FROM {{ item.relation }}
|
|
45
|
+
{% endset %}
|
|
46
|
+
{% do queries.append(q) %}
|
|
47
|
+
{% endfor %}
|
|
48
|
+
|
|
49
|
+
{{ deploy_wap(tables_to_copy, skipped_tables, queries) }}
|
|
50
|
+
{% endmacro %}
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
{% macro wap_rename_duckdb(tables_to_copy, skipped_tables, suffix) %}
|
|
54
|
+
{% set queries = [] %}
|
|
55
|
+
{% for item in tables_to_copy %}
|
|
56
|
+
{% set staging_name = item.name ~ suffix %}
|
|
57
|
+
{% set target = item.relation
|
|
58
|
+
| replace(staging_name | upper, item.name | upper)
|
|
59
|
+
| replace(staging_name, item.name) %}
|
|
60
|
+
{% set q %}
|
|
61
|
+
CREATE OR REPLACE TABLE {{ target }} AS
|
|
62
|
+
SELECT * FROM {{ item.relation }}
|
|
63
|
+
{% endset %}
|
|
64
|
+
{% do queries.append(q) %}
|
|
65
|
+
{% endfor %}
|
|
66
|
+
|
|
67
|
+
{{ deploy_wap(tables_to_copy, skipped_tables, queries) }}
|
|
68
|
+
{% endmacro %}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
{% macro wap_deploy_snowflake(tables_to_copy, skipped_tables=[]) %}
|
|
2
|
+
{% set prod_schema = var('dbt-addons')['dbt_prod_schema'] %}
|
|
3
|
+
|
|
4
|
+
{% set queries = [] %}
|
|
5
|
+
{% for item in tables_to_copy %}
|
|
6
|
+
{% set q %}
|
|
7
|
+
CREATE OR REPLACE TABLE {{ prod_schema }}.{{ item.name }}
|
|
8
|
+
CLONE {{ item.relation }}
|
|
9
|
+
{% endset %}
|
|
10
|
+
{% do queries.append(q) %}
|
|
11
|
+
{% endfor %}
|
|
12
|
+
|
|
13
|
+
{{ deploy_wap(tables_to_copy, skipped_tables, queries) }}
|
|
14
|
+
{% endmacro %}
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
{% macro wap_deploy(tables_to_copy, skipped_tables=[]) %}
|
|
2
|
+
{% set provider = adapter.type() %}
|
|
3
|
+
{% set cfg = var('dbt-addons', {}) %}
|
|
4
|
+
{% set wap_staging_suffix = cfg.get('wap_staging_suffix') %}
|
|
5
|
+
|
|
6
|
+
{% if wap_staging_suffix %}
|
|
7
|
+
{# Rename mode: models built with suffix, promoted within the same schema #}
|
|
8
|
+
{% if provider == 'snowflake' %}
|
|
9
|
+
{{ wap_rename_snowflake(tables_to_copy, skipped_tables, wap_staging_suffix) }}
|
|
10
|
+
{% elif provider == 'bigquery' %}
|
|
11
|
+
{{ wap_rename_bigquery(tables_to_copy, skipped_tables, wap_staging_suffix) }}
|
|
12
|
+
{% elif provider == 'databricks' %}
|
|
13
|
+
{{ wap_rename_databricks(tables_to_copy, skipped_tables, wap_staging_suffix) }}
|
|
14
|
+
{% elif provider == 'duckdb' %}
|
|
15
|
+
{{ wap_rename_duckdb(tables_to_copy, skipped_tables, wap_staging_suffix) }}
|
|
16
|
+
{% else %}
|
|
17
|
+
{% do exceptions.raise_compiler_error("WAP rename mode not supported for provider: " ~ provider) %}
|
|
18
|
+
{% endif %}
|
|
19
|
+
|
|
20
|
+
{% else %}
|
|
21
|
+
{# Cross-schema mode: copy from staging schema to prod schema #}
|
|
22
|
+
{% if not all_required_variables_are_setup() %}
|
|
23
|
+
{% do exceptions.raise_compiler_error("All the variables needed for WAP are not setup.") %}
|
|
24
|
+
{% endif %}
|
|
25
|
+
|
|
26
|
+
{% if provider == 'snowflake' %}
|
|
27
|
+
{{ wap_deploy_snowflake(tables_to_copy) }}
|
|
28
|
+
{% elif provider == 'bigquery' %}
|
|
29
|
+
{{ wap_deploy_bigquery(tables_to_copy) }}
|
|
30
|
+
{% elif provider == 'databricks' %}
|
|
31
|
+
{{ wap_deploy_databricks(tables_to_copy) }}
|
|
32
|
+
{% elif provider == 'duckdb' %}
|
|
33
|
+
{{ wap_deploy_duckdb(tables_to_copy, skipped_tables) }}
|
|
34
|
+
{% else %}
|
|
35
|
+
{% do log("WAP not supported for provider: " ~ provider, info=true) %}
|
|
36
|
+
{% do exceptions.raise_compiler_error("Unsupported provider for WAP") %}
|
|
37
|
+
{% endif %}
|
|
38
|
+
|
|
39
|
+
{% endif %}
|
|
40
|
+
|
|
41
|
+
{% endmacro %}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
{% macro generate_alias_name(custom_alias_name=none, node=none) -%}
|
|
2
|
+
{%- set suffix = var('dbt_wap_staging_suffix', '') if node.resource_type == 'model' else '' -%}
|
|
3
|
+
{%- if custom_alias_name is none -%}
|
|
4
|
+
{{ node.name }}{{ suffix }}
|
|
5
|
+
{%- else -%}
|
|
6
|
+
{{ custom_alias_name }}{{ suffix }}
|
|
7
|
+
{%- endif -%}
|
|
8
|
+
{%- endmacro %}
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
import json
|
|
2
|
+
import fnmatch
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
from typing import List, Optional
|
|
5
|
+
|
|
6
|
+
import yaml
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def read_wap_config() -> dict:
|
|
10
|
+
project_path = Path('dbt_project.yml')
|
|
11
|
+
if not project_path.exists():
|
|
12
|
+
return {}
|
|
13
|
+
with open(project_path) as f:
|
|
14
|
+
project = yaml.safe_load(f)
|
|
15
|
+
return project.get('vars', {}).get('dbt-addons', {})
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _matches_wap_paths(model_path: str, wap_paths: List[str]) -> bool:
|
|
19
|
+
"""Return True if model_path (e.g. 'marts/fct_customers.sql') matches any pattern."""
|
|
20
|
+
for pattern in wap_paths:
|
|
21
|
+
norm = pattern.rstrip('/')
|
|
22
|
+
# Folder prefix: 'marts' matches 'marts/fct_customers.sql'
|
|
23
|
+
if model_path.startswith(norm + '/') or model_path == norm:
|
|
24
|
+
return True
|
|
25
|
+
# Glob pattern: 'marts/*.sql' or 'core/fct_*'
|
|
26
|
+
if fnmatch.fnmatch(model_path, pattern):
|
|
27
|
+
return True
|
|
28
|
+
return False
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def get_executed_tables() -> List[str]:
|
|
32
|
+
"""Extract successfully built models whose tests all passed from run_results.json."""
|
|
33
|
+
run_results_path = Path('target/run_results.json')
|
|
34
|
+
manifest_path = Path('target/manifest.json')
|
|
35
|
+
|
|
36
|
+
if not run_results_path.exists():
|
|
37
|
+
return [], []
|
|
38
|
+
|
|
39
|
+
with open(run_results_path) as f:
|
|
40
|
+
run_results = json.load(f)
|
|
41
|
+
|
|
42
|
+
results = run_results.get('results', [])
|
|
43
|
+
|
|
44
|
+
manifest = {}
|
|
45
|
+
if manifest_path.exists():
|
|
46
|
+
with open(manifest_path) as f:
|
|
47
|
+
manifest = json.load(f)
|
|
48
|
+
|
|
49
|
+
wap_config = read_wap_config()
|
|
50
|
+
wap_paths: Optional[List[str]] = wap_config.get('wap_paths') # None = all models
|
|
51
|
+
|
|
52
|
+
# Find models blocked by a failing test via manifest dependency graph
|
|
53
|
+
failed_model_ids: set = set()
|
|
54
|
+
for result in results:
|
|
55
|
+
if result.get('status') not in ('fail', 'error'):
|
|
56
|
+
continue
|
|
57
|
+
uid = result.get('unique_id', '')
|
|
58
|
+
if not uid.startswith('test.'):
|
|
59
|
+
continue
|
|
60
|
+
test_node = manifest.get('nodes', {}).get(uid, {})
|
|
61
|
+
for dep in test_node.get('depends_on', {}).get('nodes', []):
|
|
62
|
+
if dep.startswith('model.'):
|
|
63
|
+
failed_model_ids.add(dep)
|
|
64
|
+
|
|
65
|
+
promoted = []
|
|
66
|
+
skipped = []
|
|
67
|
+
for result in results:
|
|
68
|
+
if result.get('status') != 'success':
|
|
69
|
+
continue
|
|
70
|
+
uid = result.get('unique_id', '')
|
|
71
|
+
if not uid.startswith('model.'):
|
|
72
|
+
continue
|
|
73
|
+
|
|
74
|
+
node = manifest.get('nodes', {}).get(uid, {})
|
|
75
|
+
|
|
76
|
+
if wap_paths is not None:
|
|
77
|
+
model_path = node.get('path', '')
|
|
78
|
+
if not _matches_wap_paths(model_path, wap_paths):
|
|
79
|
+
continue
|
|
80
|
+
|
|
81
|
+
table_name = uid.split('.')[-1]
|
|
82
|
+
if uid in failed_model_ids:
|
|
83
|
+
skipped.append(table_name)
|
|
84
|
+
continue
|
|
85
|
+
relation_name = result.get('relation_name')
|
|
86
|
+
if relation_name:
|
|
87
|
+
promoted.append({"name": table_name, "relation": relation_name})
|
|
88
|
+
|
|
89
|
+
return sorted(promoted, key=lambda t: t["name"]), sorted(skipped)
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def run_with_wap(args: List[str]) -> int:
|
|
94
|
+
"""Run dbt run with WAP deployment."""
|
|
95
|
+
import subprocess
|
|
96
|
+
|
|
97
|
+
return subprocess.run(['dbt', 'run'] + args).returncode
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def build_with_wap(args: List[str]) -> int:
|
|
101
|
+
"""Run dbt build with WAP deployment."""
|
|
102
|
+
import subprocess
|
|
103
|
+
|
|
104
|
+
return subprocess.run(['dbt', 'build'] + args).returncode
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: dbt-addons
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Requires-Python: >=3.8
|
|
5
|
+
Requires-Dist: dbt-core>=1.5.0
|
|
6
|
+
Requires-Dist: pyyaml>=6.0
|
|
7
|
+
Provides-Extra: dev
|
|
8
|
+
Requires-Dist: dbt-duckdb>=1.5.0; extra == "dev"
|
|
9
|
+
Dynamic: provides-extra
|
|
10
|
+
Dynamic: requires-dist
|
|
11
|
+
Dynamic: requires-python
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
setup.py
|
|
2
|
+
addons/__init__.py
|
|
3
|
+
addons/install.py
|
|
4
|
+
addons/cli/__init__.py
|
|
5
|
+
addons/cli/cli.py
|
|
6
|
+
addons/cli/logger.py
|
|
7
|
+
addons/wap/__init__.py
|
|
8
|
+
addons/wap/wap.py
|
|
9
|
+
addons/wap/macros/wap_deploy.sql
|
|
10
|
+
addons/wap/macros/providers/utils.sql
|
|
11
|
+
addons/wap/macros/providers/wap_bigquery.sql
|
|
12
|
+
addons/wap/macros/providers/wap_databricks.sql
|
|
13
|
+
addons/wap/macros/providers/wap_duckdb.sql
|
|
14
|
+
addons/wap/macros/providers/wap_rename.sql
|
|
15
|
+
addons/wap/macros/providers/wap_snowflake.sql
|
|
16
|
+
addons/wap/root_macros/generate_alias_name.sql
|
|
17
|
+
addons/wap/root_macros/generate_schema_name.sql
|
|
18
|
+
dbt_addons.egg-info/PKG-INFO
|
|
19
|
+
dbt_addons.egg-info/SOURCES.txt
|
|
20
|
+
dbt_addons.egg-info/dependency_links.txt
|
|
21
|
+
dbt_addons.egg-info/entry_points.txt
|
|
22
|
+
dbt_addons.egg-info/requires.txt
|
|
23
|
+
dbt_addons.egg-info/top_level.txt
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
addons
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
from setuptools import setup, find_packages
|
|
2
|
+
|
|
3
|
+
setup(
|
|
4
|
+
name="dbt-addons",
|
|
5
|
+
version="0.1.0",
|
|
6
|
+
packages=find_packages(),
|
|
7
|
+
package_data={
|
|
8
|
+
"addons": [
|
|
9
|
+
"wap/macros/**/*.sql",
|
|
10
|
+
"wap/macros/*.sql",
|
|
11
|
+
"wap/root_macros/*.sql",
|
|
12
|
+
],
|
|
13
|
+
},
|
|
14
|
+
install_requires=[
|
|
15
|
+
"dbt-core>=1.5.0",
|
|
16
|
+
"pyyaml>=6.0",
|
|
17
|
+
],
|
|
18
|
+
extras_require={
|
|
19
|
+
"dev": [
|
|
20
|
+
"dbt-duckdb>=1.5.0",
|
|
21
|
+
],
|
|
22
|
+
},
|
|
23
|
+
entry_points={
|
|
24
|
+
"console_scripts": [
|
|
25
|
+
"dbta=addons.cli.cli:main",
|
|
26
|
+
],
|
|
27
|
+
},
|
|
28
|
+
python_requires=">=3.8",
|
|
29
|
+
)
|