dvt-core 0.59.0a51__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dbt/__init__.py +7 -0
- dbt/_pydantic_shim.py +26 -0
- dbt/artifacts/__init__.py +0 -0
- dbt/artifacts/exceptions/__init__.py +1 -0
- dbt/artifacts/exceptions/schemas.py +31 -0
- dbt/artifacts/resources/__init__.py +116 -0
- dbt/artifacts/resources/base.py +67 -0
- dbt/artifacts/resources/types.py +93 -0
- dbt/artifacts/resources/v1/analysis.py +10 -0
- dbt/artifacts/resources/v1/catalog.py +23 -0
- dbt/artifacts/resources/v1/components.py +274 -0
- dbt/artifacts/resources/v1/config.py +277 -0
- dbt/artifacts/resources/v1/documentation.py +11 -0
- dbt/artifacts/resources/v1/exposure.py +51 -0
- dbt/artifacts/resources/v1/function.py +52 -0
- dbt/artifacts/resources/v1/generic_test.py +31 -0
- dbt/artifacts/resources/v1/group.py +21 -0
- dbt/artifacts/resources/v1/hook.py +11 -0
- dbt/artifacts/resources/v1/macro.py +29 -0
- dbt/artifacts/resources/v1/metric.py +172 -0
- dbt/artifacts/resources/v1/model.py +145 -0
- dbt/artifacts/resources/v1/owner.py +10 -0
- dbt/artifacts/resources/v1/saved_query.py +111 -0
- dbt/artifacts/resources/v1/seed.py +41 -0
- dbt/artifacts/resources/v1/semantic_layer_components.py +72 -0
- dbt/artifacts/resources/v1/semantic_model.py +314 -0
- dbt/artifacts/resources/v1/singular_test.py +14 -0
- dbt/artifacts/resources/v1/snapshot.py +91 -0
- dbt/artifacts/resources/v1/source_definition.py +84 -0
- dbt/artifacts/resources/v1/sql_operation.py +10 -0
- dbt/artifacts/resources/v1/unit_test_definition.py +77 -0
- dbt/artifacts/schemas/__init__.py +0 -0
- dbt/artifacts/schemas/base.py +191 -0
- dbt/artifacts/schemas/batch_results.py +24 -0
- dbt/artifacts/schemas/catalog/__init__.py +11 -0
- dbt/artifacts/schemas/catalog/v1/__init__.py +0 -0
- dbt/artifacts/schemas/catalog/v1/catalog.py +59 -0
- dbt/artifacts/schemas/freshness/__init__.py +1 -0
- dbt/artifacts/schemas/freshness/v3/__init__.py +0 -0
- dbt/artifacts/schemas/freshness/v3/freshness.py +158 -0
- dbt/artifacts/schemas/manifest/__init__.py +2 -0
- dbt/artifacts/schemas/manifest/v12/__init__.py +0 -0
- dbt/artifacts/schemas/manifest/v12/manifest.py +211 -0
- dbt/artifacts/schemas/results.py +147 -0
- dbt/artifacts/schemas/run/__init__.py +2 -0
- dbt/artifacts/schemas/run/v5/__init__.py +0 -0
- dbt/artifacts/schemas/run/v5/run.py +184 -0
- dbt/artifacts/schemas/upgrades/__init__.py +4 -0
- dbt/artifacts/schemas/upgrades/upgrade_manifest.py +174 -0
- dbt/artifacts/schemas/upgrades/upgrade_manifest_dbt_version.py +2 -0
- dbt/artifacts/utils/validation.py +153 -0
- dbt/cli/__init__.py +1 -0
- dbt/cli/context.py +17 -0
- dbt/cli/exceptions.py +57 -0
- dbt/cli/flags.py +560 -0
- dbt/cli/main.py +2660 -0
- dbt/cli/option_types.py +121 -0
- dbt/cli/options.py +80 -0
- dbt/cli/params.py +844 -0
- dbt/cli/requires.py +490 -0
- dbt/cli/resolvers.py +60 -0
- dbt/cli/types.py +40 -0
- dbt/clients/__init__.py +0 -0
- dbt/clients/checked_load.py +83 -0
- dbt/clients/git.py +164 -0
- dbt/clients/jinja.py +206 -0
- dbt/clients/jinja_static.py +245 -0
- dbt/clients/registry.py +192 -0
- dbt/clients/yaml_helper.py +68 -0
- dbt/compilation.py +876 -0
- dbt/compute/__init__.py +14 -0
- dbt/compute/engines/__init__.py +12 -0
- dbt/compute/engines/spark_engine.py +642 -0
- dbt/compute/federated_executor.py +1080 -0
- dbt/compute/filter_pushdown.py +273 -0
- dbt/compute/jar_provisioning.py +273 -0
- dbt/compute/java_compat.py +689 -0
- dbt/compute/jdbc_utils.py +1252 -0
- dbt/compute/metadata/__init__.py +63 -0
- dbt/compute/metadata/adapters_registry.py +370 -0
- dbt/compute/metadata/catalog_store.py +1036 -0
- dbt/compute/metadata/registry.py +674 -0
- dbt/compute/metadata/store.py +1020 -0
- dbt/compute/smart_selector.py +377 -0
- dbt/compute/spark_logger.py +272 -0
- dbt/compute/strategies/__init__.py +55 -0
- dbt/compute/strategies/base.py +165 -0
- dbt/compute/strategies/dataproc.py +207 -0
- dbt/compute/strategies/emr.py +203 -0
- dbt/compute/strategies/local.py +472 -0
- dbt/compute/strategies/standalone.py +262 -0
- dbt/config/__init__.py +4 -0
- dbt/config/catalogs.py +94 -0
- dbt/config/compute.py +513 -0
- dbt/config/dvt_profile.py +408 -0
- dbt/config/profile.py +422 -0
- dbt/config/project.py +888 -0
- dbt/config/project_utils.py +48 -0
- dbt/config/renderer.py +231 -0
- dbt/config/runtime.py +564 -0
- dbt/config/selectors.py +208 -0
- dbt/config/utils.py +77 -0
- dbt/constants.py +28 -0
- dbt/context/__init__.py +0 -0
- dbt/context/base.py +745 -0
- dbt/context/configured.py +135 -0
- dbt/context/context_config.py +382 -0
- dbt/context/docs.py +82 -0
- dbt/context/exceptions_jinja.py +178 -0
- dbt/context/macro_resolver.py +195 -0
- dbt/context/macros.py +171 -0
- dbt/context/manifest.py +72 -0
- dbt/context/providers.py +2249 -0
- dbt/context/query_header.py +13 -0
- dbt/context/secret.py +58 -0
- dbt/context/target.py +74 -0
- dbt/contracts/__init__.py +0 -0
- dbt/contracts/files.py +413 -0
- dbt/contracts/graph/__init__.py +0 -0
- dbt/contracts/graph/manifest.py +1904 -0
- dbt/contracts/graph/metrics.py +97 -0
- dbt/contracts/graph/model_config.py +70 -0
- dbt/contracts/graph/node_args.py +42 -0
- dbt/contracts/graph/nodes.py +1806 -0
- dbt/contracts/graph/semantic_manifest.py +232 -0
- dbt/contracts/graph/unparsed.py +811 -0
- dbt/contracts/project.py +419 -0
- dbt/contracts/results.py +53 -0
- dbt/contracts/selection.py +23 -0
- dbt/contracts/sql.py +85 -0
- dbt/contracts/state.py +68 -0
- dbt/contracts/util.py +46 -0
- dbt/deprecations.py +348 -0
- dbt/deps/__init__.py +0 -0
- dbt/deps/base.py +152 -0
- dbt/deps/git.py +195 -0
- dbt/deps/local.py +79 -0
- dbt/deps/registry.py +130 -0
- dbt/deps/resolver.py +149 -0
- dbt/deps/tarball.py +120 -0
- dbt/docs/source/_ext/dbt_click.py +119 -0
- dbt/docs/source/conf.py +32 -0
- dbt/env_vars.py +64 -0
- dbt/event_time/event_time.py +40 -0
- dbt/event_time/sample_window.py +60 -0
- dbt/events/__init__.py +15 -0
- dbt/events/base_types.py +36 -0
- dbt/events/core_types_pb2.py +2 -0
- dbt/events/logging.py +108 -0
- dbt/events/types.py +2516 -0
- dbt/exceptions.py +1486 -0
- dbt/flags.py +89 -0
- dbt/graph/__init__.py +11 -0
- dbt/graph/cli.py +249 -0
- dbt/graph/graph.py +172 -0
- dbt/graph/queue.py +214 -0
- dbt/graph/selector.py +374 -0
- dbt/graph/selector_methods.py +975 -0
- dbt/graph/selector_spec.py +222 -0
- dbt/graph/thread_pool.py +18 -0
- dbt/hooks.py +21 -0
- dbt/include/README.md +49 -0
- dbt/include/__init__.py +3 -0
- dbt/include/data/adapters_registry.duckdb +0 -0
- dbt/include/data/build_comprehensive_registry.py +1254 -0
- dbt/include/data/build_registry.py +242 -0
- dbt/include/data/csv/adapter_queries.csv +33 -0
- dbt/include/data/csv/syntax_rules.csv +9 -0
- dbt/include/data/csv/type_mappings_bigquery.csv +28 -0
- dbt/include/data/csv/type_mappings_databricks.csv +30 -0
- dbt/include/data/csv/type_mappings_mysql.csv +40 -0
- dbt/include/data/csv/type_mappings_oracle.csv +30 -0
- dbt/include/data/csv/type_mappings_postgres.csv +56 -0
- dbt/include/data/csv/type_mappings_redshift.csv +33 -0
- dbt/include/data/csv/type_mappings_snowflake.csv +38 -0
- dbt/include/data/csv/type_mappings_sqlserver.csv +35 -0
- dbt/include/dvt_starter_project/README.md +15 -0
- dbt/include/dvt_starter_project/__init__.py +3 -0
- dbt/include/dvt_starter_project/analyses/PLACEHOLDER +0 -0
- dbt/include/dvt_starter_project/dvt_project.yml +39 -0
- dbt/include/dvt_starter_project/logs/PLACEHOLDER +0 -0
- dbt/include/dvt_starter_project/macros/PLACEHOLDER +0 -0
- dbt/include/dvt_starter_project/models/example/my_first_dbt_model.sql +27 -0
- dbt/include/dvt_starter_project/models/example/my_second_dbt_model.sql +6 -0
- dbt/include/dvt_starter_project/models/example/schema.yml +21 -0
- dbt/include/dvt_starter_project/seeds/PLACEHOLDER +0 -0
- dbt/include/dvt_starter_project/snapshots/PLACEHOLDER +0 -0
- dbt/include/dvt_starter_project/tests/PLACEHOLDER +0 -0
- dbt/internal_deprecations.py +26 -0
- dbt/jsonschemas/__init__.py +3 -0
- dbt/jsonschemas/jsonschemas.py +309 -0
- dbt/jsonschemas/project/0.0.110.json +4717 -0
- dbt/jsonschemas/project/0.0.85.json +2015 -0
- dbt/jsonschemas/resources/0.0.110.json +2636 -0
- dbt/jsonschemas/resources/0.0.85.json +2536 -0
- dbt/jsonschemas/resources/latest.json +6773 -0
- dbt/links.py +4 -0
- dbt/materializations/__init__.py +0 -0
- dbt/materializations/incremental/__init__.py +0 -0
- dbt/materializations/incremental/microbatch.py +236 -0
- dbt/mp_context.py +8 -0
- dbt/node_types.py +37 -0
- dbt/parser/__init__.py +23 -0
- dbt/parser/analysis.py +21 -0
- dbt/parser/base.py +548 -0
- dbt/parser/common.py +266 -0
- dbt/parser/docs.py +52 -0
- dbt/parser/fixtures.py +51 -0
- dbt/parser/functions.py +30 -0
- dbt/parser/generic_test.py +100 -0
- dbt/parser/generic_test_builders.py +333 -0
- dbt/parser/hooks.py +122 -0
- dbt/parser/macros.py +137 -0
- dbt/parser/manifest.py +2208 -0
- dbt/parser/models.py +573 -0
- dbt/parser/partial.py +1178 -0
- dbt/parser/read_files.py +445 -0
- dbt/parser/schema_generic_tests.py +422 -0
- dbt/parser/schema_renderer.py +111 -0
- dbt/parser/schema_yaml_readers.py +935 -0
- dbt/parser/schemas.py +1466 -0
- dbt/parser/search.py +149 -0
- dbt/parser/seeds.py +28 -0
- dbt/parser/singular_test.py +20 -0
- dbt/parser/snapshots.py +44 -0
- dbt/parser/sources.py +558 -0
- dbt/parser/sql.py +62 -0
- dbt/parser/unit_tests.py +621 -0
- dbt/plugins/__init__.py +20 -0
- dbt/plugins/contracts.py +9 -0
- dbt/plugins/exceptions.py +2 -0
- dbt/plugins/manager.py +163 -0
- dbt/plugins/manifest.py +21 -0
- dbt/profiler.py +20 -0
- dbt/py.typed +1 -0
- dbt/query_analyzer.py +410 -0
- dbt/runners/__init__.py +2 -0
- dbt/runners/exposure_runner.py +7 -0
- dbt/runners/no_op_runner.py +45 -0
- dbt/runners/saved_query_runner.py +7 -0
- dbt/selected_resources.py +8 -0
- dbt/task/__init__.py +0 -0
- dbt/task/base.py +506 -0
- dbt/task/build.py +197 -0
- dbt/task/clean.py +56 -0
- dbt/task/clone.py +161 -0
- dbt/task/compile.py +150 -0
- dbt/task/compute.py +458 -0
- dbt/task/debug.py +513 -0
- dbt/task/deps.py +280 -0
- dbt/task/docs/__init__.py +3 -0
- dbt/task/docs/api/__init__.py +23 -0
- dbt/task/docs/api/catalog.py +204 -0
- dbt/task/docs/api/lineage.py +234 -0
- dbt/task/docs/api/profile.py +204 -0
- dbt/task/docs/api/spark.py +186 -0
- dbt/task/docs/generate.py +1002 -0
- dbt/task/docs/index.html +250 -0
- dbt/task/docs/serve.py +174 -0
- dbt/task/dvt_output.py +509 -0
- dbt/task/dvt_run.py +282 -0
- dbt/task/dvt_seed.py +806 -0
- dbt/task/freshness.py +322 -0
- dbt/task/function.py +121 -0
- dbt/task/group_lookup.py +46 -0
- dbt/task/init.py +1022 -0
- dbt/task/java.py +316 -0
- dbt/task/list.py +236 -0
- dbt/task/metadata.py +804 -0
- dbt/task/migrate.py +714 -0
- dbt/task/printer.py +175 -0
- dbt/task/profile.py +1489 -0
- dbt/task/profile_serve.py +662 -0
- dbt/task/retract.py +441 -0
- dbt/task/retry.py +175 -0
- dbt/task/run.py +1647 -0
- dbt/task/run_operation.py +141 -0
- dbt/task/runnable.py +758 -0
- dbt/task/seed.py +103 -0
- dbt/task/show.py +149 -0
- dbt/task/snapshot.py +56 -0
- dbt/task/spark.py +414 -0
- dbt/task/sql.py +110 -0
- dbt/task/target_sync.py +814 -0
- dbt/task/test.py +464 -0
- dbt/tests/fixtures/__init__.py +1 -0
- dbt/tests/fixtures/project.py +620 -0
- dbt/tests/util.py +651 -0
- dbt/tracking.py +529 -0
- dbt/utils/__init__.py +3 -0
- dbt/utils/artifact_upload.py +151 -0
- dbt/utils/utils.py +408 -0
- dbt/version.py +271 -0
- dvt_cli/__init__.py +158 -0
- dvt_core-0.59.0a51.dist-info/METADATA +288 -0
- dvt_core-0.59.0a51.dist-info/RECORD +299 -0
- dvt_core-0.59.0a51.dist-info/WHEEL +5 -0
- dvt_core-0.59.0a51.dist-info/entry_points.txt +2 -0
- dvt_core-0.59.0a51.dist-info/top_level.txt +2 -0
dbt/task/migrate.py
ADDED
|
@@ -0,0 +1,714 @@
|
|
|
1
|
+
"""
|
|
2
|
+
DVT Migration Task - Migrate from dbt to DVT configuration.
|
|
3
|
+
|
|
4
|
+
Two modes:
|
|
5
|
+
- Mode A: Convert dbt project to DVT (not in DVT project)
|
|
6
|
+
- Mode B: Import dbt project INTO DVT project (in DVT project)
|
|
7
|
+
"""
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from dataclasses import dataclass
|
|
10
|
+
from typing import List, Optional
|
|
11
|
+
import shutil
|
|
12
|
+
import click
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@dataclass
|
|
16
|
+
class MigrationResult:
|
|
17
|
+
"""Result of a migration operation."""
|
|
18
|
+
|
|
19
|
+
success: bool
|
|
20
|
+
message: str
|
|
21
|
+
files_copied: int = 0
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class MigrateTask:
|
|
25
|
+
"""
|
|
26
|
+
DVT Migration Task.
|
|
27
|
+
|
|
28
|
+
Handles two modes:
|
|
29
|
+
- Mode A: Convert dbt project to DVT (when not in DVT project)
|
|
30
|
+
- Mode B: Import dbt project INTO DVT project (when in DVT project)
|
|
31
|
+
"""
|
|
32
|
+
|
|
33
|
+
def __init__(
|
|
34
|
+
self,
|
|
35
|
+
source_path: Optional[str] = None,
|
|
36
|
+
profiles_only: bool = False,
|
|
37
|
+
project_only: bool = False,
|
|
38
|
+
full: bool = False,
|
|
39
|
+
dry_run: bool = False,
|
|
40
|
+
project_dir: Optional[str] = None,
|
|
41
|
+
):
|
|
42
|
+
self.source_path = Path(source_path) if source_path else None
|
|
43
|
+
self.profiles_only = profiles_only
|
|
44
|
+
self.project_only = project_only
|
|
45
|
+
self.full = full
|
|
46
|
+
self.dry_run = dry_run
|
|
47
|
+
self.project_dir = Path(project_dir) if project_dir else Path.cwd()
|
|
48
|
+
|
|
49
|
+
def run(self) -> bool:
|
|
50
|
+
"""Execute migration based on context and flags."""
|
|
51
|
+
# Detect which mode we're in
|
|
52
|
+
in_dvt_project = (self.project_dir / "dvt_project.yml").exists()
|
|
53
|
+
|
|
54
|
+
if in_dvt_project and self.source_path:
|
|
55
|
+
# Mode B: Import dbt project INTO current DVT project
|
|
56
|
+
return self._import_dbt_project()
|
|
57
|
+
elif in_dvt_project and not self.source_path:
|
|
58
|
+
# Inside DVT project but no path - show help
|
|
59
|
+
click.echo("You are inside a DVT project.")
|
|
60
|
+
click.echo("To import a dbt project, provide the path:")
|
|
61
|
+
click.echo(" dvt migrate /path/to/dbt_project")
|
|
62
|
+
return True
|
|
63
|
+
else:
|
|
64
|
+
# Mode A: Convert current dbt project to DVT
|
|
65
|
+
return self._convert_to_dvt()
|
|
66
|
+
|
|
67
|
+
def _convert_to_dvt(self) -> bool:
|
|
68
|
+
"""Mode A: Convert dbt project in current directory to DVT."""
|
|
69
|
+
results = []
|
|
70
|
+
|
|
71
|
+
if self.profiles_only:
|
|
72
|
+
results.append(self._migrate_profiles())
|
|
73
|
+
elif self.project_only:
|
|
74
|
+
results.append(self._migrate_project_file())
|
|
75
|
+
elif self.full:
|
|
76
|
+
results.append(self._migrate_profiles())
|
|
77
|
+
results.append(self._migrate_project_file())
|
|
78
|
+
else:
|
|
79
|
+
results = self._auto_detect_and_migrate()
|
|
80
|
+
|
|
81
|
+
self._show_summary(results)
|
|
82
|
+
return all(r.success for r in results if r)
|
|
83
|
+
|
|
84
|
+
def _auto_detect_and_migrate(self) -> List[MigrationResult]:
|
|
85
|
+
"""Auto-detect what needs to be migrated."""
|
|
86
|
+
results = []
|
|
87
|
+
|
|
88
|
+
click.echo("Detecting dbt configuration...")
|
|
89
|
+
|
|
90
|
+
dbt_profiles_path = Path.home() / ".dbt" / "profiles.yml"
|
|
91
|
+
dbt_project_path = self.project_dir / "dbt_project.yml"
|
|
92
|
+
|
|
93
|
+
has_dbt_profiles = dbt_profiles_path.exists()
|
|
94
|
+
has_dbt_project = dbt_project_path.exists()
|
|
95
|
+
|
|
96
|
+
if has_dbt_profiles:
|
|
97
|
+
click.echo(f" Found ~/.dbt/profiles.yml")
|
|
98
|
+
if has_dbt_project:
|
|
99
|
+
click.echo(f" Found dbt_project.yml")
|
|
100
|
+
|
|
101
|
+
if not has_dbt_profiles and not has_dbt_project:
|
|
102
|
+
click.echo("\nNo dbt configuration found - nothing to migrate.")
|
|
103
|
+
click.echo("Use 'dvt init' to create a new DVT project.")
|
|
104
|
+
return [MigrationResult(True, "Nothing to migrate")]
|
|
105
|
+
|
|
106
|
+
click.echo("")
|
|
107
|
+
|
|
108
|
+
# Migrate profiles if found
|
|
109
|
+
if has_dbt_profiles:
|
|
110
|
+
click.echo("Migrating profiles...")
|
|
111
|
+
results.append(self._migrate_profiles())
|
|
112
|
+
|
|
113
|
+
# Migrate project if found
|
|
114
|
+
if has_dbt_project:
|
|
115
|
+
click.echo("\nMigrating project...")
|
|
116
|
+
results.append(self._migrate_project_file())
|
|
117
|
+
|
|
118
|
+
return results
|
|
119
|
+
|
|
120
|
+
def _migrate_profiles(self) -> MigrationResult:
|
|
121
|
+
"""Migrate project-specific profile from ~/.dbt/profiles.yml to ~/.dvt/profiles.yml.
|
|
122
|
+
|
|
123
|
+
v0.59.0a2: Changed to be surgical - only copies the profile matching the current
|
|
124
|
+
project, not all profiles from ~/.dbt/profiles.yml.
|
|
125
|
+
|
|
126
|
+
v0.59.0a19: If profile not found in ~/.dbt/profiles.yml, creates a starter
|
|
127
|
+
DuckDB profile (like dvt init does).
|
|
128
|
+
"""
|
|
129
|
+
try:
|
|
130
|
+
import yaml
|
|
131
|
+
except ImportError:
|
|
132
|
+
import ruamel.yaml as yaml
|
|
133
|
+
|
|
134
|
+
old_path = Path.home() / ".dbt" / "profiles.yml"
|
|
135
|
+
new_path = Path.home() / ".dvt" / "profiles.yml"
|
|
136
|
+
|
|
137
|
+
# Get the profile name from dbt_project.yml
|
|
138
|
+
dbt_project_path = self.project_dir / "dbt_project.yml"
|
|
139
|
+
if dbt_project_path.exists():
|
|
140
|
+
with open(dbt_project_path) as f:
|
|
141
|
+
project_config = yaml.safe_load(f) or {}
|
|
142
|
+
# Use 'profile' field, fall back to 'name', then directory name
|
|
143
|
+
profile_name = project_config.get("profile", project_config.get("name", self.project_dir.name))
|
|
144
|
+
else:
|
|
145
|
+
# No dbt_project.yml, use directory name
|
|
146
|
+
profile_name = self.project_dir.name
|
|
147
|
+
|
|
148
|
+
# Check if profile exists in ~/.dbt/profiles.yml
|
|
149
|
+
profile_to_migrate = None
|
|
150
|
+
if old_path.exists():
|
|
151
|
+
with open(old_path) as f:
|
|
152
|
+
old_profiles = yaml.safe_load(f) or {}
|
|
153
|
+
if profile_name in old_profiles:
|
|
154
|
+
profile_to_migrate = {profile_name: old_profiles[profile_name]}
|
|
155
|
+
|
|
156
|
+
# v0.59.0a19: If profile not found, create a starter DuckDB profile
|
|
157
|
+
if profile_to_migrate is None:
|
|
158
|
+
click.echo(f" No profile '{profile_name}' found in ~/.dbt/profiles.yml")
|
|
159
|
+
click.echo(f" Creating starter DuckDB profile...")
|
|
160
|
+
|
|
161
|
+
# Create .dvt directory and default.duckdb
|
|
162
|
+
dvt_dir = self.project_dir / ".dvt"
|
|
163
|
+
dvt_dir.mkdir(parents=True, exist_ok=True)
|
|
164
|
+
duckdb_path = dvt_dir / "default.duckdb"
|
|
165
|
+
|
|
166
|
+
# Create empty DuckDB file if it doesn't exist
|
|
167
|
+
if not duckdb_path.exists():
|
|
168
|
+
try:
|
|
169
|
+
import duckdb
|
|
170
|
+
conn = duckdb.connect(str(duckdb_path))
|
|
171
|
+
conn.close()
|
|
172
|
+
except Exception:
|
|
173
|
+
# Just create empty file if duckdb import fails
|
|
174
|
+
duckdb_path.touch()
|
|
175
|
+
|
|
176
|
+
# Create starter profile
|
|
177
|
+
profile_to_migrate = {
|
|
178
|
+
profile_name: {
|
|
179
|
+
"target": "dev",
|
|
180
|
+
"outputs": {
|
|
181
|
+
"dev": {
|
|
182
|
+
"type": "duckdb",
|
|
183
|
+
"path": str(duckdb_path.resolve()),
|
|
184
|
+
"threads": 4,
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
# Merge with existing DVT profiles if any
|
|
191
|
+
if new_path.exists():
|
|
192
|
+
with open(new_path) as f:
|
|
193
|
+
existing = yaml.safe_load(f) or {}
|
|
194
|
+
# DVT profiles take precedence (don't overwrite)
|
|
195
|
+
if profile_name not in existing:
|
|
196
|
+
existing[profile_name] = profile_to_migrate[profile_name]
|
|
197
|
+
merged = True
|
|
198
|
+
else:
|
|
199
|
+
click.echo(f" Profile '{profile_name}' already exists in ~/.dvt/profiles.yml - skipping")
|
|
200
|
+
return MigrationResult(True, f"Profile '{profile_name}' already exists")
|
|
201
|
+
profiles = existing
|
|
202
|
+
else:
|
|
203
|
+
profiles = profile_to_migrate
|
|
204
|
+
merged = False
|
|
205
|
+
|
|
206
|
+
# Write to DVT profiles
|
|
207
|
+
if not self.dry_run:
|
|
208
|
+
new_path.parent.mkdir(parents=True, exist_ok=True)
|
|
209
|
+
with open(new_path, "w") as f:
|
|
210
|
+
f.write("# DVT Profiles Configuration\n\n")
|
|
211
|
+
yaml.dump(profiles, f, default_flow_style=False)
|
|
212
|
+
|
|
213
|
+
action = "Merged" if merged else "Migrated"
|
|
214
|
+
click.echo(f" {action} profile '{profile_name}' to ~/.dvt/profiles.yml")
|
|
215
|
+
else:
|
|
216
|
+
click.echo(f" [DRY RUN] Would migrate profile '{profile_name}'")
|
|
217
|
+
|
|
218
|
+
return MigrationResult(True, f"Migrated profile '{profile_name}'")
|
|
219
|
+
|
|
220
|
+
def _migrate_project_file(self) -> MigrationResult:
|
|
221
|
+
"""Create dvt_project.yml from fresh template (like dbt init) + DVT additions.
|
|
222
|
+
|
|
223
|
+
v0.59.0a2: Creates fresh template matching dbt init format, plus DVT-specific
|
|
224
|
+
flatfile-paths config. Creates flatfiles/ directory. User can copy project-specific
|
|
225
|
+
configs from the backed-up dbt_project.yml.bak if needed.
|
|
226
|
+
"""
|
|
227
|
+
try:
|
|
228
|
+
import yaml
|
|
229
|
+
except ImportError:
|
|
230
|
+
import ruamel.yaml as yaml
|
|
231
|
+
|
|
232
|
+
old_path = self.project_dir / "dbt_project.yml"
|
|
233
|
+
new_path = self.project_dir / "dvt_project.yml"
|
|
234
|
+
backup_path = self.project_dir / "dbt_project.yml.bak"
|
|
235
|
+
|
|
236
|
+
if not old_path.exists():
|
|
237
|
+
click.echo(" No dbt_project.yml found")
|
|
238
|
+
return MigrationResult(False, "No dbt_project.yml found")
|
|
239
|
+
|
|
240
|
+
if new_path.exists():
|
|
241
|
+
click.echo(" dvt_project.yml already exists - skipping")
|
|
242
|
+
return MigrationResult(True, "dvt_project.yml already exists")
|
|
243
|
+
|
|
244
|
+
# Get project name and profile from dbt_project.yml
|
|
245
|
+
project_name = self.project_dir.name
|
|
246
|
+
profile_name = project_name # Default to project name
|
|
247
|
+
with open(old_path, "r") as f:
|
|
248
|
+
try:
|
|
249
|
+
config = yaml.safe_load(f)
|
|
250
|
+
if config:
|
|
251
|
+
if "name" in config:
|
|
252
|
+
project_name = config["name"]
|
|
253
|
+
if "profile" in config:
|
|
254
|
+
profile_name = config["profile"]
|
|
255
|
+
except Exception:
|
|
256
|
+
pass
|
|
257
|
+
|
|
258
|
+
if not self.dry_run:
|
|
259
|
+
# 1. Backup dbt_project.yml
|
|
260
|
+
if backup_path.exists():
|
|
261
|
+
# Add timestamp if backup already exists
|
|
262
|
+
import time
|
|
263
|
+
|
|
264
|
+
timestamp = time.strftime("%Y%m%d_%H%M%S")
|
|
265
|
+
backup_path = self.project_dir / f"dbt_project.yml.{timestamp}.bak"
|
|
266
|
+
shutil.copy2(old_path, backup_path)
|
|
267
|
+
click.echo(f" Backed up dbt_project.yml → {backup_path.name}")
|
|
268
|
+
|
|
269
|
+
# 2. Create fresh dvt_project.yml from standard dbt init template + DVT additions
|
|
270
|
+
dvt_project_content = f"""name: '{project_name}'
|
|
271
|
+
version: '1.0.0'
|
|
272
|
+
|
|
273
|
+
# This setting configures which "profile" dbt uses for this project.
|
|
274
|
+
profile: '{profile_name}'
|
|
275
|
+
|
|
276
|
+
# These configurations specify where dbt should look for different types of files.
|
|
277
|
+
# The `model-paths` config, for example, states that models in this project can be
|
|
278
|
+
# found in the "models/" directory. You probably won't need to change these!
|
|
279
|
+
model-paths: ["models"]
|
|
280
|
+
analysis-paths: ["analyses"]
|
|
281
|
+
test-paths: ["tests"]
|
|
282
|
+
seed-paths: ["seeds"]
|
|
283
|
+
macro-paths: ["macros"]
|
|
284
|
+
snapshot-paths: ["snapshots"]
|
|
285
|
+
|
|
286
|
+
# DVT-specific: Path for flat files (CSV, Parquet, etc.) for local data ingestion
|
|
287
|
+
flatfile-paths: ["flatfiles"]
|
|
288
|
+
|
|
289
|
+
clean-targets:
|
|
290
|
+
- "target"
|
|
291
|
+
- "dbt_packages"
|
|
292
|
+
|
|
293
|
+
# Configuring models
|
|
294
|
+
# Full documentation: https://docs.getdbt.com/docs/configuring-models
|
|
295
|
+
|
|
296
|
+
# In this example config, we tell dbt to build all models in the example/
|
|
297
|
+
# directory as views. These settings can be overridden in the individual model
|
|
298
|
+
# files using the `{{ config(...) }}` macro.
|
|
299
|
+
models:
|
|
300
|
+
{project_name}:
|
|
301
|
+
# Config indicated by + and applies to all files under models/example/
|
|
302
|
+
+materialized: view
|
|
303
|
+
"""
|
|
304
|
+
with open(new_path, "w") as f:
|
|
305
|
+
f.write(dvt_project_content)
|
|
306
|
+
click.echo(" Created dvt_project.yml")
|
|
307
|
+
|
|
308
|
+
# 3. Create flatfiles/ directory
|
|
309
|
+
flatfiles_dir = self.project_dir / "flatfiles"
|
|
310
|
+
flatfiles_dir.mkdir(exist_ok=True)
|
|
311
|
+
gitignore_path = flatfiles_dir / ".gitignore"
|
|
312
|
+
if not gitignore_path.exists():
|
|
313
|
+
with open(gitignore_path, "w") as f:
|
|
314
|
+
f.write("# Ignore all flat files (CSV, Parquet, etc.)\n*\n!.gitignore\n")
|
|
315
|
+
click.echo(" Created flatfiles/ directory")
|
|
316
|
+
|
|
317
|
+
# 4. Create .dvt/ directory with computes.yml and metadata_store.duckdb
|
|
318
|
+
dvt_dir = self.project_dir / ".dvt"
|
|
319
|
+
dvt_dir.mkdir(exist_ok=True)
|
|
320
|
+
|
|
321
|
+
# Create computes.yml
|
|
322
|
+
computes_path = dvt_dir / "computes.yml"
|
|
323
|
+
if not computes_path.exists():
|
|
324
|
+
computes_content = """# DVT Compute Configuration
|
|
325
|
+
# See: https://github.com/dvt-core/dvt-core#compute-configuration
|
|
326
|
+
|
|
327
|
+
# Default compute engine for federation queries
|
|
328
|
+
target_compute: spark-local
|
|
329
|
+
|
|
330
|
+
# Available compute engines
|
|
331
|
+
computes:
|
|
332
|
+
spark-local:
|
|
333
|
+
type: local
|
|
334
|
+
app_name: DVT-Spark
|
|
335
|
+
config:
|
|
336
|
+
spark.sql.execution.arrow.pyspark.enabled: "true"
|
|
337
|
+
spark.sql.adaptive.enabled: "true"
|
|
338
|
+
"""
|
|
339
|
+
with open(computes_path, "w") as f:
|
|
340
|
+
f.write(computes_content)
|
|
341
|
+
click.echo(" Created .dvt/computes.yml")
|
|
342
|
+
|
|
343
|
+
# Create metadata_store.duckdb
|
|
344
|
+
metadata_store_path = dvt_dir / "metadata_store.duckdb"
|
|
345
|
+
if not metadata_store_path.exists():
|
|
346
|
+
try:
|
|
347
|
+
import duckdb
|
|
348
|
+
conn = duckdb.connect(str(metadata_store_path))
|
|
349
|
+
conn.close()
|
|
350
|
+
except Exception:
|
|
351
|
+
metadata_store_path.touch()
|
|
352
|
+
click.echo(" Created .dvt/metadata_store.duckdb")
|
|
353
|
+
else:
|
|
354
|
+
click.echo(" [DRY RUN] Would backup dbt_project.yml")
|
|
355
|
+
click.echo(" [DRY RUN] Would create dvt_project.yml")
|
|
356
|
+
click.echo(" [DRY RUN] Would create flatfiles/ directory")
|
|
357
|
+
|
|
358
|
+
return MigrationResult(True, "Project migrated")
|
|
359
|
+
|
|
360
|
+
def _import_dbt_project(self) -> bool:
|
|
361
|
+
"""Mode B: Import dbt/DVT project INTO current DVT project.
|
|
362
|
+
|
|
363
|
+
v0.59.0a23: Now supports importing both dbt and DVT projects.
|
|
364
|
+
Checks for dvt_project.yml first, then dbt_project.yml.
|
|
365
|
+
|
|
366
|
+
v0.59.0a24: For dbt projects ONLY, patches sources.yml files to add
|
|
367
|
+
DVT's required connection: config. DVT projects are expected to
|
|
368
|
+
already have this config (user's responsibility).
|
|
369
|
+
"""
|
|
370
|
+
try:
|
|
371
|
+
import yaml
|
|
372
|
+
except ImportError:
|
|
373
|
+
import ruamel.yaml as yaml
|
|
374
|
+
|
|
375
|
+
# Verify source is a dbt or DVT project (check DVT first)
|
|
376
|
+
dvt_config_path = self.source_path / "dvt_project.yml"
|
|
377
|
+
dbt_config_path = self.source_path / "dbt_project.yml"
|
|
378
|
+
|
|
379
|
+
if dvt_config_path.exists():
|
|
380
|
+
source_config_path = dvt_config_path
|
|
381
|
+
project_type = "DVT"
|
|
382
|
+
elif dbt_config_path.exists():
|
|
383
|
+
source_config_path = dbt_config_path
|
|
384
|
+
project_type = "dbt"
|
|
385
|
+
else:
|
|
386
|
+
click.echo(f"Error: No dvt_project.yml or dbt_project.yml found in {self.source_path}")
|
|
387
|
+
return False
|
|
388
|
+
|
|
389
|
+
# Get source project name and profile from project config
|
|
390
|
+
with open(source_config_path) as f:
|
|
391
|
+
project_config = yaml.safe_load(f)
|
|
392
|
+
source_name = project_config.get("name", self.source_path.name)
|
|
393
|
+
source_profile_name = project_config.get("profile", source_name)
|
|
394
|
+
|
|
395
|
+
if self.dry_run:
|
|
396
|
+
click.echo(f"Importing {project_type} project '{source_name}' (DRY RUN)...")
|
|
397
|
+
else:
|
|
398
|
+
click.echo(f"Importing {project_type} project '{source_name}' into current DVT project...")
|
|
399
|
+
|
|
400
|
+
click.echo("")
|
|
401
|
+
results = []
|
|
402
|
+
|
|
403
|
+
# Copy directories
|
|
404
|
+
dirs_to_copy = ["models", "seeds", "tests", "macros", "snapshots", "analyses"]
|
|
405
|
+
models_copied = False
|
|
406
|
+
for dir_name in dirs_to_copy:
|
|
407
|
+
result = self._copy_directory(dir_name, source_name)
|
|
408
|
+
if result:
|
|
409
|
+
results.append(result)
|
|
410
|
+
if dir_name == "models" and result.success and result.files_copied > 0:
|
|
411
|
+
models_copied = True
|
|
412
|
+
|
|
413
|
+
# v0.59.0a24: Patch sources.yml for dbt projects ONLY
|
|
414
|
+
# DVT projects should already have connection: config - user's responsibility
|
|
415
|
+
if project_type == "dbt" and models_copied:
|
|
416
|
+
# Get the default target from source profile to build connection name
|
|
417
|
+
default_target = self._get_source_default_target(source_profile_name)
|
|
418
|
+
connection_name = f"{source_name}_{default_target}"
|
|
419
|
+
patched = self._patch_sources_connection(
|
|
420
|
+
self.project_dir / "models" / source_name,
|
|
421
|
+
connection_name
|
|
422
|
+
)
|
|
423
|
+
if patched > 0:
|
|
424
|
+
click.echo(f" Patched {patched} source(s) with connection: {connection_name}")
|
|
425
|
+
elif project_type == "DVT":
|
|
426
|
+
# Warn DVT project users to verify connection config
|
|
427
|
+
models_dir = self.project_dir / "models" / source_name
|
|
428
|
+
if models_dir.exists():
|
|
429
|
+
sources_files = list(models_dir.rglob("*sources*.yml"))
|
|
430
|
+
if sources_files:
|
|
431
|
+
click.echo(f" Note: DVT project detected - verify sources.yml has connection: config")
|
|
432
|
+
|
|
433
|
+
# Merge profile targets
|
|
434
|
+
profile_result = self._import_profile_targets(source_name)
|
|
435
|
+
results.append(profile_result)
|
|
436
|
+
|
|
437
|
+
self._show_import_summary(source_name, results)
|
|
438
|
+
return all(r.success for r in results if r)
|
|
439
|
+
|
|
440
|
+
def _get_source_default_target(self, profile_name: str) -> str:
|
|
441
|
+
"""Get the default target from a source profile.
|
|
442
|
+
|
|
443
|
+
Checks ~/.dvt/profiles.yml first, then ~/.dbt/profiles.yml.
|
|
444
|
+
Returns 'default' if profile not found.
|
|
445
|
+
"""
|
|
446
|
+
try:
|
|
447
|
+
import yaml
|
|
448
|
+
except ImportError:
|
|
449
|
+
import ruamel.yaml as yaml
|
|
450
|
+
|
|
451
|
+
# Check DVT profiles first
|
|
452
|
+
dvt_profiles_path = Path.home() / ".dvt" / "profiles.yml"
|
|
453
|
+
if dvt_profiles_path.exists():
|
|
454
|
+
with open(dvt_profiles_path) as f:
|
|
455
|
+
profiles = yaml.safe_load(f) or {}
|
|
456
|
+
if profile_name in profiles:
|
|
457
|
+
return profiles[profile_name].get("target", "default")
|
|
458
|
+
|
|
459
|
+
# Then check dbt profiles
|
|
460
|
+
dbt_profiles_path = Path.home() / ".dbt" / "profiles.yml"
|
|
461
|
+
if dbt_profiles_path.exists():
|
|
462
|
+
with open(dbt_profiles_path) as f:
|
|
463
|
+
profiles = yaml.safe_load(f) or {}
|
|
464
|
+
if profile_name in profiles:
|
|
465
|
+
return profiles[profile_name].get("target", "default")
|
|
466
|
+
|
|
467
|
+
return "default"
|
|
468
|
+
|
|
469
|
+
def _patch_sources_connection(self, target_dir: Path, connection_name: str) -> int:
|
|
470
|
+
"""Add connection: config to all .yml files that contain sources definitions.
|
|
471
|
+
|
|
472
|
+
ONLY called for dbt projects (not DVT projects).
|
|
473
|
+
DVT requires sources to specify which connection they read from.
|
|
474
|
+
For imported dbt projects, default to the imported project's connection.
|
|
475
|
+
|
|
476
|
+
v0.59.0a24: New method for surgical sources.yml patching.
|
|
477
|
+
Note: Sources can be defined in ANY .yml file (not just *sources*.yml).
|
|
478
|
+
A file contains sources if it has a top-level 'sources:' key.
|
|
479
|
+
|
|
480
|
+
Args:
|
|
481
|
+
target_dir: Directory containing the copied models
|
|
482
|
+
connection_name: Connection name to add (e.g., "Cocacola_DWH_postgres")
|
|
483
|
+
|
|
484
|
+
Returns:
|
|
485
|
+
Count of patched sources.
|
|
486
|
+
"""
|
|
487
|
+
try:
|
|
488
|
+
import yaml
|
|
489
|
+
except ImportError:
|
|
490
|
+
import ruamel.yaml as yaml
|
|
491
|
+
|
|
492
|
+
if self.dry_run:
|
|
493
|
+
# Count what would be patched by scanning ALL .yml files for sources: key
|
|
494
|
+
count = 0
|
|
495
|
+
for yml_file in target_dir.rglob("*.yml"):
|
|
496
|
+
try:
|
|
497
|
+
with open(yml_file) as f:
|
|
498
|
+
content = yaml.safe_load(f) or {}
|
|
499
|
+
if content.get("sources"):
|
|
500
|
+
for source in content.get("sources", []):
|
|
501
|
+
if "connection" not in source:
|
|
502
|
+
count += 1
|
|
503
|
+
except Exception:
|
|
504
|
+
pass
|
|
505
|
+
if count > 0:
|
|
506
|
+
click.echo(f" [DRY RUN] Would patch {count} source(s) with connection: {connection_name}")
|
|
507
|
+
return count
|
|
508
|
+
|
|
509
|
+
patched_count = 0
|
|
510
|
+
|
|
511
|
+
# Scan ALL .yml files for sources: key (sources can be in any yml file)
|
|
512
|
+
for yml_file in target_dir.rglob("*.yml"):
|
|
513
|
+
try:
|
|
514
|
+
with open(yml_file) as f:
|
|
515
|
+
content = yaml.safe_load(f) or {}
|
|
516
|
+
|
|
517
|
+
# Skip if no sources defined in this file
|
|
518
|
+
if not content.get("sources"):
|
|
519
|
+
continue
|
|
520
|
+
|
|
521
|
+
modified = False
|
|
522
|
+
for source in content.get("sources", []):
|
|
523
|
+
if "connection" not in source:
|
|
524
|
+
source["connection"] = connection_name
|
|
525
|
+
modified = True
|
|
526
|
+
patched_count += 1
|
|
527
|
+
|
|
528
|
+
if modified:
|
|
529
|
+
with open(yml_file, "w") as f:
|
|
530
|
+
# Add header comment
|
|
531
|
+
f.write("# DVT: Added connection config for multi-source support\n")
|
|
532
|
+
f.write(f"# connection: {connection_name} (auto-added by dvt migrate)\n\n")
|
|
533
|
+
yaml.dump(content, f, default_flow_style=False, sort_keys=False)
|
|
534
|
+
click.echo(f" + Patched {yml_file.name}: added connection: {connection_name}")
|
|
535
|
+
except Exception as e:
|
|
536
|
+
click.echo(f" ! Warning: Could not patch {yml_file.name}: {e}")
|
|
537
|
+
|
|
538
|
+
return patched_count
|
|
539
|
+
|
|
540
|
+
def _copy_directory(self, dir_name: str, source_name: str) -> Optional[MigrationResult]:
|
|
541
|
+
"""Copy a directory from source to target/<source_name>/.
|
|
542
|
+
|
|
543
|
+
v0.59.0a23: Now skips if target directory already exists to prevent duplicates.
|
|
544
|
+
"""
|
|
545
|
+
source_dir = self.source_path / dir_name
|
|
546
|
+
if not source_dir.exists():
|
|
547
|
+
return None
|
|
548
|
+
|
|
549
|
+
target_dir = self.project_dir / dir_name / source_name
|
|
550
|
+
|
|
551
|
+
# Check if already imported - skip to prevent duplicates
|
|
552
|
+
if target_dir.exists():
|
|
553
|
+
file_count = sum(1 for _ in target_dir.rglob("*") if _.is_file())
|
|
554
|
+
click.echo(f" Skipped {dir_name}/ (already exists with {file_count} files)")
|
|
555
|
+
return MigrationResult(True, f"{dir_name}/ (skipped)", 0)
|
|
556
|
+
|
|
557
|
+
if self.dry_run:
|
|
558
|
+
file_count = sum(1 for _ in source_dir.rglob("*") if _.is_file())
|
|
559
|
+
click.echo(f" [DRY RUN] Would copy {dir_name}/ ({file_count} files) → {dir_name}/{source_name}/")
|
|
560
|
+
return MigrationResult(True, f"{dir_name}/ (dry run)", file_count)
|
|
561
|
+
|
|
562
|
+
target_dir.parent.mkdir(parents=True, exist_ok=True)
|
|
563
|
+
shutil.copytree(source_dir, target_dir, dirs_exist_ok=True)
|
|
564
|
+
|
|
565
|
+
file_count = sum(1 for _ in target_dir.rglob("*") if _.is_file())
|
|
566
|
+
click.echo(f" Copied {dir_name}/ ({file_count} files)")
|
|
567
|
+
|
|
568
|
+
return MigrationResult(True, f"{dir_name}/", file_count)
|
|
569
|
+
|
|
570
|
+
def _import_profile_targets(self, source_name: str) -> MigrationResult:
|
|
571
|
+
"""Import profile targets from source dbt project into DVT profile.
|
|
572
|
+
|
|
573
|
+
v0.59.0a23: Now checks both ~/.dvt/profiles.yml and ~/.dbt/profiles.yml
|
|
574
|
+
for the source profile, preferring DVT profiles.
|
|
575
|
+
"""
|
|
576
|
+
try:
|
|
577
|
+
import yaml
|
|
578
|
+
except ImportError:
|
|
579
|
+
import ruamel.yaml as yaml
|
|
580
|
+
|
|
581
|
+
# Read DVT project to get profile name
|
|
582
|
+
dvt_config_path = self.project_dir / "dvt_project.yml"
|
|
583
|
+
with open(dvt_config_path) as f:
|
|
584
|
+
dvt_config = yaml.safe_load(f)
|
|
585
|
+
dvt_profile_name = dvt_config.get("profile", dvt_config.get("name"))
|
|
586
|
+
|
|
587
|
+
# Read source dbt project to get its profile name
|
|
588
|
+
source_config_path = self.source_path / "dbt_project.yml"
|
|
589
|
+
with open(source_config_path) as f:
|
|
590
|
+
source_config = yaml.safe_load(f)
|
|
591
|
+
source_profile_name = source_config.get("profile", source_config.get("name"))
|
|
592
|
+
|
|
593
|
+
# Read source profiles - check DVT profiles first, then dbt profiles
|
|
594
|
+
dvt_profiles_path = Path.home() / ".dvt" / "profiles.yml"
|
|
595
|
+
dbt_profiles_path = Path.home() / ".dbt" / "profiles.yml"
|
|
596
|
+
|
|
597
|
+
source_profile = {}
|
|
598
|
+
source_outputs = {}
|
|
599
|
+
profile_source = None
|
|
600
|
+
|
|
601
|
+
# First check ~/.dvt/profiles.yml
|
|
602
|
+
if dvt_profiles_path.exists():
|
|
603
|
+
with open(dvt_profiles_path) as f:
|
|
604
|
+
dvt_all_profiles = yaml.safe_load(f) or {}
|
|
605
|
+
if source_profile_name in dvt_all_profiles:
|
|
606
|
+
source_profile = dvt_all_profiles[source_profile_name]
|
|
607
|
+
source_outputs = source_profile.get("outputs", {})
|
|
608
|
+
profile_source = "~/.dvt/profiles.yml"
|
|
609
|
+
|
|
610
|
+
# If not found in DVT profiles, try ~/.dbt/profiles.yml
|
|
611
|
+
if not source_outputs and dbt_profiles_path.exists():
|
|
612
|
+
with open(dbt_profiles_path) as f:
|
|
613
|
+
dbt_profiles = yaml.safe_load(f) or {}
|
|
614
|
+
if source_profile_name in dbt_profiles:
|
|
615
|
+
source_profile = dbt_profiles[source_profile_name]
|
|
616
|
+
source_outputs = source_profile.get("outputs", {})
|
|
617
|
+
profile_source = "~/.dbt/profiles.yml"
|
|
618
|
+
|
|
619
|
+
if not source_outputs:
|
|
620
|
+
click.echo(f" No profile '{source_profile_name}' found in ~/.dvt/profiles.yml or ~/.dbt/profiles.yml")
|
|
621
|
+
return MigrationResult(True, "No source profile found")
|
|
622
|
+
|
|
623
|
+
click.echo(f" Found profile '{source_profile_name}' in {profile_source} with {len(source_outputs)} target(s)")
|
|
624
|
+
|
|
625
|
+
# Read current DVT profiles for destination
|
|
626
|
+
if not dvt_profiles_path.exists():
|
|
627
|
+
click.echo(" No ~/.dvt/profiles.yml found - skipping target import")
|
|
628
|
+
return MigrationResult(False, "No DVT profiles found")
|
|
629
|
+
|
|
630
|
+
with open(dvt_profiles_path) as f:
|
|
631
|
+
dvt_profiles = yaml.safe_load(f) or {}
|
|
632
|
+
|
|
633
|
+
dvt_profile = dvt_profiles.get(dvt_profile_name, {"target": "dev", "outputs": {}})
|
|
634
|
+
|
|
635
|
+
# Merge source outputs with prefix
|
|
636
|
+
merged_count = 0
|
|
637
|
+
skipped_count = 0
|
|
638
|
+
merged_names = []
|
|
639
|
+
for output_name, output_config in source_outputs.items():
|
|
640
|
+
new_name = f"{source_name}_{output_name}"
|
|
641
|
+
if new_name not in dvt_profile.get("outputs", {}):
|
|
642
|
+
dvt_profile.setdefault("outputs", {})[new_name] = output_config
|
|
643
|
+
merged_count += 1
|
|
644
|
+
merged_names.append(new_name)
|
|
645
|
+
else:
|
|
646
|
+
skipped_count += 1
|
|
647
|
+
|
|
648
|
+
dvt_profiles[dvt_profile_name] = dvt_profile
|
|
649
|
+
|
|
650
|
+
if not self.dry_run:
|
|
651
|
+
if merged_count > 0:
|
|
652
|
+
with open(dvt_profiles_path, "w") as f:
|
|
653
|
+
f.write("# DVT Profiles Configuration\n\n")
|
|
654
|
+
yaml.dump(dvt_profiles, f, default_flow_style=False)
|
|
655
|
+
click.echo(f" Merged {merged_count} target(s) into profile '{dvt_profile_name}'")
|
|
656
|
+
if merged_names:
|
|
657
|
+
for name in merged_names:
|
|
658
|
+
click.echo(f" + {name}")
|
|
659
|
+
if skipped_count > 0:
|
|
660
|
+
click.echo(f" Skipped {skipped_count} target(s) (already exist)")
|
|
661
|
+
else:
|
|
662
|
+
click.echo(f" [DRY RUN] Would merge {merged_count} target(s) into profile")
|
|
663
|
+
|
|
664
|
+
return MigrationResult(True, f"Merged {merged_count} targets", merged_count)
|
|
665
|
+
|
|
666
|
+
def _show_summary(self, results: List[MigrationResult]) -> None:
|
|
667
|
+
"""Show migration summary for Mode A."""
|
|
668
|
+
if not results:
|
|
669
|
+
return
|
|
670
|
+
|
|
671
|
+
successful = [r for r in results if r and r.success]
|
|
672
|
+
if not successful:
|
|
673
|
+
return
|
|
674
|
+
|
|
675
|
+
click.echo("")
|
|
676
|
+
click.echo("=" * 60)
|
|
677
|
+
click.echo("Migration complete!")
|
|
678
|
+
click.echo("=" * 60)
|
|
679
|
+
|
|
680
|
+
for result in successful:
|
|
681
|
+
if result:
|
|
682
|
+
click.echo(f" {result.message}")
|
|
683
|
+
|
|
684
|
+
click.echo("")
|
|
685
|
+
click.echo("Your dbt configuration has been preserved as backups.")
|
|
686
|
+
click.echo("Run 'dvt target list' to verify your connections.")
|
|
687
|
+
click.echo("=" * 60)
|
|
688
|
+
|
|
689
|
+
def _show_import_summary(self, source_name: str, results: List[MigrationResult]) -> None:
|
|
690
|
+
"""Show import summary for Mode B."""
|
|
691
|
+
if self.dry_run:
|
|
692
|
+
click.echo("")
|
|
693
|
+
click.echo("No changes made (dry run).")
|
|
694
|
+
return
|
|
695
|
+
|
|
696
|
+
# Get copied directories
|
|
697
|
+
copied_dirs = [r.message for r in results if r and r.success and "/" in r.message]
|
|
698
|
+
|
|
699
|
+
click.echo("")
|
|
700
|
+
click.echo("=" * 60)
|
|
701
|
+
click.echo("Import complete!")
|
|
702
|
+
click.echo("=" * 60)
|
|
703
|
+
click.echo(f" Project: {source_name}")
|
|
704
|
+
|
|
705
|
+
for dir_result in copied_dirs:
|
|
706
|
+
dir_name = dir_result.rstrip("/")
|
|
707
|
+
click.echo(f" {dir_name.capitalize()}: {dir_name}/{source_name}/")
|
|
708
|
+
|
|
709
|
+
click.echo("")
|
|
710
|
+
click.echo("Next steps:")
|
|
711
|
+
click.echo(f" 1. Review {copied_dirs[0].split('/')[0] if copied_dirs else 'models'}/{source_name}/ for ref() adjustments")
|
|
712
|
+
click.echo(" 2. Update dvt_project.yml if needed")
|
|
713
|
+
click.echo(" 3. Run 'dvt target list' to see all targets")
|
|
714
|
+
click.echo("=" * 60)
|