dvt-core 0.59.0a51__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (299) hide show
  1. dbt/__init__.py +7 -0
  2. dbt/_pydantic_shim.py +26 -0
  3. dbt/artifacts/__init__.py +0 -0
  4. dbt/artifacts/exceptions/__init__.py +1 -0
  5. dbt/artifacts/exceptions/schemas.py +31 -0
  6. dbt/artifacts/resources/__init__.py +116 -0
  7. dbt/artifacts/resources/base.py +67 -0
  8. dbt/artifacts/resources/types.py +93 -0
  9. dbt/artifacts/resources/v1/analysis.py +10 -0
  10. dbt/artifacts/resources/v1/catalog.py +23 -0
  11. dbt/artifacts/resources/v1/components.py +274 -0
  12. dbt/artifacts/resources/v1/config.py +277 -0
  13. dbt/artifacts/resources/v1/documentation.py +11 -0
  14. dbt/artifacts/resources/v1/exposure.py +51 -0
  15. dbt/artifacts/resources/v1/function.py +52 -0
  16. dbt/artifacts/resources/v1/generic_test.py +31 -0
  17. dbt/artifacts/resources/v1/group.py +21 -0
  18. dbt/artifacts/resources/v1/hook.py +11 -0
  19. dbt/artifacts/resources/v1/macro.py +29 -0
  20. dbt/artifacts/resources/v1/metric.py +172 -0
  21. dbt/artifacts/resources/v1/model.py +145 -0
  22. dbt/artifacts/resources/v1/owner.py +10 -0
  23. dbt/artifacts/resources/v1/saved_query.py +111 -0
  24. dbt/artifacts/resources/v1/seed.py +41 -0
  25. dbt/artifacts/resources/v1/semantic_layer_components.py +72 -0
  26. dbt/artifacts/resources/v1/semantic_model.py +314 -0
  27. dbt/artifacts/resources/v1/singular_test.py +14 -0
  28. dbt/artifacts/resources/v1/snapshot.py +91 -0
  29. dbt/artifacts/resources/v1/source_definition.py +84 -0
  30. dbt/artifacts/resources/v1/sql_operation.py +10 -0
  31. dbt/artifacts/resources/v1/unit_test_definition.py +77 -0
  32. dbt/artifacts/schemas/__init__.py +0 -0
  33. dbt/artifacts/schemas/base.py +191 -0
  34. dbt/artifacts/schemas/batch_results.py +24 -0
  35. dbt/artifacts/schemas/catalog/__init__.py +11 -0
  36. dbt/artifacts/schemas/catalog/v1/__init__.py +0 -0
  37. dbt/artifacts/schemas/catalog/v1/catalog.py +59 -0
  38. dbt/artifacts/schemas/freshness/__init__.py +1 -0
  39. dbt/artifacts/schemas/freshness/v3/__init__.py +0 -0
  40. dbt/artifacts/schemas/freshness/v3/freshness.py +158 -0
  41. dbt/artifacts/schemas/manifest/__init__.py +2 -0
  42. dbt/artifacts/schemas/manifest/v12/__init__.py +0 -0
  43. dbt/artifacts/schemas/manifest/v12/manifest.py +211 -0
  44. dbt/artifacts/schemas/results.py +147 -0
  45. dbt/artifacts/schemas/run/__init__.py +2 -0
  46. dbt/artifacts/schemas/run/v5/__init__.py +0 -0
  47. dbt/artifacts/schemas/run/v5/run.py +184 -0
  48. dbt/artifacts/schemas/upgrades/__init__.py +4 -0
  49. dbt/artifacts/schemas/upgrades/upgrade_manifest.py +174 -0
  50. dbt/artifacts/schemas/upgrades/upgrade_manifest_dbt_version.py +2 -0
  51. dbt/artifacts/utils/validation.py +153 -0
  52. dbt/cli/__init__.py +1 -0
  53. dbt/cli/context.py +17 -0
  54. dbt/cli/exceptions.py +57 -0
  55. dbt/cli/flags.py +560 -0
  56. dbt/cli/main.py +2660 -0
  57. dbt/cli/option_types.py +121 -0
  58. dbt/cli/options.py +80 -0
  59. dbt/cli/params.py +844 -0
  60. dbt/cli/requires.py +490 -0
  61. dbt/cli/resolvers.py +60 -0
  62. dbt/cli/types.py +40 -0
  63. dbt/clients/__init__.py +0 -0
  64. dbt/clients/checked_load.py +83 -0
  65. dbt/clients/git.py +164 -0
  66. dbt/clients/jinja.py +206 -0
  67. dbt/clients/jinja_static.py +245 -0
  68. dbt/clients/registry.py +192 -0
  69. dbt/clients/yaml_helper.py +68 -0
  70. dbt/compilation.py +876 -0
  71. dbt/compute/__init__.py +14 -0
  72. dbt/compute/engines/__init__.py +12 -0
  73. dbt/compute/engines/spark_engine.py +642 -0
  74. dbt/compute/federated_executor.py +1080 -0
  75. dbt/compute/filter_pushdown.py +273 -0
  76. dbt/compute/jar_provisioning.py +273 -0
  77. dbt/compute/java_compat.py +689 -0
  78. dbt/compute/jdbc_utils.py +1252 -0
  79. dbt/compute/metadata/__init__.py +63 -0
  80. dbt/compute/metadata/adapters_registry.py +370 -0
  81. dbt/compute/metadata/catalog_store.py +1036 -0
  82. dbt/compute/metadata/registry.py +674 -0
  83. dbt/compute/metadata/store.py +1020 -0
  84. dbt/compute/smart_selector.py +377 -0
  85. dbt/compute/spark_logger.py +272 -0
  86. dbt/compute/strategies/__init__.py +55 -0
  87. dbt/compute/strategies/base.py +165 -0
  88. dbt/compute/strategies/dataproc.py +207 -0
  89. dbt/compute/strategies/emr.py +203 -0
  90. dbt/compute/strategies/local.py +472 -0
  91. dbt/compute/strategies/standalone.py +262 -0
  92. dbt/config/__init__.py +4 -0
  93. dbt/config/catalogs.py +94 -0
  94. dbt/config/compute.py +513 -0
  95. dbt/config/dvt_profile.py +408 -0
  96. dbt/config/profile.py +422 -0
  97. dbt/config/project.py +888 -0
  98. dbt/config/project_utils.py +48 -0
  99. dbt/config/renderer.py +231 -0
  100. dbt/config/runtime.py +564 -0
  101. dbt/config/selectors.py +208 -0
  102. dbt/config/utils.py +77 -0
  103. dbt/constants.py +28 -0
  104. dbt/context/__init__.py +0 -0
  105. dbt/context/base.py +745 -0
  106. dbt/context/configured.py +135 -0
  107. dbt/context/context_config.py +382 -0
  108. dbt/context/docs.py +82 -0
  109. dbt/context/exceptions_jinja.py +178 -0
  110. dbt/context/macro_resolver.py +195 -0
  111. dbt/context/macros.py +171 -0
  112. dbt/context/manifest.py +72 -0
  113. dbt/context/providers.py +2249 -0
  114. dbt/context/query_header.py +13 -0
  115. dbt/context/secret.py +58 -0
  116. dbt/context/target.py +74 -0
  117. dbt/contracts/__init__.py +0 -0
  118. dbt/contracts/files.py +413 -0
  119. dbt/contracts/graph/__init__.py +0 -0
  120. dbt/contracts/graph/manifest.py +1904 -0
  121. dbt/contracts/graph/metrics.py +97 -0
  122. dbt/contracts/graph/model_config.py +70 -0
  123. dbt/contracts/graph/node_args.py +42 -0
  124. dbt/contracts/graph/nodes.py +1806 -0
  125. dbt/contracts/graph/semantic_manifest.py +232 -0
  126. dbt/contracts/graph/unparsed.py +811 -0
  127. dbt/contracts/project.py +419 -0
  128. dbt/contracts/results.py +53 -0
  129. dbt/contracts/selection.py +23 -0
  130. dbt/contracts/sql.py +85 -0
  131. dbt/contracts/state.py +68 -0
  132. dbt/contracts/util.py +46 -0
  133. dbt/deprecations.py +348 -0
  134. dbt/deps/__init__.py +0 -0
  135. dbt/deps/base.py +152 -0
  136. dbt/deps/git.py +195 -0
  137. dbt/deps/local.py +79 -0
  138. dbt/deps/registry.py +130 -0
  139. dbt/deps/resolver.py +149 -0
  140. dbt/deps/tarball.py +120 -0
  141. dbt/docs/source/_ext/dbt_click.py +119 -0
  142. dbt/docs/source/conf.py +32 -0
  143. dbt/env_vars.py +64 -0
  144. dbt/event_time/event_time.py +40 -0
  145. dbt/event_time/sample_window.py +60 -0
  146. dbt/events/__init__.py +15 -0
  147. dbt/events/base_types.py +36 -0
  148. dbt/events/core_types_pb2.py +2 -0
  149. dbt/events/logging.py +108 -0
  150. dbt/events/types.py +2516 -0
  151. dbt/exceptions.py +1486 -0
  152. dbt/flags.py +89 -0
  153. dbt/graph/__init__.py +11 -0
  154. dbt/graph/cli.py +249 -0
  155. dbt/graph/graph.py +172 -0
  156. dbt/graph/queue.py +214 -0
  157. dbt/graph/selector.py +374 -0
  158. dbt/graph/selector_methods.py +975 -0
  159. dbt/graph/selector_spec.py +222 -0
  160. dbt/graph/thread_pool.py +18 -0
  161. dbt/hooks.py +21 -0
  162. dbt/include/README.md +49 -0
  163. dbt/include/__init__.py +3 -0
  164. dbt/include/data/adapters_registry.duckdb +0 -0
  165. dbt/include/data/build_comprehensive_registry.py +1254 -0
  166. dbt/include/data/build_registry.py +242 -0
  167. dbt/include/data/csv/adapter_queries.csv +33 -0
  168. dbt/include/data/csv/syntax_rules.csv +9 -0
  169. dbt/include/data/csv/type_mappings_bigquery.csv +28 -0
  170. dbt/include/data/csv/type_mappings_databricks.csv +30 -0
  171. dbt/include/data/csv/type_mappings_mysql.csv +40 -0
  172. dbt/include/data/csv/type_mappings_oracle.csv +30 -0
  173. dbt/include/data/csv/type_mappings_postgres.csv +56 -0
  174. dbt/include/data/csv/type_mappings_redshift.csv +33 -0
  175. dbt/include/data/csv/type_mappings_snowflake.csv +38 -0
  176. dbt/include/data/csv/type_mappings_sqlserver.csv +35 -0
  177. dbt/include/dvt_starter_project/README.md +15 -0
  178. dbt/include/dvt_starter_project/__init__.py +3 -0
  179. dbt/include/dvt_starter_project/analyses/PLACEHOLDER +0 -0
  180. dbt/include/dvt_starter_project/dvt_project.yml +39 -0
  181. dbt/include/dvt_starter_project/logs/PLACEHOLDER +0 -0
  182. dbt/include/dvt_starter_project/macros/PLACEHOLDER +0 -0
  183. dbt/include/dvt_starter_project/models/example/my_first_dbt_model.sql +27 -0
  184. dbt/include/dvt_starter_project/models/example/my_second_dbt_model.sql +6 -0
  185. dbt/include/dvt_starter_project/models/example/schema.yml +21 -0
  186. dbt/include/dvt_starter_project/seeds/PLACEHOLDER +0 -0
  187. dbt/include/dvt_starter_project/snapshots/PLACEHOLDER +0 -0
  188. dbt/include/dvt_starter_project/tests/PLACEHOLDER +0 -0
  189. dbt/internal_deprecations.py +26 -0
  190. dbt/jsonschemas/__init__.py +3 -0
  191. dbt/jsonschemas/jsonschemas.py +309 -0
  192. dbt/jsonschemas/project/0.0.110.json +4717 -0
  193. dbt/jsonschemas/project/0.0.85.json +2015 -0
  194. dbt/jsonschemas/resources/0.0.110.json +2636 -0
  195. dbt/jsonschemas/resources/0.0.85.json +2536 -0
  196. dbt/jsonschemas/resources/latest.json +6773 -0
  197. dbt/links.py +4 -0
  198. dbt/materializations/__init__.py +0 -0
  199. dbt/materializations/incremental/__init__.py +0 -0
  200. dbt/materializations/incremental/microbatch.py +236 -0
  201. dbt/mp_context.py +8 -0
  202. dbt/node_types.py +37 -0
  203. dbt/parser/__init__.py +23 -0
  204. dbt/parser/analysis.py +21 -0
  205. dbt/parser/base.py +548 -0
  206. dbt/parser/common.py +266 -0
  207. dbt/parser/docs.py +52 -0
  208. dbt/parser/fixtures.py +51 -0
  209. dbt/parser/functions.py +30 -0
  210. dbt/parser/generic_test.py +100 -0
  211. dbt/parser/generic_test_builders.py +333 -0
  212. dbt/parser/hooks.py +122 -0
  213. dbt/parser/macros.py +137 -0
  214. dbt/parser/manifest.py +2208 -0
  215. dbt/parser/models.py +573 -0
  216. dbt/parser/partial.py +1178 -0
  217. dbt/parser/read_files.py +445 -0
  218. dbt/parser/schema_generic_tests.py +422 -0
  219. dbt/parser/schema_renderer.py +111 -0
  220. dbt/parser/schema_yaml_readers.py +935 -0
  221. dbt/parser/schemas.py +1466 -0
  222. dbt/parser/search.py +149 -0
  223. dbt/parser/seeds.py +28 -0
  224. dbt/parser/singular_test.py +20 -0
  225. dbt/parser/snapshots.py +44 -0
  226. dbt/parser/sources.py +558 -0
  227. dbt/parser/sql.py +62 -0
  228. dbt/parser/unit_tests.py +621 -0
  229. dbt/plugins/__init__.py +20 -0
  230. dbt/plugins/contracts.py +9 -0
  231. dbt/plugins/exceptions.py +2 -0
  232. dbt/plugins/manager.py +163 -0
  233. dbt/plugins/manifest.py +21 -0
  234. dbt/profiler.py +20 -0
  235. dbt/py.typed +1 -0
  236. dbt/query_analyzer.py +410 -0
  237. dbt/runners/__init__.py +2 -0
  238. dbt/runners/exposure_runner.py +7 -0
  239. dbt/runners/no_op_runner.py +45 -0
  240. dbt/runners/saved_query_runner.py +7 -0
  241. dbt/selected_resources.py +8 -0
  242. dbt/task/__init__.py +0 -0
  243. dbt/task/base.py +506 -0
  244. dbt/task/build.py +197 -0
  245. dbt/task/clean.py +56 -0
  246. dbt/task/clone.py +161 -0
  247. dbt/task/compile.py +150 -0
  248. dbt/task/compute.py +458 -0
  249. dbt/task/debug.py +513 -0
  250. dbt/task/deps.py +280 -0
  251. dbt/task/docs/__init__.py +3 -0
  252. dbt/task/docs/api/__init__.py +23 -0
  253. dbt/task/docs/api/catalog.py +204 -0
  254. dbt/task/docs/api/lineage.py +234 -0
  255. dbt/task/docs/api/profile.py +204 -0
  256. dbt/task/docs/api/spark.py +186 -0
  257. dbt/task/docs/generate.py +1002 -0
  258. dbt/task/docs/index.html +250 -0
  259. dbt/task/docs/serve.py +174 -0
  260. dbt/task/dvt_output.py +509 -0
  261. dbt/task/dvt_run.py +282 -0
  262. dbt/task/dvt_seed.py +806 -0
  263. dbt/task/freshness.py +322 -0
  264. dbt/task/function.py +121 -0
  265. dbt/task/group_lookup.py +46 -0
  266. dbt/task/init.py +1022 -0
  267. dbt/task/java.py +316 -0
  268. dbt/task/list.py +236 -0
  269. dbt/task/metadata.py +804 -0
  270. dbt/task/migrate.py +714 -0
  271. dbt/task/printer.py +175 -0
  272. dbt/task/profile.py +1489 -0
  273. dbt/task/profile_serve.py +662 -0
  274. dbt/task/retract.py +441 -0
  275. dbt/task/retry.py +175 -0
  276. dbt/task/run.py +1647 -0
  277. dbt/task/run_operation.py +141 -0
  278. dbt/task/runnable.py +758 -0
  279. dbt/task/seed.py +103 -0
  280. dbt/task/show.py +149 -0
  281. dbt/task/snapshot.py +56 -0
  282. dbt/task/spark.py +414 -0
  283. dbt/task/sql.py +110 -0
  284. dbt/task/target_sync.py +814 -0
  285. dbt/task/test.py +464 -0
  286. dbt/tests/fixtures/__init__.py +1 -0
  287. dbt/tests/fixtures/project.py +620 -0
  288. dbt/tests/util.py +651 -0
  289. dbt/tracking.py +529 -0
  290. dbt/utils/__init__.py +3 -0
  291. dbt/utils/artifact_upload.py +151 -0
  292. dbt/utils/utils.py +408 -0
  293. dbt/version.py +271 -0
  294. dvt_cli/__init__.py +158 -0
  295. dvt_core-0.59.0a51.dist-info/METADATA +288 -0
  296. dvt_core-0.59.0a51.dist-info/RECORD +299 -0
  297. dvt_core-0.59.0a51.dist-info/WHEEL +5 -0
  298. dvt_core-0.59.0a51.dist-info/entry_points.txt +2 -0
  299. dvt_core-0.59.0a51.dist-info/top_level.txt +2 -0
dbt/task/migrate.py ADDED
@@ -0,0 +1,714 @@
1
+ """
2
+ DVT Migration Task - Migrate from dbt to DVT configuration.
3
+
4
+ Two modes:
5
+ - Mode A: Convert dbt project to DVT (not in DVT project)
6
+ - Mode B: Import dbt project INTO DVT project (in DVT project)
7
+ """
8
+ from pathlib import Path
9
+ from dataclasses import dataclass
10
+ from typing import List, Optional
11
+ import shutil
12
+ import click
13
+
14
+
15
+ @dataclass
16
+ class MigrationResult:
17
+ """Result of a migration operation."""
18
+
19
+ success: bool
20
+ message: str
21
+ files_copied: int = 0
22
+
23
+
24
+ class MigrateTask:
25
+ """
26
+ DVT Migration Task.
27
+
28
+ Handles two modes:
29
+ - Mode A: Convert dbt project to DVT (when not in DVT project)
30
+ - Mode B: Import dbt project INTO DVT project (when in DVT project)
31
+ """
32
+
33
+ def __init__(
34
+ self,
35
+ source_path: Optional[str] = None,
36
+ profiles_only: bool = False,
37
+ project_only: bool = False,
38
+ full: bool = False,
39
+ dry_run: bool = False,
40
+ project_dir: Optional[str] = None,
41
+ ):
42
+ self.source_path = Path(source_path) if source_path else None
43
+ self.profiles_only = profiles_only
44
+ self.project_only = project_only
45
+ self.full = full
46
+ self.dry_run = dry_run
47
+ self.project_dir = Path(project_dir) if project_dir else Path.cwd()
48
+
49
+ def run(self) -> bool:
50
+ """Execute migration based on context and flags."""
51
+ # Detect which mode we're in
52
+ in_dvt_project = (self.project_dir / "dvt_project.yml").exists()
53
+
54
+ if in_dvt_project and self.source_path:
55
+ # Mode B: Import dbt project INTO current DVT project
56
+ return self._import_dbt_project()
57
+ elif in_dvt_project and not self.source_path:
58
+ # Inside DVT project but no path - show help
59
+ click.echo("You are inside a DVT project.")
60
+ click.echo("To import a dbt project, provide the path:")
61
+ click.echo(" dvt migrate /path/to/dbt_project")
62
+ return True
63
+ else:
64
+ # Mode A: Convert current dbt project to DVT
65
+ return self._convert_to_dvt()
66
+
67
+ def _convert_to_dvt(self) -> bool:
68
+ """Mode A: Convert dbt project in current directory to DVT."""
69
+ results = []
70
+
71
+ if self.profiles_only:
72
+ results.append(self._migrate_profiles())
73
+ elif self.project_only:
74
+ results.append(self._migrate_project_file())
75
+ elif self.full:
76
+ results.append(self._migrate_profiles())
77
+ results.append(self._migrate_project_file())
78
+ else:
79
+ results = self._auto_detect_and_migrate()
80
+
81
+ self._show_summary(results)
82
+ return all(r.success for r in results if r)
83
+
84
+ def _auto_detect_and_migrate(self) -> List[MigrationResult]:
85
+ """Auto-detect what needs to be migrated."""
86
+ results = []
87
+
88
+ click.echo("Detecting dbt configuration...")
89
+
90
+ dbt_profiles_path = Path.home() / ".dbt" / "profiles.yml"
91
+ dbt_project_path = self.project_dir / "dbt_project.yml"
92
+
93
+ has_dbt_profiles = dbt_profiles_path.exists()
94
+ has_dbt_project = dbt_project_path.exists()
95
+
96
+ if has_dbt_profiles:
97
+ click.echo(f" Found ~/.dbt/profiles.yml")
98
+ if has_dbt_project:
99
+ click.echo(f" Found dbt_project.yml")
100
+
101
+ if not has_dbt_profiles and not has_dbt_project:
102
+ click.echo("\nNo dbt configuration found - nothing to migrate.")
103
+ click.echo("Use 'dvt init' to create a new DVT project.")
104
+ return [MigrationResult(True, "Nothing to migrate")]
105
+
106
+ click.echo("")
107
+
108
+ # Migrate profiles if found
109
+ if has_dbt_profiles:
110
+ click.echo("Migrating profiles...")
111
+ results.append(self._migrate_profiles())
112
+
113
+ # Migrate project if found
114
+ if has_dbt_project:
115
+ click.echo("\nMigrating project...")
116
+ results.append(self._migrate_project_file())
117
+
118
+ return results
119
+
120
+ def _migrate_profiles(self) -> MigrationResult:
121
+ """Migrate project-specific profile from ~/.dbt/profiles.yml to ~/.dvt/profiles.yml.
122
+
123
+ v0.59.0a2: Changed to be surgical - only copies the profile matching the current
124
+ project, not all profiles from ~/.dbt/profiles.yml.
125
+
126
+ v0.59.0a19: If profile not found in ~/.dbt/profiles.yml, creates a starter
127
+ DuckDB profile (like dvt init does).
128
+ """
129
+ try:
130
+ import yaml
131
+ except ImportError:
132
+ import ruamel.yaml as yaml
133
+
134
+ old_path = Path.home() / ".dbt" / "profiles.yml"
135
+ new_path = Path.home() / ".dvt" / "profiles.yml"
136
+
137
+ # Get the profile name from dbt_project.yml
138
+ dbt_project_path = self.project_dir / "dbt_project.yml"
139
+ if dbt_project_path.exists():
140
+ with open(dbt_project_path) as f:
141
+ project_config = yaml.safe_load(f) or {}
142
+ # Use 'profile' field, fall back to 'name', then directory name
143
+ profile_name = project_config.get("profile", project_config.get("name", self.project_dir.name))
144
+ else:
145
+ # No dbt_project.yml, use directory name
146
+ profile_name = self.project_dir.name
147
+
148
+ # Check if profile exists in ~/.dbt/profiles.yml
149
+ profile_to_migrate = None
150
+ if old_path.exists():
151
+ with open(old_path) as f:
152
+ old_profiles = yaml.safe_load(f) or {}
153
+ if profile_name in old_profiles:
154
+ profile_to_migrate = {profile_name: old_profiles[profile_name]}
155
+
156
+ # v0.59.0a19: If profile not found, create a starter DuckDB profile
157
+ if profile_to_migrate is None:
158
+ click.echo(f" No profile '{profile_name}' found in ~/.dbt/profiles.yml")
159
+ click.echo(f" Creating starter DuckDB profile...")
160
+
161
+ # Create .dvt directory and default.duckdb
162
+ dvt_dir = self.project_dir / ".dvt"
163
+ dvt_dir.mkdir(parents=True, exist_ok=True)
164
+ duckdb_path = dvt_dir / "default.duckdb"
165
+
166
+ # Create empty DuckDB file if it doesn't exist
167
+ if not duckdb_path.exists():
168
+ try:
169
+ import duckdb
170
+ conn = duckdb.connect(str(duckdb_path))
171
+ conn.close()
172
+ except Exception:
173
+ # Just create empty file if duckdb import fails
174
+ duckdb_path.touch()
175
+
176
+ # Create starter profile
177
+ profile_to_migrate = {
178
+ profile_name: {
179
+ "target": "dev",
180
+ "outputs": {
181
+ "dev": {
182
+ "type": "duckdb",
183
+ "path": str(duckdb_path.resolve()),
184
+ "threads": 4,
185
+ }
186
+ }
187
+ }
188
+ }
189
+
190
+ # Merge with existing DVT profiles if any
191
+ if new_path.exists():
192
+ with open(new_path) as f:
193
+ existing = yaml.safe_load(f) or {}
194
+ # DVT profiles take precedence (don't overwrite)
195
+ if profile_name not in existing:
196
+ existing[profile_name] = profile_to_migrate[profile_name]
197
+ merged = True
198
+ else:
199
+ click.echo(f" Profile '{profile_name}' already exists in ~/.dvt/profiles.yml - skipping")
200
+ return MigrationResult(True, f"Profile '{profile_name}' already exists")
201
+ profiles = existing
202
+ else:
203
+ profiles = profile_to_migrate
204
+ merged = False
205
+
206
+ # Write to DVT profiles
207
+ if not self.dry_run:
208
+ new_path.parent.mkdir(parents=True, exist_ok=True)
209
+ with open(new_path, "w") as f:
210
+ f.write("# DVT Profiles Configuration\n\n")
211
+ yaml.dump(profiles, f, default_flow_style=False)
212
+
213
+ action = "Merged" if merged else "Migrated"
214
+ click.echo(f" {action} profile '{profile_name}' to ~/.dvt/profiles.yml")
215
+ else:
216
+ click.echo(f" [DRY RUN] Would migrate profile '{profile_name}'")
217
+
218
+ return MigrationResult(True, f"Migrated profile '{profile_name}'")
219
+
220
+ def _migrate_project_file(self) -> MigrationResult:
221
+ """Create dvt_project.yml from fresh template (like dbt init) + DVT additions.
222
+
223
+ v0.59.0a2: Creates fresh template matching dbt init format, plus DVT-specific
224
+ flatfile-paths config. Creates flatfiles/ directory. User can copy project-specific
225
+ configs from the backed-up dbt_project.yml.bak if needed.
226
+ """
227
+ try:
228
+ import yaml
229
+ except ImportError:
230
+ import ruamel.yaml as yaml
231
+
232
+ old_path = self.project_dir / "dbt_project.yml"
233
+ new_path = self.project_dir / "dvt_project.yml"
234
+ backup_path = self.project_dir / "dbt_project.yml.bak"
235
+
236
+ if not old_path.exists():
237
+ click.echo(" No dbt_project.yml found")
238
+ return MigrationResult(False, "No dbt_project.yml found")
239
+
240
+ if new_path.exists():
241
+ click.echo(" dvt_project.yml already exists - skipping")
242
+ return MigrationResult(True, "dvt_project.yml already exists")
243
+
244
+ # Get project name and profile from dbt_project.yml
245
+ project_name = self.project_dir.name
246
+ profile_name = project_name # Default to project name
247
+ with open(old_path, "r") as f:
248
+ try:
249
+ config = yaml.safe_load(f)
250
+ if config:
251
+ if "name" in config:
252
+ project_name = config["name"]
253
+ if "profile" in config:
254
+ profile_name = config["profile"]
255
+ except Exception:
256
+ pass
257
+
258
+ if not self.dry_run:
259
+ # 1. Backup dbt_project.yml
260
+ if backup_path.exists():
261
+ # Add timestamp if backup already exists
262
+ import time
263
+
264
+ timestamp = time.strftime("%Y%m%d_%H%M%S")
265
+ backup_path = self.project_dir / f"dbt_project.yml.{timestamp}.bak"
266
+ shutil.copy2(old_path, backup_path)
267
+ click.echo(f" Backed up dbt_project.yml → {backup_path.name}")
268
+
269
+ # 2. Create fresh dvt_project.yml from standard dbt init template + DVT additions
270
+ dvt_project_content = f"""name: '{project_name}'
271
+ version: '1.0.0'
272
+
273
+ # This setting configures which "profile" dbt uses for this project.
274
+ profile: '{profile_name}'
275
+
276
+ # These configurations specify where dbt should look for different types of files.
277
+ # The `model-paths` config, for example, states that models in this project can be
278
+ # found in the "models/" directory. You probably won't need to change these!
279
+ model-paths: ["models"]
280
+ analysis-paths: ["analyses"]
281
+ test-paths: ["tests"]
282
+ seed-paths: ["seeds"]
283
+ macro-paths: ["macros"]
284
+ snapshot-paths: ["snapshots"]
285
+
286
+ # DVT-specific: Path for flat files (CSV, Parquet, etc.) for local data ingestion
287
+ flatfile-paths: ["flatfiles"]
288
+
289
+ clean-targets:
290
+ - "target"
291
+ - "dbt_packages"
292
+
293
+ # Configuring models
294
+ # Full documentation: https://docs.getdbt.com/docs/configuring-models
295
+
296
+ # In this example config, we tell dbt to build all models in the example/
297
+ # directory as views. These settings can be overridden in the individual model
298
+ # files using the `{{ config(...) }}` macro.
299
+ models:
300
+ {project_name}:
301
+ # Config indicated by + and applies to all files under models/example/
302
+ +materialized: view
303
+ """
304
+ with open(new_path, "w") as f:
305
+ f.write(dvt_project_content)
306
+ click.echo(" Created dvt_project.yml")
307
+
308
+ # 3. Create flatfiles/ directory
309
+ flatfiles_dir = self.project_dir / "flatfiles"
310
+ flatfiles_dir.mkdir(exist_ok=True)
311
+ gitignore_path = flatfiles_dir / ".gitignore"
312
+ if not gitignore_path.exists():
313
+ with open(gitignore_path, "w") as f:
314
+ f.write("# Ignore all flat files (CSV, Parquet, etc.)\n*\n!.gitignore\n")
315
+ click.echo(" Created flatfiles/ directory")
316
+
317
+ # 4. Create .dvt/ directory with computes.yml and metadata_store.duckdb
318
+ dvt_dir = self.project_dir / ".dvt"
319
+ dvt_dir.mkdir(exist_ok=True)
320
+
321
+ # Create computes.yml
322
+ computes_path = dvt_dir / "computes.yml"
323
+ if not computes_path.exists():
324
+ computes_content = """# DVT Compute Configuration
325
+ # See: https://github.com/dvt-core/dvt-core#compute-configuration
326
+
327
+ # Default compute engine for federation queries
328
+ target_compute: spark-local
329
+
330
+ # Available compute engines
331
+ computes:
332
+ spark-local:
333
+ type: local
334
+ app_name: DVT-Spark
335
+ config:
336
+ spark.sql.execution.arrow.pyspark.enabled: "true"
337
+ spark.sql.adaptive.enabled: "true"
338
+ """
339
+ with open(computes_path, "w") as f:
340
+ f.write(computes_content)
341
+ click.echo(" Created .dvt/computes.yml")
342
+
343
+ # Create metadata_store.duckdb
344
+ metadata_store_path = dvt_dir / "metadata_store.duckdb"
345
+ if not metadata_store_path.exists():
346
+ try:
347
+ import duckdb
348
+ conn = duckdb.connect(str(metadata_store_path))
349
+ conn.close()
350
+ except Exception:
351
+ metadata_store_path.touch()
352
+ click.echo(" Created .dvt/metadata_store.duckdb")
353
+ else:
354
+ click.echo(" [DRY RUN] Would backup dbt_project.yml")
355
+ click.echo(" [DRY RUN] Would create dvt_project.yml")
356
+ click.echo(" [DRY RUN] Would create flatfiles/ directory")
357
+
358
+ return MigrationResult(True, "Project migrated")
359
+
360
+ def _import_dbt_project(self) -> bool:
361
+ """Mode B: Import dbt/DVT project INTO current DVT project.
362
+
363
+ v0.59.0a23: Now supports importing both dbt and DVT projects.
364
+ Checks for dvt_project.yml first, then dbt_project.yml.
365
+
366
+ v0.59.0a24: For dbt projects ONLY, patches sources.yml files to add
367
+ DVT's required connection: config. DVT projects are expected to
368
+ already have this config (user's responsibility).
369
+ """
370
+ try:
371
+ import yaml
372
+ except ImportError:
373
+ import ruamel.yaml as yaml
374
+
375
+ # Verify source is a dbt or DVT project (check DVT first)
376
+ dvt_config_path = self.source_path / "dvt_project.yml"
377
+ dbt_config_path = self.source_path / "dbt_project.yml"
378
+
379
+ if dvt_config_path.exists():
380
+ source_config_path = dvt_config_path
381
+ project_type = "DVT"
382
+ elif dbt_config_path.exists():
383
+ source_config_path = dbt_config_path
384
+ project_type = "dbt"
385
+ else:
386
+ click.echo(f"Error: No dvt_project.yml or dbt_project.yml found in {self.source_path}")
387
+ return False
388
+
389
+ # Get source project name and profile from project config
390
+ with open(source_config_path) as f:
391
+ project_config = yaml.safe_load(f)
392
+ source_name = project_config.get("name", self.source_path.name)
393
+ source_profile_name = project_config.get("profile", source_name)
394
+
395
+ if self.dry_run:
396
+ click.echo(f"Importing {project_type} project '{source_name}' (DRY RUN)...")
397
+ else:
398
+ click.echo(f"Importing {project_type} project '{source_name}' into current DVT project...")
399
+
400
+ click.echo("")
401
+ results = []
402
+
403
+ # Copy directories
404
+ dirs_to_copy = ["models", "seeds", "tests", "macros", "snapshots", "analyses"]
405
+ models_copied = False
406
+ for dir_name in dirs_to_copy:
407
+ result = self._copy_directory(dir_name, source_name)
408
+ if result:
409
+ results.append(result)
410
+ if dir_name == "models" and result.success and result.files_copied > 0:
411
+ models_copied = True
412
+
413
+ # v0.59.0a24: Patch sources.yml for dbt projects ONLY
414
+ # DVT projects should already have connection: config - user's responsibility
415
+ if project_type == "dbt" and models_copied:
416
+ # Get the default target from source profile to build connection name
417
+ default_target = self._get_source_default_target(source_profile_name)
418
+ connection_name = f"{source_name}_{default_target}"
419
+ patched = self._patch_sources_connection(
420
+ self.project_dir / "models" / source_name,
421
+ connection_name
422
+ )
423
+ if patched > 0:
424
+ click.echo(f" Patched {patched} source(s) with connection: {connection_name}")
425
+ elif project_type == "DVT":
426
+ # Warn DVT project users to verify connection config
427
+ models_dir = self.project_dir / "models" / source_name
428
+ if models_dir.exists():
429
+ sources_files = list(models_dir.rglob("*sources*.yml"))
430
+ if sources_files:
431
+ click.echo(f" Note: DVT project detected - verify sources.yml has connection: config")
432
+
433
+ # Merge profile targets
434
+ profile_result = self._import_profile_targets(source_name)
435
+ results.append(profile_result)
436
+
437
+ self._show_import_summary(source_name, results)
438
+ return all(r.success for r in results if r)
439
+
440
+ def _get_source_default_target(self, profile_name: str) -> str:
441
+ """Get the default target from a source profile.
442
+
443
+ Checks ~/.dvt/profiles.yml first, then ~/.dbt/profiles.yml.
444
+ Returns 'default' if profile not found.
445
+ """
446
+ try:
447
+ import yaml
448
+ except ImportError:
449
+ import ruamel.yaml as yaml
450
+
451
+ # Check DVT profiles first
452
+ dvt_profiles_path = Path.home() / ".dvt" / "profiles.yml"
453
+ if dvt_profiles_path.exists():
454
+ with open(dvt_profiles_path) as f:
455
+ profiles = yaml.safe_load(f) or {}
456
+ if profile_name in profiles:
457
+ return profiles[profile_name].get("target", "default")
458
+
459
+ # Then check dbt profiles
460
+ dbt_profiles_path = Path.home() / ".dbt" / "profiles.yml"
461
+ if dbt_profiles_path.exists():
462
+ with open(dbt_profiles_path) as f:
463
+ profiles = yaml.safe_load(f) or {}
464
+ if profile_name in profiles:
465
+ return profiles[profile_name].get("target", "default")
466
+
467
+ return "default"
468
+
469
+ def _patch_sources_connection(self, target_dir: Path, connection_name: str) -> int:
470
+ """Add connection: config to all .yml files that contain sources definitions.
471
+
472
+ ONLY called for dbt projects (not DVT projects).
473
+ DVT requires sources to specify which connection they read from.
474
+ For imported dbt projects, default to the imported project's connection.
475
+
476
+ v0.59.0a24: New method for surgical sources.yml patching.
477
+ Note: Sources can be defined in ANY .yml file (not just *sources*.yml).
478
+ A file contains sources if it has a top-level 'sources:' key.
479
+
480
+ Args:
481
+ target_dir: Directory containing the copied models
482
+ connection_name: Connection name to add (e.g., "Cocacola_DWH_postgres")
483
+
484
+ Returns:
485
+ Count of patched sources.
486
+ """
487
+ try:
488
+ import yaml
489
+ except ImportError:
490
+ import ruamel.yaml as yaml
491
+
492
+ if self.dry_run:
493
+ # Count what would be patched by scanning ALL .yml files for sources: key
494
+ count = 0
495
+ for yml_file in target_dir.rglob("*.yml"):
496
+ try:
497
+ with open(yml_file) as f:
498
+ content = yaml.safe_load(f) or {}
499
+ if content.get("sources"):
500
+ for source in content.get("sources", []):
501
+ if "connection" not in source:
502
+ count += 1
503
+ except Exception:
504
+ pass
505
+ if count > 0:
506
+ click.echo(f" [DRY RUN] Would patch {count} source(s) with connection: {connection_name}")
507
+ return count
508
+
509
+ patched_count = 0
510
+
511
+ # Scan ALL .yml files for sources: key (sources can be in any yml file)
512
+ for yml_file in target_dir.rglob("*.yml"):
513
+ try:
514
+ with open(yml_file) as f:
515
+ content = yaml.safe_load(f) or {}
516
+
517
+ # Skip if no sources defined in this file
518
+ if not content.get("sources"):
519
+ continue
520
+
521
+ modified = False
522
+ for source in content.get("sources", []):
523
+ if "connection" not in source:
524
+ source["connection"] = connection_name
525
+ modified = True
526
+ patched_count += 1
527
+
528
+ if modified:
529
+ with open(yml_file, "w") as f:
530
+ # Add header comment
531
+ f.write("# DVT: Added connection config for multi-source support\n")
532
+ f.write(f"# connection: {connection_name} (auto-added by dvt migrate)\n\n")
533
+ yaml.dump(content, f, default_flow_style=False, sort_keys=False)
534
+ click.echo(f" + Patched {yml_file.name}: added connection: {connection_name}")
535
+ except Exception as e:
536
+ click.echo(f" ! Warning: Could not patch {yml_file.name}: {e}")
537
+
538
+ return patched_count
539
+
540
+ def _copy_directory(self, dir_name: str, source_name: str) -> Optional[MigrationResult]:
541
+ """Copy a directory from source to target/<source_name>/.
542
+
543
+ v0.59.0a23: Now skips if target directory already exists to prevent duplicates.
544
+ """
545
+ source_dir = self.source_path / dir_name
546
+ if not source_dir.exists():
547
+ return None
548
+
549
+ target_dir = self.project_dir / dir_name / source_name
550
+
551
+ # Check if already imported - skip to prevent duplicates
552
+ if target_dir.exists():
553
+ file_count = sum(1 for _ in target_dir.rglob("*") if _.is_file())
554
+ click.echo(f" Skipped {dir_name}/ (already exists with {file_count} files)")
555
+ return MigrationResult(True, f"{dir_name}/ (skipped)", 0)
556
+
557
+ if self.dry_run:
558
+ file_count = sum(1 for _ in source_dir.rglob("*") if _.is_file())
559
+ click.echo(f" [DRY RUN] Would copy {dir_name}/ ({file_count} files) → {dir_name}/{source_name}/")
560
+ return MigrationResult(True, f"{dir_name}/ (dry run)", file_count)
561
+
562
+ target_dir.parent.mkdir(parents=True, exist_ok=True)
563
+ shutil.copytree(source_dir, target_dir, dirs_exist_ok=True)
564
+
565
+ file_count = sum(1 for _ in target_dir.rglob("*") if _.is_file())
566
+ click.echo(f" Copied {dir_name}/ ({file_count} files)")
567
+
568
+ return MigrationResult(True, f"{dir_name}/", file_count)
569
+
570
+ def _import_profile_targets(self, source_name: str) -> MigrationResult:
571
+ """Import profile targets from source dbt project into DVT profile.
572
+
573
+ v0.59.0a23: Now checks both ~/.dvt/profiles.yml and ~/.dbt/profiles.yml
574
+ for the source profile, preferring DVT profiles.
575
+ """
576
+ try:
577
+ import yaml
578
+ except ImportError:
579
+ import ruamel.yaml as yaml
580
+
581
+ # Read DVT project to get profile name
582
+ dvt_config_path = self.project_dir / "dvt_project.yml"
583
+ with open(dvt_config_path) as f:
584
+ dvt_config = yaml.safe_load(f)
585
+ dvt_profile_name = dvt_config.get("profile", dvt_config.get("name"))
586
+
587
+ # Read source dbt project to get its profile name
588
+ source_config_path = self.source_path / "dbt_project.yml"
589
+ with open(source_config_path) as f:
590
+ source_config = yaml.safe_load(f)
591
+ source_profile_name = source_config.get("profile", source_config.get("name"))
592
+
593
+ # Read source profiles - check DVT profiles first, then dbt profiles
594
+ dvt_profiles_path = Path.home() / ".dvt" / "profiles.yml"
595
+ dbt_profiles_path = Path.home() / ".dbt" / "profiles.yml"
596
+
597
+ source_profile = {}
598
+ source_outputs = {}
599
+ profile_source = None
600
+
601
+ # First check ~/.dvt/profiles.yml
602
+ if dvt_profiles_path.exists():
603
+ with open(dvt_profiles_path) as f:
604
+ dvt_all_profiles = yaml.safe_load(f) or {}
605
+ if source_profile_name in dvt_all_profiles:
606
+ source_profile = dvt_all_profiles[source_profile_name]
607
+ source_outputs = source_profile.get("outputs", {})
608
+ profile_source = "~/.dvt/profiles.yml"
609
+
610
+ # If not found in DVT profiles, try ~/.dbt/profiles.yml
611
+ if not source_outputs and dbt_profiles_path.exists():
612
+ with open(dbt_profiles_path) as f:
613
+ dbt_profiles = yaml.safe_load(f) or {}
614
+ if source_profile_name in dbt_profiles:
615
+ source_profile = dbt_profiles[source_profile_name]
616
+ source_outputs = source_profile.get("outputs", {})
617
+ profile_source = "~/.dbt/profiles.yml"
618
+
619
+ if not source_outputs:
620
+ click.echo(f" No profile '{source_profile_name}' found in ~/.dvt/profiles.yml or ~/.dbt/profiles.yml")
621
+ return MigrationResult(True, "No source profile found")
622
+
623
+ click.echo(f" Found profile '{source_profile_name}' in {profile_source} with {len(source_outputs)} target(s)")
624
+
625
+ # Read current DVT profiles for destination
626
+ if not dvt_profiles_path.exists():
627
+ click.echo(" No ~/.dvt/profiles.yml found - skipping target import")
628
+ return MigrationResult(False, "No DVT profiles found")
629
+
630
+ with open(dvt_profiles_path) as f:
631
+ dvt_profiles = yaml.safe_load(f) or {}
632
+
633
+ dvt_profile = dvt_profiles.get(dvt_profile_name, {"target": "dev", "outputs": {}})
634
+
635
+ # Merge source outputs with prefix
636
+ merged_count = 0
637
+ skipped_count = 0
638
+ merged_names = []
639
+ for output_name, output_config in source_outputs.items():
640
+ new_name = f"{source_name}_{output_name}"
641
+ if new_name not in dvt_profile.get("outputs", {}):
642
+ dvt_profile.setdefault("outputs", {})[new_name] = output_config
643
+ merged_count += 1
644
+ merged_names.append(new_name)
645
+ else:
646
+ skipped_count += 1
647
+
648
+ dvt_profiles[dvt_profile_name] = dvt_profile
649
+
650
+ if not self.dry_run:
651
+ if merged_count > 0:
652
+ with open(dvt_profiles_path, "w") as f:
653
+ f.write("# DVT Profiles Configuration\n\n")
654
+ yaml.dump(dvt_profiles, f, default_flow_style=False)
655
+ click.echo(f" Merged {merged_count} target(s) into profile '{dvt_profile_name}'")
656
+ if merged_names:
657
+ for name in merged_names:
658
+ click.echo(f" + {name}")
659
+ if skipped_count > 0:
660
+ click.echo(f" Skipped {skipped_count} target(s) (already exist)")
661
+ else:
662
+ click.echo(f" [DRY RUN] Would merge {merged_count} target(s) into profile")
663
+
664
+ return MigrationResult(True, f"Merged {merged_count} targets", merged_count)
665
+
666
+ def _show_summary(self, results: List[MigrationResult]) -> None:
667
+ """Show migration summary for Mode A."""
668
+ if not results:
669
+ return
670
+
671
+ successful = [r for r in results if r and r.success]
672
+ if not successful:
673
+ return
674
+
675
+ click.echo("")
676
+ click.echo("=" * 60)
677
+ click.echo("Migration complete!")
678
+ click.echo("=" * 60)
679
+
680
+ for result in successful:
681
+ if result:
682
+ click.echo(f" {result.message}")
683
+
684
+ click.echo("")
685
+ click.echo("Your dbt configuration has been preserved as backups.")
686
+ click.echo("Run 'dvt target list' to verify your connections.")
687
+ click.echo("=" * 60)
688
+
689
+ def _show_import_summary(self, source_name: str, results: List[MigrationResult]) -> None:
690
+ """Show import summary for Mode B."""
691
+ if self.dry_run:
692
+ click.echo("")
693
+ click.echo("No changes made (dry run).")
694
+ return
695
+
696
+ # Get copied directories
697
+ copied_dirs = [r.message for r in results if r and r.success and "/" in r.message]
698
+
699
+ click.echo("")
700
+ click.echo("=" * 60)
701
+ click.echo("Import complete!")
702
+ click.echo("=" * 60)
703
+ click.echo(f" Project: {source_name}")
704
+
705
+ for dir_result in copied_dirs:
706
+ dir_name = dir_result.rstrip("/")
707
+ click.echo(f" {dir_name.capitalize()}: {dir_name}/{source_name}/")
708
+
709
+ click.echo("")
710
+ click.echo("Next steps:")
711
+ click.echo(f" 1. Review {copied_dirs[0].split('/')[0] if copied_dirs else 'models'}/{source_name}/ for ref() adjustments")
712
+ click.echo(" 2. Update dvt_project.yml if needed")
713
+ click.echo(" 3. Run 'dvt target list' to see all targets")
714
+ click.echo("=" * 60)