dvt-core 0.58.6__cp311-cp311-macosx_10_9_x86_64.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (324) hide show
  1. dbt/__init__.py +7 -0
  2. dbt/_pydantic_shim.py +26 -0
  3. dbt/artifacts/__init__.py +0 -0
  4. dbt/artifacts/exceptions/__init__.py +1 -0
  5. dbt/artifacts/exceptions/schemas.py +31 -0
  6. dbt/artifacts/resources/__init__.py +116 -0
  7. dbt/artifacts/resources/base.py +67 -0
  8. dbt/artifacts/resources/types.py +93 -0
  9. dbt/artifacts/resources/v1/analysis.py +10 -0
  10. dbt/artifacts/resources/v1/catalog.py +23 -0
  11. dbt/artifacts/resources/v1/components.py +274 -0
  12. dbt/artifacts/resources/v1/config.py +277 -0
  13. dbt/artifacts/resources/v1/documentation.py +11 -0
  14. dbt/artifacts/resources/v1/exposure.py +51 -0
  15. dbt/artifacts/resources/v1/function.py +52 -0
  16. dbt/artifacts/resources/v1/generic_test.py +31 -0
  17. dbt/artifacts/resources/v1/group.py +21 -0
  18. dbt/artifacts/resources/v1/hook.py +11 -0
  19. dbt/artifacts/resources/v1/macro.py +29 -0
  20. dbt/artifacts/resources/v1/metric.py +172 -0
  21. dbt/artifacts/resources/v1/model.py +145 -0
  22. dbt/artifacts/resources/v1/owner.py +10 -0
  23. dbt/artifacts/resources/v1/saved_query.py +111 -0
  24. dbt/artifacts/resources/v1/seed.py +41 -0
  25. dbt/artifacts/resources/v1/semantic_layer_components.py +72 -0
  26. dbt/artifacts/resources/v1/semantic_model.py +314 -0
  27. dbt/artifacts/resources/v1/singular_test.py +14 -0
  28. dbt/artifacts/resources/v1/snapshot.py +91 -0
  29. dbt/artifacts/resources/v1/source_definition.py +84 -0
  30. dbt/artifacts/resources/v1/sql_operation.py +10 -0
  31. dbt/artifacts/resources/v1/unit_test_definition.py +77 -0
  32. dbt/artifacts/schemas/__init__.py +0 -0
  33. dbt/artifacts/schemas/base.py +191 -0
  34. dbt/artifacts/schemas/batch_results.py +24 -0
  35. dbt/artifacts/schemas/catalog/__init__.py +11 -0
  36. dbt/artifacts/schemas/catalog/v1/__init__.py +0 -0
  37. dbt/artifacts/schemas/catalog/v1/catalog.py +59 -0
  38. dbt/artifacts/schemas/freshness/__init__.py +1 -0
  39. dbt/artifacts/schemas/freshness/v3/__init__.py +0 -0
  40. dbt/artifacts/schemas/freshness/v3/freshness.py +158 -0
  41. dbt/artifacts/schemas/manifest/__init__.py +2 -0
  42. dbt/artifacts/schemas/manifest/v12/__init__.py +0 -0
  43. dbt/artifacts/schemas/manifest/v12/manifest.py +211 -0
  44. dbt/artifacts/schemas/results.py +147 -0
  45. dbt/artifacts/schemas/run/__init__.py +2 -0
  46. dbt/artifacts/schemas/run/v5/__init__.py +0 -0
  47. dbt/artifacts/schemas/run/v5/run.py +184 -0
  48. dbt/artifacts/schemas/upgrades/__init__.py +4 -0
  49. dbt/artifacts/schemas/upgrades/upgrade_manifest.py +174 -0
  50. dbt/artifacts/schemas/upgrades/upgrade_manifest_dbt_version.py +2 -0
  51. dbt/artifacts/utils/validation.py +153 -0
  52. dbt/cli/__init__.py +1 -0
  53. dbt/cli/context.py +17 -0
  54. dbt/cli/exceptions.py +57 -0
  55. dbt/cli/flags.py +560 -0
  56. dbt/cli/main.py +2403 -0
  57. dbt/cli/option_types.py +121 -0
  58. dbt/cli/options.py +80 -0
  59. dbt/cli/params.py +844 -0
  60. dbt/cli/requires.py +490 -0
  61. dbt/cli/resolvers.py +50 -0
  62. dbt/cli/types.py +40 -0
  63. dbt/clients/__init__.py +0 -0
  64. dbt/clients/checked_load.py +83 -0
  65. dbt/clients/git.py +164 -0
  66. dbt/clients/jinja.py +206 -0
  67. dbt/clients/jinja_static.py +245 -0
  68. dbt/clients/registry.py +192 -0
  69. dbt/clients/yaml_helper.py +68 -0
  70. dbt/compilation.py +876 -0
  71. dbt/compute/__init__.py +14 -0
  72. dbt/compute/engines/__init__.py +12 -0
  73. dbt/compute/engines/spark_engine.cpython-311-darwin.so +0 -0
  74. dbt/compute/engines/spark_engine.py +642 -0
  75. dbt/compute/federated_executor.cpython-311-darwin.so +0 -0
  76. dbt/compute/federated_executor.py +1080 -0
  77. dbt/compute/filter_pushdown.cpython-311-darwin.so +0 -0
  78. dbt/compute/filter_pushdown.py +273 -0
  79. dbt/compute/jar_provisioning.cpython-311-darwin.so +0 -0
  80. dbt/compute/jar_provisioning.py +255 -0
  81. dbt/compute/java_compat.cpython-311-darwin.so +0 -0
  82. dbt/compute/java_compat.py +689 -0
  83. dbt/compute/jdbc_utils.cpython-311-darwin.so +0 -0
  84. dbt/compute/jdbc_utils.py +678 -0
  85. dbt/compute/metadata/__init__.py +40 -0
  86. dbt/compute/metadata/adapters_registry.cpython-311-darwin.so +0 -0
  87. dbt/compute/metadata/adapters_registry.py +370 -0
  88. dbt/compute/metadata/registry.cpython-311-darwin.so +0 -0
  89. dbt/compute/metadata/registry.py +674 -0
  90. dbt/compute/metadata/store.cpython-311-darwin.so +0 -0
  91. dbt/compute/metadata/store.py +1499 -0
  92. dbt/compute/smart_selector.cpython-311-darwin.so +0 -0
  93. dbt/compute/smart_selector.py +377 -0
  94. dbt/compute/strategies/__init__.py +55 -0
  95. dbt/compute/strategies/base.cpython-311-darwin.so +0 -0
  96. dbt/compute/strategies/base.py +165 -0
  97. dbt/compute/strategies/dataproc.cpython-311-darwin.so +0 -0
  98. dbt/compute/strategies/dataproc.py +207 -0
  99. dbt/compute/strategies/emr.cpython-311-darwin.so +0 -0
  100. dbt/compute/strategies/emr.py +203 -0
  101. dbt/compute/strategies/local.cpython-311-darwin.so +0 -0
  102. dbt/compute/strategies/local.py +443 -0
  103. dbt/compute/strategies/standalone.cpython-311-darwin.so +0 -0
  104. dbt/compute/strategies/standalone.py +262 -0
  105. dbt/config/__init__.py +4 -0
  106. dbt/config/catalogs.py +94 -0
  107. dbt/config/compute.cpython-311-darwin.so +0 -0
  108. dbt/config/compute.py +513 -0
  109. dbt/config/dvt_profile.cpython-311-darwin.so +0 -0
  110. dbt/config/dvt_profile.py +342 -0
  111. dbt/config/profile.py +422 -0
  112. dbt/config/project.py +873 -0
  113. dbt/config/project_utils.py +28 -0
  114. dbt/config/renderer.py +231 -0
  115. dbt/config/runtime.py +553 -0
  116. dbt/config/selectors.py +208 -0
  117. dbt/config/utils.py +77 -0
  118. dbt/constants.py +28 -0
  119. dbt/context/__init__.py +0 -0
  120. dbt/context/base.py +745 -0
  121. dbt/context/configured.py +135 -0
  122. dbt/context/context_config.py +382 -0
  123. dbt/context/docs.py +82 -0
  124. dbt/context/exceptions_jinja.py +178 -0
  125. dbt/context/macro_resolver.py +195 -0
  126. dbt/context/macros.py +171 -0
  127. dbt/context/manifest.py +72 -0
  128. dbt/context/providers.py +2249 -0
  129. dbt/context/query_header.py +13 -0
  130. dbt/context/secret.py +58 -0
  131. dbt/context/target.py +74 -0
  132. dbt/contracts/__init__.py +0 -0
  133. dbt/contracts/files.py +413 -0
  134. dbt/contracts/graph/__init__.py +0 -0
  135. dbt/contracts/graph/manifest.py +1904 -0
  136. dbt/contracts/graph/metrics.py +97 -0
  137. dbt/contracts/graph/model_config.py +70 -0
  138. dbt/contracts/graph/node_args.py +42 -0
  139. dbt/contracts/graph/nodes.py +1806 -0
  140. dbt/contracts/graph/semantic_manifest.py +232 -0
  141. dbt/contracts/graph/unparsed.py +811 -0
  142. dbt/contracts/project.py +417 -0
  143. dbt/contracts/results.py +53 -0
  144. dbt/contracts/selection.py +23 -0
  145. dbt/contracts/sql.py +85 -0
  146. dbt/contracts/state.py +68 -0
  147. dbt/contracts/util.py +46 -0
  148. dbt/deprecations.py +348 -0
  149. dbt/deps/__init__.py +0 -0
  150. dbt/deps/base.py +152 -0
  151. dbt/deps/git.py +195 -0
  152. dbt/deps/local.py +79 -0
  153. dbt/deps/registry.py +130 -0
  154. dbt/deps/resolver.py +149 -0
  155. dbt/deps/tarball.py +120 -0
  156. dbt/docs/source/_ext/dbt_click.py +119 -0
  157. dbt/docs/source/conf.py +32 -0
  158. dbt/env_vars.py +64 -0
  159. dbt/event_time/event_time.py +40 -0
  160. dbt/event_time/sample_window.py +60 -0
  161. dbt/events/__init__.py +15 -0
  162. dbt/events/base_types.py +36 -0
  163. dbt/events/core_types_pb2.py +2 -0
  164. dbt/events/logging.py +108 -0
  165. dbt/events/types.py +2516 -0
  166. dbt/exceptions.py +1486 -0
  167. dbt/flags.py +89 -0
  168. dbt/graph/__init__.py +11 -0
  169. dbt/graph/cli.py +249 -0
  170. dbt/graph/graph.py +172 -0
  171. dbt/graph/queue.py +214 -0
  172. dbt/graph/selector.py +374 -0
  173. dbt/graph/selector_methods.py +975 -0
  174. dbt/graph/selector_spec.py +222 -0
  175. dbt/graph/thread_pool.py +18 -0
  176. dbt/hooks.py +21 -0
  177. dbt/include/README.md +49 -0
  178. dbt/include/__init__.py +3 -0
  179. dbt/include/data/adapters_registry.duckdb +0 -0
  180. dbt/include/data/build_registry.py +242 -0
  181. dbt/include/data/csv/adapter_queries.csv +33 -0
  182. dbt/include/data/csv/syntax_rules.csv +9 -0
  183. dbt/include/data/csv/type_mappings_bigquery.csv +28 -0
  184. dbt/include/data/csv/type_mappings_databricks.csv +30 -0
  185. dbt/include/data/csv/type_mappings_mysql.csv +40 -0
  186. dbt/include/data/csv/type_mappings_oracle.csv +30 -0
  187. dbt/include/data/csv/type_mappings_postgres.csv +56 -0
  188. dbt/include/data/csv/type_mappings_redshift.csv +33 -0
  189. dbt/include/data/csv/type_mappings_snowflake.csv +38 -0
  190. dbt/include/data/csv/type_mappings_sqlserver.csv +35 -0
  191. dbt/include/starter_project/.gitignore +4 -0
  192. dbt/include/starter_project/README.md +15 -0
  193. dbt/include/starter_project/__init__.py +3 -0
  194. dbt/include/starter_project/analyses/.gitkeep +0 -0
  195. dbt/include/starter_project/dbt_project.yml +36 -0
  196. dbt/include/starter_project/macros/.gitkeep +0 -0
  197. dbt/include/starter_project/models/example/my_first_dbt_model.sql +27 -0
  198. dbt/include/starter_project/models/example/my_second_dbt_model.sql +6 -0
  199. dbt/include/starter_project/models/example/schema.yml +21 -0
  200. dbt/include/starter_project/seeds/.gitkeep +0 -0
  201. dbt/include/starter_project/snapshots/.gitkeep +0 -0
  202. dbt/include/starter_project/tests/.gitkeep +0 -0
  203. dbt/internal_deprecations.py +26 -0
  204. dbt/jsonschemas/__init__.py +3 -0
  205. dbt/jsonschemas/jsonschemas.py +309 -0
  206. dbt/jsonschemas/project/0.0.110.json +4717 -0
  207. dbt/jsonschemas/project/0.0.85.json +2015 -0
  208. dbt/jsonschemas/resources/0.0.110.json +2636 -0
  209. dbt/jsonschemas/resources/0.0.85.json +2536 -0
  210. dbt/jsonschemas/resources/latest.json +6773 -0
  211. dbt/links.py +4 -0
  212. dbt/materializations/__init__.py +0 -0
  213. dbt/materializations/incremental/__init__.py +0 -0
  214. dbt/materializations/incremental/microbatch.py +236 -0
  215. dbt/mp_context.py +8 -0
  216. dbt/node_types.py +37 -0
  217. dbt/parser/__init__.py +23 -0
  218. dbt/parser/analysis.py +21 -0
  219. dbt/parser/base.py +548 -0
  220. dbt/parser/common.py +266 -0
  221. dbt/parser/docs.py +52 -0
  222. dbt/parser/fixtures.py +51 -0
  223. dbt/parser/functions.py +30 -0
  224. dbt/parser/generic_test.py +100 -0
  225. dbt/parser/generic_test_builders.py +333 -0
  226. dbt/parser/hooks.py +118 -0
  227. dbt/parser/macros.py +137 -0
  228. dbt/parser/manifest.py +2204 -0
  229. dbt/parser/models.py +573 -0
  230. dbt/parser/partial.py +1178 -0
  231. dbt/parser/read_files.py +445 -0
  232. dbt/parser/schema_generic_tests.py +422 -0
  233. dbt/parser/schema_renderer.py +111 -0
  234. dbt/parser/schema_yaml_readers.py +935 -0
  235. dbt/parser/schemas.py +1466 -0
  236. dbt/parser/search.py +149 -0
  237. dbt/parser/seeds.py +28 -0
  238. dbt/parser/singular_test.py +20 -0
  239. dbt/parser/snapshots.py +44 -0
  240. dbt/parser/sources.py +558 -0
  241. dbt/parser/sql.py +62 -0
  242. dbt/parser/unit_tests.py +621 -0
  243. dbt/plugins/__init__.py +20 -0
  244. dbt/plugins/contracts.py +9 -0
  245. dbt/plugins/exceptions.py +2 -0
  246. dbt/plugins/manager.py +163 -0
  247. dbt/plugins/manifest.py +21 -0
  248. dbt/profiler.py +20 -0
  249. dbt/py.typed +1 -0
  250. dbt/query_analyzer.cpython-311-darwin.so +0 -0
  251. dbt/query_analyzer.py +410 -0
  252. dbt/runners/__init__.py +2 -0
  253. dbt/runners/exposure_runner.py +7 -0
  254. dbt/runners/no_op_runner.py +45 -0
  255. dbt/runners/saved_query_runner.py +7 -0
  256. dbt/selected_resources.py +8 -0
  257. dbt/task/__init__.py +0 -0
  258. dbt/task/base.py +503 -0
  259. dbt/task/build.py +197 -0
  260. dbt/task/clean.py +56 -0
  261. dbt/task/clone.py +161 -0
  262. dbt/task/compile.py +150 -0
  263. dbt/task/compute.cpython-311-darwin.so +0 -0
  264. dbt/task/compute.py +458 -0
  265. dbt/task/debug.py +505 -0
  266. dbt/task/deps.py +280 -0
  267. dbt/task/docs/__init__.py +3 -0
  268. dbt/task/docs/api/__init__.py +23 -0
  269. dbt/task/docs/api/catalog.cpython-311-darwin.so +0 -0
  270. dbt/task/docs/api/catalog.py +204 -0
  271. dbt/task/docs/api/lineage.cpython-311-darwin.so +0 -0
  272. dbt/task/docs/api/lineage.py +234 -0
  273. dbt/task/docs/api/profile.cpython-311-darwin.so +0 -0
  274. dbt/task/docs/api/profile.py +204 -0
  275. dbt/task/docs/api/spark.cpython-311-darwin.so +0 -0
  276. dbt/task/docs/api/spark.py +186 -0
  277. dbt/task/docs/generate.py +947 -0
  278. dbt/task/docs/index.html +250 -0
  279. dbt/task/docs/serve.cpython-311-darwin.so +0 -0
  280. dbt/task/docs/serve.py +174 -0
  281. dbt/task/dvt_output.py +362 -0
  282. dbt/task/dvt_run.py +204 -0
  283. dbt/task/freshness.py +322 -0
  284. dbt/task/function.py +121 -0
  285. dbt/task/group_lookup.py +46 -0
  286. dbt/task/init.cpython-311-darwin.so +0 -0
  287. dbt/task/init.py +604 -0
  288. dbt/task/java.cpython-311-darwin.so +0 -0
  289. dbt/task/java.py +316 -0
  290. dbt/task/list.py +236 -0
  291. dbt/task/metadata.cpython-311-darwin.so +0 -0
  292. dbt/task/metadata.py +804 -0
  293. dbt/task/printer.py +175 -0
  294. dbt/task/profile.cpython-311-darwin.so +0 -0
  295. dbt/task/profile.py +1307 -0
  296. dbt/task/profile_serve.py +615 -0
  297. dbt/task/retract.py +438 -0
  298. dbt/task/retry.py +175 -0
  299. dbt/task/run.py +1387 -0
  300. dbt/task/run_operation.py +141 -0
  301. dbt/task/runnable.py +758 -0
  302. dbt/task/seed.py +103 -0
  303. dbt/task/show.py +149 -0
  304. dbt/task/snapshot.py +56 -0
  305. dbt/task/spark.cpython-311-darwin.so +0 -0
  306. dbt/task/spark.py +414 -0
  307. dbt/task/sql.py +110 -0
  308. dbt/task/target_sync.cpython-311-darwin.so +0 -0
  309. dbt/task/target_sync.py +766 -0
  310. dbt/task/test.py +464 -0
  311. dbt/tests/fixtures/__init__.py +1 -0
  312. dbt/tests/fixtures/project.py +620 -0
  313. dbt/tests/util.py +651 -0
  314. dbt/tracking.py +529 -0
  315. dbt/utils/__init__.py +3 -0
  316. dbt/utils/artifact_upload.py +151 -0
  317. dbt/utils/utils.py +408 -0
  318. dbt/version.py +270 -0
  319. dvt_cli/__init__.py +72 -0
  320. dvt_core-0.58.6.dist-info/METADATA +288 -0
  321. dvt_core-0.58.6.dist-info/RECORD +324 -0
  322. dvt_core-0.58.6.dist-info/WHEEL +5 -0
  323. dvt_core-0.58.6.dist-info/entry_points.txt +2 -0
  324. dvt_core-0.58.6.dist-info/top_level.txt +2 -0
dbt/task/init.py ADDED
@@ -0,0 +1,604 @@
1
+ import copy
2
+ import os
3
+ import re
4
+ import shutil
5
+ from pathlib import Path
6
+ from typing import Optional
7
+
8
+ import click
9
+ import yaml
10
+
11
+ import dbt.config
12
+ import dbt_common.clients.system
13
+ from dbt.adapters.factory import get_include_paths, load_plugin
14
+ from dbt.compute.metadata import ProjectMetadataStore
15
+ from dbt.config.profile import read_profile
16
+ from dbt.contracts.util import Identifier as ProjectName
17
+ from dbt.events.types import (
18
+ ConfigFolderDirectory,
19
+ InvalidProfileTemplateYAML,
20
+ NoSampleProfileFound,
21
+ ProfileWrittenWithProjectTemplateYAML,
22
+ ProfileWrittenWithSample,
23
+ ProfileWrittenWithTargetTemplateYAML,
24
+ ProjectCreated,
25
+ ProjectNameAlreadyExists,
26
+ SettingUpProfile,
27
+ StarterProjectPath,
28
+ )
29
+ from dbt.flags import get_flags
30
+ from dbt.task.base import BaseTask, move_to_nearest_project_dir
31
+ from dbt.version import _get_adapter_plugin_names
32
+ from dbt_common.events.functions import fire_event
33
+ from dbt_common.exceptions import DbtRuntimeError
34
+
35
+ DOCS_URL = "https://docs.getdbt.com/docs/configure-your-profile"
36
+ SLACK_URL = "https://community.getdbt.com/"
37
+
38
+ # This file is not needed for the starter project but exists for finding the resource path
39
+ IGNORE_FILES = ["__init__.py", "__pycache__"]
40
+
41
+
42
+ # https://click.palletsprojects.com/en/8.0.x/api/#types
43
+ # click v7.0 has UNPROCESSED, STRING, INT, FLOAT, BOOL, and UUID available.
44
+ click_type_mapping = {
45
+ "string": click.STRING,
46
+ "int": click.INT,
47
+ "float": click.FLOAT,
48
+ "bool": click.BOOL,
49
+ None: None,
50
+ }
51
+
52
+
53
+ class InitTask(BaseTask):
54
+ def copy_starter_repo(self, project_name: str) -> None:
55
+ # Lazy import to avoid ModuleNotFoundError
56
+ from dbt.include.starter_project import (
57
+ PACKAGE_PATH as starter_project_directory,
58
+ )
59
+
60
+ fire_event(StarterProjectPath(dir=starter_project_directory))
61
+ shutil.copytree(
62
+ starter_project_directory, project_name, ignore=shutil.ignore_patterns(*IGNORE_FILES)
63
+ )
64
+
65
+ def create_profiles_dir(self, profiles_dir: str) -> bool:
66
+ """Create the user's profiles directory if it doesn't already exist."""
67
+ profiles_path = Path(profiles_dir)
68
+ if not profiles_path.exists():
69
+ fire_event(ConfigFolderDirectory(dir=str(profiles_dir)))
70
+ dbt_common.clients.system.make_directory(profiles_dir)
71
+ return True
72
+ return False
73
+
74
+ def create_profile_from_sample(self, adapter: str, profile_name: str):
75
+ """Create a profile entry using the adapter's sample_profiles.yml
76
+
77
+ Renames the profile in sample_profiles.yml to match that of the project."""
78
+ # Line below raises an exception if the specified adapter is not found
79
+ load_plugin(adapter)
80
+ adapter_path = get_include_paths(adapter)[0]
81
+ sample_profiles_path = adapter_path / "sample_profiles.yml"
82
+
83
+ if not sample_profiles_path.exists():
84
+ fire_event(NoSampleProfileFound(adapter=adapter))
85
+ else:
86
+ with open(sample_profiles_path, "r") as f:
87
+ sample_profile = f.read()
88
+ sample_profile_name = list(yaml.safe_load(sample_profile).keys())[0]
89
+ # Use a regex to replace the name of the sample_profile with
90
+ # that of the project without losing any comments from the sample
91
+ sample_profile = re.sub(f"^{sample_profile_name}:", f"{profile_name}:", sample_profile)
92
+ profiles_filepath = Path(get_flags().PROFILES_DIR) / Path("profiles.yml")
93
+ if profiles_filepath.exists():
94
+ with open(profiles_filepath, "a") as f:
95
+ f.write("\n" + sample_profile)
96
+ else:
97
+ with open(profiles_filepath, "w") as f:
98
+ f.write(sample_profile)
99
+ fire_event(
100
+ ProfileWrittenWithSample(name=profile_name, path=str(profiles_filepath))
101
+ )
102
+
103
+ def generate_target_from_input(self, profile_template: dict, target: dict = {}) -> dict:
104
+ """Generate a target configuration from profile_template and user input."""
105
+ profile_template_local = copy.deepcopy(profile_template)
106
+ for key, value in profile_template_local.items():
107
+ if key.startswith("_choose"):
108
+ choice_type = key[8:].replace("_", " ")
109
+ option_list = list(value.keys())
110
+ prompt_msg = (
111
+ "\n".join([f"[{n + 1}] {v}" for n, v in enumerate(option_list)])
112
+ + f"\nDesired {choice_type} option (enter a number)"
113
+ )
114
+ numeric_choice = click.prompt(prompt_msg, type=click.INT)
115
+ choice = option_list[numeric_choice - 1]
116
+ # Complete the chosen option's values in a recursive call
117
+ target = self.generate_target_from_input(
118
+ profile_template_local[key][choice], target
119
+ )
120
+ else:
121
+ if key.startswith("_fixed"):
122
+ # _fixed prefixed keys are not presented to the user
123
+ target[key[7:]] = value
124
+ else:
125
+ hide_input = value.get("hide_input", False)
126
+ default = value.get("default", None)
127
+ hint = value.get("hint", None)
128
+ type = click_type_mapping[value.get("type", None)]
129
+ text = key + (f" ({hint})" if hint else "")
130
+ target[key] = click.prompt(
131
+ text, default=default, hide_input=hide_input, type=type
132
+ )
133
+ return target
134
+
135
+ def get_profile_name_from_current_project(self) -> str:
136
+ """Reads dbt_project.yml in the current directory to retrieve the
137
+ profile name.
138
+ """
139
+ with open("dbt_project.yml") as f:
140
+ dbt_project = yaml.safe_load(f)
141
+ return dbt_project["profile"]
142
+
143
+ def write_profile(self, profile: dict, profile_name: str):
144
+ """Given a profile, write it to the current project's profiles.yml.
145
+ This will overwrite any profile with a matching name."""
146
+ # Create the profile directory if it doesn't exist
147
+ profiles_filepath = Path(get_flags().PROFILES_DIR) / Path("profiles.yml")
148
+
149
+ profiles = {profile_name: profile}
150
+
151
+ if profiles_filepath.exists():
152
+ with open(profiles_filepath, "r") as f:
153
+ profiles = yaml.safe_load(f) or {}
154
+ profiles[profile_name] = profile
155
+
156
+ # Write the profiles dictionary to a brand-new or pre-existing file
157
+ with open(profiles_filepath, "w") as f:
158
+ yaml.dump(profiles, f)
159
+
160
+ def create_profile_from_profile_template(self, profile_template: dict, profile_name: str):
161
+ """Create and write a profile using the supplied profile_template."""
162
+ initial_target = profile_template.get("fixed", {})
163
+ prompts = profile_template.get("prompts", {})
164
+ target = self.generate_target_from_input(prompts, initial_target)
165
+ target_name = target.pop("target", "dev")
166
+ profile = {"outputs": {target_name: target}, "target": target_name}
167
+ self.write_profile(profile, profile_name)
168
+
169
+ def create_profile_from_target(self, adapter: str, profile_name: str):
170
+ """Create a profile without defaults using target's profile_template.yml if available, or
171
+ sample_profiles.yml as a fallback."""
172
+ # Line below raises an exception if the specified adapter is not found
173
+ load_plugin(adapter)
174
+ adapter_path = get_include_paths(adapter)[0]
175
+ profile_template_path = adapter_path / "profile_template.yml"
176
+
177
+ if profile_template_path.exists():
178
+ with open(profile_template_path) as f:
179
+ profile_template = yaml.safe_load(f)
180
+ self.create_profile_from_profile_template(profile_template, profile_name)
181
+ profiles_filepath = Path(get_flags().PROFILES_DIR) / Path("profiles.yml")
182
+ fire_event(
183
+ ProfileWrittenWithTargetTemplateYAML(
184
+ name=profile_name, path=str(profiles_filepath)
185
+ )
186
+ )
187
+ else:
188
+ # For adapters without a profile_template.yml defined, fallback on
189
+ # sample_profiles.yml
190
+ self.create_profile_from_sample(adapter, profile_name)
191
+
192
+ def check_if_profile_exists(self, profile_name: str) -> bool:
193
+ """
194
+ Validate that the specified profile exists. Can't use the regular profile validation
195
+ routine because it assumes the project file exists
196
+ """
197
+ profiles_dir = get_flags().PROFILES_DIR
198
+ raw_profiles = read_profile(profiles_dir)
199
+ return profile_name in raw_profiles
200
+
201
+ def check_if_can_write_profile(self, profile_name: Optional[str] = None) -> bool:
202
+ """Using either a provided profile name or that specified in dbt_project.yml,
203
+ check if the profile already exists in profiles.yml, and if so ask the
204
+ user whether to proceed and overwrite it."""
205
+ profiles_file = Path(get_flags().PROFILES_DIR) / Path("profiles.yml")
206
+ if not profiles_file.exists():
207
+ return True
208
+ profile_name = profile_name or self.get_profile_name_from_current_project()
209
+ with open(profiles_file, "r") as f:
210
+ profiles = yaml.safe_load(f) or {}
211
+ if profile_name in profiles.keys():
212
+ # Profile already exists, just skip profile setup
213
+ click.echo(f"Profile '{profile_name}' already exists in {profiles_file}, skipping profile setup.")
214
+ return False
215
+ else:
216
+ return True
217
+
218
+ def create_profile_using_project_profile_template(self, profile_name):
219
+ """Create a profile using the project's profile_template.yml"""
220
+ with open("profile_template.yml") as f:
221
+ profile_template = yaml.safe_load(f)
222
+ self.create_profile_from_profile_template(profile_template, profile_name)
223
+ profiles_filepath = Path(get_flags().PROFILES_DIR) / Path("profiles.yml")
224
+ fire_event(
225
+ ProfileWrittenWithProjectTemplateYAML(name=profile_name, path=str(profiles_filepath))
226
+ )
227
+
228
+ def ask_for_adapter_choice(self) -> str:
229
+ """Ask the user which adapter (database) they'd like to use."""
230
+ available_adapters = list(_get_adapter_plugin_names())
231
+
232
+ if not available_adapters:
233
+ raise dbt.exceptions.NoAdaptersAvailableError()
234
+
235
+ prompt_msg = (
236
+ "Which database would you like to use?\n"
237
+ + "\n".join([f"[{n + 1}] {v}" for n, v in enumerate(available_adapters)])
238
+ + "\n\n(Don't see the one you want? https://docs.getdbt.com/docs/available-adapters)"
239
+ + "\n\nEnter a number"
240
+ )
241
+ numeric_choice = click.prompt(prompt_msg, type=click.INT)
242
+ return available_adapters[numeric_choice - 1]
243
+
244
+ def setup_profile(self, profile_name: str) -> None:
245
+ """Set up a new profile for a project"""
246
+ fire_event(SettingUpProfile())
247
+ if not self.check_if_can_write_profile(profile_name=profile_name):
248
+ return
249
+ # If a profile_template.yml exists in the project root, that effectively
250
+ # overrides the profile_template.yml for the given target.
251
+ profile_template_path = Path("profile_template.yml")
252
+ if profile_template_path.exists():
253
+ try:
254
+ # This relies on a valid profile_template.yml from the user,
255
+ # so use a try: except to fall back to the default on failure
256
+ self.create_profile_using_project_profile_template(profile_name)
257
+ return
258
+ except Exception:
259
+ fire_event(InvalidProfileTemplateYAML())
260
+ adapter = self.ask_for_adapter_choice()
261
+ self.create_profile_from_target(adapter, profile_name=profile_name)
262
+
263
+ def get_adapter_metadata(self) -> dict:
264
+ """Get categorized adapter information with descriptions - COMPREHENSIVE."""
265
+ return {
266
+ "Cloud Data Warehouses": {
267
+ "snowflake": {"name": "Snowflake", "desc": "Cloud data warehouse", "jdbc": True},
268
+ "bigquery": {"name": "Google BigQuery", "desc": "Serverless warehouse", "jdbc": True},
269
+ "databricks": {"name": "Databricks", "desc": "Lakehouse platform", "jdbc": True},
270
+ "redshift": {"name": "Amazon Redshift", "desc": "AWS warehouse", "jdbc": True},
271
+ "firebolt": {"name": "Firebolt", "desc": "Cloud DW for analytics", "jdbc": True},
272
+ },
273
+ "Microsoft Ecosystem": {
274
+ "fabric": {"name": "Microsoft Fabric", "desc": "Unified analytics", "jdbc": True},
275
+ "synapse": {"name": "Azure Synapse", "desc": "Analytics service", "jdbc": True},
276
+ "sqlserver": {"name": "SQL Server", "desc": "Enterprise RDBMS", "jdbc": True},
277
+ },
278
+ "Enterprise Data Warehouses": {
279
+ "teradata": {"name": "Teradata", "desc": "Enterprise warehouse", "jdbc": True},
280
+ "oracle": {"name": "Oracle Database", "desc": "Enterprise RDBMS", "jdbc": True},
281
+ "db2": {"name": "IBM DB2", "desc": "IBM database system", "jdbc": True},
282
+ "exasol": {"name": "Exasol", "desc": "In-memory analytics", "jdbc": True},
283
+ "vertica": {"name": "Vertica", "desc": "Columnar analytics", "jdbc": True},
284
+ },
285
+ "SQL Engines & Query Platforms": {
286
+ "spark": {"name": "Apache Spark", "desc": "Unified analytics", "jdbc": True},
287
+ "trino": {"name": "Trino", "desc": "Distributed SQL engine", "jdbc": True},
288
+ "presto": {"name": "Presto", "desc": "Meta's query engine", "jdbc": True},
289
+ "athena": {"name": "Amazon Athena", "desc": "Query S3 data", "jdbc": True},
290
+ "dremio": {"name": "Dremio", "desc": "Data lakehouse platform", "jdbc": True},
291
+ "hive": {"name": "Apache Hive", "desc": "Hadoop data warehouse", "jdbc": True},
292
+ "impala": {"name": "Cloudera Impala", "desc": "MPP SQL engine", "jdbc": True},
293
+ "glue": {"name": "AWS Glue", "desc": "Serverless ETL", "jdbc": True},
294
+ },
295
+ "Open Source Databases": {
296
+ "postgres": {"name": "PostgreSQL", "desc": "Popular open-source DB", "jdbc": True},
297
+ "mysql": {"name": "MySQL", "desc": "World's most popular DB", "jdbc": True},
298
+ "mariadb": {"name": "MariaDB", "desc": "MySQL fork", "jdbc": True},
299
+ "sqlite": {"name": "SQLite", "desc": "Embedded database", "jdbc": False},
300
+ "duckdb": {"name": "DuckDB", "desc": "In-process OLAP", "jdbc": False},
301
+ "cratedb": {"name": "CrateDB", "desc": "Distributed SQL", "jdbc": True},
302
+ },
303
+ "OLAP & Analytics Databases": {
304
+ "clickhouse": {"name": "ClickHouse", "desc": "Fast OLAP database", "jdbc": True},
305
+ "starrocks": {"name": "StarRocks", "desc": "MPP analytics", "jdbc": True},
306
+ "doris": {"name": "Apache Doris", "desc": "Real-time analytics", "jdbc": True},
307
+ "greenplum": {"name": "Greenplum", "desc": "MPP database", "jdbc": True},
308
+ "monetdb": {"name": "MonetDB", "desc": "Columnar database", "jdbc": True},
309
+ },
310
+ "Time-Series & Streaming": {
311
+ "timescaledb": {"name": "TimescaleDB", "desc": "PostgreSQL for time-series", "jdbc": True},
312
+ "questdb": {"name": "QuestDB", "desc": "Fast time-series", "jdbc": True},
313
+ "materialize": {"name": "Materialize", "desc": "Streaming SQL", "jdbc": True},
314
+ "rockset": {"name": "Rockset", "desc": "Real-time analytics", "jdbc": True},
315
+ },
316
+ "Data Lakes & Modern Formats": {
317
+ "iceberg": {"name": "Apache Iceberg", "desc": "Table format", "jdbc": True},
318
+ },
319
+ "Specialized & Emerging": {
320
+ "singlestore": {"name": "SingleStore", "desc": "Real-time analytics", "jdbc": True},
321
+ "neo4j": {"name": "Neo4j", "desc": "Graph database", "jdbc": True},
322
+ "mindsdb": {"name": "MindsDB", "desc": "ML database", "jdbc": True},
323
+ },
324
+ }
325
+
326
+ def ask_for_adapter_choice_enhanced(self, prompt_prefix: str = "") -> str:
327
+ """Enhanced adapter selection with categories."""
328
+ metadata = self.get_adapter_metadata()
329
+ menu_lines = []
330
+ adapter_list = []
331
+ counter = 1
332
+
333
+ for category, adapters in metadata.items():
334
+ menu_lines.append(f"\n{category}:")
335
+ for adapter_key, info in adapters.items():
336
+ jdbc = " [JDBC]" if info["jdbc"] else ""
337
+ menu_lines.append(f" [{counter}] {info['name']}{jdbc} - {info['desc']}")
338
+ adapter_list.append(adapter_key)
339
+ counter += 1
340
+
341
+ prompt_msg = (
342
+ f"{prompt_prefix}" + "\n".join(menu_lines) +
343
+ "\n\nAll adapters support Spark JDBC federation\nEnter a number"
344
+ )
345
+
346
+ numeric_choice = click.prompt(prompt_msg, type=click.INT)
347
+ return adapter_list[numeric_choice - 1]
348
+
349
+ def ask_for_multi_connection_setup(self) -> bool:
350
+ """Ask if user wants multi-connection setup."""
351
+ msg = (
352
+ "\nDVT supports multi-source data federation.\n"
353
+ "Set up multiple database connections?\n"
354
+ " [1] Yes - Multiple connections (recommended for federation)\n"
355
+ " [2] No - Single connection (can add more later)\n"
356
+ "\nEnter a number"
357
+ )
358
+ choice = click.prompt(msg, type=click.INT, default=1)
359
+ return choice == 1
360
+
361
+ def setup_multi_connection_profile(self, profile_name: str) -> int:
362
+ """
363
+ Set up profile with multiple connections interactively.
364
+ Returns number of connections configured.
365
+ """
366
+ fire_event(SettingUpProfile())
367
+
368
+ if not self.check_if_can_write_profile(profile_name=profile_name):
369
+ return 0
370
+
371
+ # Ask how many
372
+ num = click.prompt(
373
+ "\nHow many database connections? (1-10)",
374
+ type=click.INT,
375
+ default=2
376
+ )
377
+ num = max(1, min(10, num))
378
+
379
+ outputs = {}
380
+
381
+ for i in range(num):
382
+ click.echo(f"\n--- Connection {i+1}/{num} ---")
383
+
384
+ adapter = self.ask_for_adapter_choice_enhanced(
385
+ prompt_prefix=f"\nSelect database type for connection {i+1}:\n"
386
+ )
387
+
388
+ default_name = f"{adapter}_{i+1}" if i > 0 else adapter
389
+ conn_name = click.prompt(
390
+ f"Connection name",
391
+ default=default_name
392
+ )
393
+
394
+ # Use adapter's profile template for prompts
395
+ load_plugin(adapter)
396
+ adapter_path = get_include_paths(adapter)[0]
397
+ template_path = adapter_path / "profile_template.yml"
398
+
399
+ if template_path.exists():
400
+ with open(template_path) as f:
401
+ template = yaml.safe_load(f)
402
+ prompts = template.get("prompts", {})
403
+ fixed = template.get("fixed", {})
404
+ target_config = self.generate_target_from_input(prompts, fixed)
405
+ target_config.pop("target", None)
406
+ outputs[conn_name] = target_config
407
+ else:
408
+ outputs[conn_name] = {"type": adapter}
409
+
410
+ # Set default target
411
+ output_names = list(outputs.keys())
412
+ default_target = output_names[0]
413
+
414
+ if len(output_names) > 1:
415
+ click.echo("\nAvailable connections:")
416
+ for idx, name in enumerate(output_names):
417
+ click.echo(f" [{idx+1}] {name}")
418
+
419
+ default_choice = click.prompt(
420
+ "\nDefault target? (enter number)",
421
+ type=click.INT,
422
+ default=1
423
+ )
424
+ default_target = output_names[max(0, min(default_choice - 1, len(output_names) - 1))]
425
+
426
+ # Write profile
427
+ profile = {"outputs": outputs, "target": default_target}
428
+ self.write_profile(profile, profile_name)
429
+
430
+ profiles_path = Path(get_flags().PROFILES_DIR) / "profiles.yml"
431
+ fire_event(ProfileWrittenWithTargetTemplateYAML(name=profile_name, path=str(profiles_path)))
432
+
433
+ return len(outputs)
434
+
435
+ def show_next_steps(self, project_name: str, num_connections: int) -> None:
436
+ """Show helpful next steps."""
437
+ click.echo("\n" + "=" * 60)
438
+ click.echo("🎉 DVT project initialized successfully!")
439
+ click.echo("=" * 60)
440
+ click.echo(f"\nProject: {project_name}")
441
+ click.echo(f"Connections: {num_connections} configured")
442
+ click.echo("\n🚀 Next Steps:")
443
+ click.echo(f" 1. cd {project_name}")
444
+ click.echo(" 2. Edit models/example/my_first_dbt_model.sql")
445
+ click.echo(" 3. dvt run")
446
+ click.echo(" 4. dvt test")
447
+ click.echo("\n🔗 Useful Commands:")
448
+ click.echo(" dvt target list # List connections")
449
+ click.echo(" dvt target test-all # Test all connections")
450
+ click.echo(" dvt compute list # Show Spark config")
451
+ click.echo(" dvt --help # See all commands")
452
+ click.echo("\n" + "=" * 60)
453
+
454
+ def get_valid_project_name(self) -> str:
455
+ """Returns a valid project name, either from CLI arg or user prompt."""
456
+
457
+ # Lazy import to avoid ModuleNotFoundError
458
+ from dbt.include.global_project import PROJECT_NAME as GLOBAL_PROJECT_NAME
459
+
460
+ name = self.args.project_name
461
+ internal_package_names = {GLOBAL_PROJECT_NAME}
462
+ available_adapters = list(_get_adapter_plugin_names())
463
+ for adapter_name in available_adapters:
464
+ internal_package_names.update(f"dbt_{adapter_name}")
465
+ while not ProjectName.is_valid(name) or name in internal_package_names:
466
+ if name:
467
+ click.echo(name + " is not a valid project name.")
468
+ name = click.prompt("Enter a name for your project (letters, digits, underscore)")
469
+
470
+ return name
471
+
472
+ def create_new_project(self, project_name: str, profile_name: str):
473
+ self.copy_starter_repo(project_name)
474
+ os.chdir(project_name)
475
+ with open("dbt_project.yml", "r") as f:
476
+ content = f"{f.read()}".format(project_name=project_name, profile_name=profile_name)
477
+ with open("dbt_project.yml", "w") as f:
478
+ f.write(content)
479
+
480
+ # v0.55.0: Create project-level .dvt/ structure
481
+ # 1. Create .dvt/ directory
482
+ project_dvt_dir = Path(".") / ".dvt"
483
+ project_dvt_dir.mkdir(parents=True, exist_ok=True)
484
+
485
+ # 2. Create .dvt/jdbc_jars/ directory
486
+ from dbt.config.compute import ComputeRegistry
487
+ ComputeRegistry.ensure_jdbc_jars_dir(".")
488
+
489
+ # 3. Create .dvt/computes.yml with defaults
490
+ registry = ComputeRegistry(project_dir=".")
491
+ registry.ensure_config_exists()
492
+ click.echo(" ✓ Compute config initialized (.dvt/computes.yml)")
493
+
494
+ # 4. Initialize project metadata store (.dvt/metadata_store.duckdb)
495
+ self._initialize_metadata_store(Path("."))
496
+
497
+ fire_event(
498
+ ProjectCreated(
499
+ project_name=project_name,
500
+ docs_url=DOCS_URL,
501
+ slack_url=SLACK_URL,
502
+ )
503
+ )
504
+
505
+ def _initialize_metadata_store(self, project_root: Path) -> None:
506
+ """
507
+ Initialize the DVT metadata store in .dvt/metadata_store.duckdb.
508
+
509
+ v0.55.0: Creates project-level metadata store with:
510
+ - column_metadata table for schema info
511
+ - row_counts table for cached row counts
512
+
513
+ NOTE: Static registry data (type mappings, syntax rules) comes from
514
+ the shipped adapters_registry.duckdb, not the project store.
515
+
516
+ This is idempotent - calling on an existing store will reinitialize it.
517
+ """
518
+ try:
519
+ store = ProjectMetadataStore(project_root)
520
+ store.initialize()
521
+ store.close()
522
+ click.echo(" ✓ Metadata store initialized (.dvt/metadata_store.duckdb)")
523
+ except ImportError:
524
+ # DuckDB not installed - skip metadata store (optional feature)
525
+ click.echo(" ⚠ DuckDB not installed - skipping metadata store")
526
+ except Exception as e:
527
+ # Don't fail init on metadata store errors
528
+ click.echo(f" ⚠ Could not initialize metadata store: {e}")
529
+
530
+ def run(self):
531
+ """Entry point for the init task."""
532
+ profiles_dir = get_flags().PROFILES_DIR
533
+ # Ensure profiles_dir is a string (may be PosixPath from default_profiles_dir())
534
+ if hasattr(profiles_dir, '__fspath__'):
535
+ profiles_dir = str(profiles_dir)
536
+ self.create_profiles_dir(profiles_dir)
537
+
538
+ try:
539
+ move_to_nearest_project_dir(self.args.project_dir)
540
+ in_project = True
541
+ except dbt_common.exceptions.DbtRuntimeError:
542
+ in_project = False
543
+
544
+ if in_project:
545
+ # If --profile was specified, it means use an existing profile, which is not
546
+ # applicable to this case
547
+ if self.args.profile:
548
+ raise DbtRuntimeError(
549
+ msg="Can not init existing project with specified profile, edit dbt_project.yml instead"
550
+ )
551
+
552
+ # v0.55.0: Ensure project-level .dvt/ structure exists
553
+ from dbt.config.compute import ComputeRegistry
554
+
555
+ # Create .dvt/ directory and jdbc_jars/
556
+ ComputeRegistry.ensure_jdbc_jars_dir(".")
557
+
558
+ # Ensure computes.yml exists at project level
559
+ registry = ComputeRegistry(project_dir=".")
560
+ registry.ensure_config_exists()
561
+
562
+ # Initialize metadata store if not already present
563
+ self._initialize_metadata_store(Path("."))
564
+
565
+ # When dbt init is run inside an existing project,
566
+ # just setup the user's profile.
567
+ if not self.args.skip_profile_setup:
568
+ # Get profile name from dbt_project.yml
569
+ profile_name = self.get_profile_name_from_current_project()
570
+ self.setup_profile(profile_name)
571
+ else:
572
+ # When dbt init is run outside of an existing project,
573
+ # create a new project and set up the user's profile.
574
+ project_name = self.get_valid_project_name()
575
+ project_path = Path(project_name)
576
+ if project_path.exists():
577
+ fire_event(ProjectNameAlreadyExists(name=project_name))
578
+ return
579
+
580
+ # If the user specified an existing profile to use, use it instead of generating a new one
581
+ user_profile_name = self.args.profile
582
+ if user_profile_name:
583
+ if not self.check_if_profile_exists(user_profile_name):
584
+ raise DbtRuntimeError(
585
+ msg="Could not find profile named '{}'".format(user_profile_name)
586
+ )
587
+ self.create_new_project(project_name, user_profile_name)
588
+ self.show_next_steps(project_name, 1)
589
+ else:
590
+ profile_name = project_name
591
+ # Create the profile after creating the project to avoid leaving a random profile
592
+ # if the former fails.
593
+ self.create_new_project(project_name, profile_name)
594
+
595
+ # DVT v0.5.1: Enhanced multi-connection init wizard
596
+ if not self.args.skip_profile_setup:
597
+ # Ask about multi-connection setup
598
+ if self.ask_for_multi_connection_setup():
599
+ num_conn = self.setup_multi_connection_profile(profile_name)
600
+ else:
601
+ self.setup_profile(profile_name)
602
+ num_conn = 1
603
+
604
+ self.show_next_steps(project_name, num_conn)
Binary file