dvt-core 1.11.0b4__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of dvt-core might be problematic. Click here for more details.

Files changed (261) hide show
  1. dvt/__init__.py +7 -0
  2. dvt/_pydantic_shim.py +26 -0
  3. dvt/adapters/__init__.py +16 -0
  4. dvt/adapters/multi_adapter_manager.py +268 -0
  5. dvt/artifacts/__init__.py +0 -0
  6. dvt/artifacts/exceptions/__init__.py +1 -0
  7. dvt/artifacts/exceptions/schemas.py +31 -0
  8. dvt/artifacts/resources/__init__.py +116 -0
  9. dvt/artifacts/resources/base.py +68 -0
  10. dvt/artifacts/resources/types.py +93 -0
  11. dvt/artifacts/resources/v1/analysis.py +10 -0
  12. dvt/artifacts/resources/v1/catalog.py +23 -0
  13. dvt/artifacts/resources/v1/components.py +275 -0
  14. dvt/artifacts/resources/v1/config.py +282 -0
  15. dvt/artifacts/resources/v1/documentation.py +11 -0
  16. dvt/artifacts/resources/v1/exposure.py +52 -0
  17. dvt/artifacts/resources/v1/function.py +53 -0
  18. dvt/artifacts/resources/v1/generic_test.py +32 -0
  19. dvt/artifacts/resources/v1/group.py +22 -0
  20. dvt/artifacts/resources/v1/hook.py +11 -0
  21. dvt/artifacts/resources/v1/macro.py +30 -0
  22. dvt/artifacts/resources/v1/metric.py +173 -0
  23. dvt/artifacts/resources/v1/model.py +146 -0
  24. dvt/artifacts/resources/v1/owner.py +10 -0
  25. dvt/artifacts/resources/v1/saved_query.py +112 -0
  26. dvt/artifacts/resources/v1/seed.py +42 -0
  27. dvt/artifacts/resources/v1/semantic_layer_components.py +72 -0
  28. dvt/artifacts/resources/v1/semantic_model.py +315 -0
  29. dvt/artifacts/resources/v1/singular_test.py +14 -0
  30. dvt/artifacts/resources/v1/snapshot.py +92 -0
  31. dvt/artifacts/resources/v1/source_definition.py +85 -0
  32. dvt/artifacts/resources/v1/sql_operation.py +10 -0
  33. dvt/artifacts/resources/v1/unit_test_definition.py +78 -0
  34. dvt/artifacts/schemas/__init__.py +0 -0
  35. dvt/artifacts/schemas/base.py +191 -0
  36. dvt/artifacts/schemas/batch_results.py +24 -0
  37. dvt/artifacts/schemas/catalog/__init__.py +12 -0
  38. dvt/artifacts/schemas/catalog/v1/__init__.py +0 -0
  39. dvt/artifacts/schemas/catalog/v1/catalog.py +60 -0
  40. dvt/artifacts/schemas/freshness/__init__.py +1 -0
  41. dvt/artifacts/schemas/freshness/v3/__init__.py +0 -0
  42. dvt/artifacts/schemas/freshness/v3/freshness.py +159 -0
  43. dvt/artifacts/schemas/manifest/__init__.py +2 -0
  44. dvt/artifacts/schemas/manifest/v12/__init__.py +0 -0
  45. dvt/artifacts/schemas/manifest/v12/manifest.py +212 -0
  46. dvt/artifacts/schemas/results.py +148 -0
  47. dvt/artifacts/schemas/run/__init__.py +2 -0
  48. dvt/artifacts/schemas/run/v5/__init__.py +0 -0
  49. dvt/artifacts/schemas/run/v5/run.py +184 -0
  50. dvt/artifacts/schemas/upgrades/__init__.py +4 -0
  51. dvt/artifacts/schemas/upgrades/upgrade_manifest.py +174 -0
  52. dvt/artifacts/schemas/upgrades/upgrade_manifest_dbt_version.py +2 -0
  53. dvt/artifacts/utils/validation.py +153 -0
  54. dvt/cli/__init__.py +1 -0
  55. dvt/cli/context.py +16 -0
  56. dvt/cli/exceptions.py +56 -0
  57. dvt/cli/flags.py +558 -0
  58. dvt/cli/main.py +971 -0
  59. dvt/cli/option_types.py +121 -0
  60. dvt/cli/options.py +79 -0
  61. dvt/cli/params.py +803 -0
  62. dvt/cli/requires.py +478 -0
  63. dvt/cli/resolvers.py +32 -0
  64. dvt/cli/types.py +40 -0
  65. dvt/clients/__init__.py +0 -0
  66. dvt/clients/checked_load.py +82 -0
  67. dvt/clients/git.py +164 -0
  68. dvt/clients/jinja.py +206 -0
  69. dvt/clients/jinja_static.py +245 -0
  70. dvt/clients/registry.py +192 -0
  71. dvt/clients/yaml_helper.py +68 -0
  72. dvt/compilation.py +833 -0
  73. dvt/compute/__init__.py +26 -0
  74. dvt/compute/base.py +288 -0
  75. dvt/compute/engines/__init__.py +13 -0
  76. dvt/compute/engines/duckdb_engine.py +368 -0
  77. dvt/compute/engines/spark_engine.py +273 -0
  78. dvt/compute/query_analyzer.py +212 -0
  79. dvt/compute/router.py +483 -0
  80. dvt/config/__init__.py +4 -0
  81. dvt/config/catalogs.py +95 -0
  82. dvt/config/compute_config.py +406 -0
  83. dvt/config/profile.py +411 -0
  84. dvt/config/profiles_v2.py +464 -0
  85. dvt/config/project.py +893 -0
  86. dvt/config/renderer.py +232 -0
  87. dvt/config/runtime.py +491 -0
  88. dvt/config/selectors.py +209 -0
  89. dvt/config/utils.py +78 -0
  90. dvt/connectors/.gitignore +6 -0
  91. dvt/connectors/README.md +306 -0
  92. dvt/connectors/catalog.yml +217 -0
  93. dvt/connectors/download_connectors.py +300 -0
  94. dvt/constants.py +29 -0
  95. dvt/context/__init__.py +0 -0
  96. dvt/context/base.py +746 -0
  97. dvt/context/configured.py +136 -0
  98. dvt/context/context_config.py +350 -0
  99. dvt/context/docs.py +82 -0
  100. dvt/context/exceptions_jinja.py +179 -0
  101. dvt/context/macro_resolver.py +195 -0
  102. dvt/context/macros.py +171 -0
  103. dvt/context/manifest.py +73 -0
  104. dvt/context/providers.py +2198 -0
  105. dvt/context/query_header.py +14 -0
  106. dvt/context/secret.py +59 -0
  107. dvt/context/target.py +74 -0
  108. dvt/contracts/__init__.py +0 -0
  109. dvt/contracts/files.py +413 -0
  110. dvt/contracts/graph/__init__.py +0 -0
  111. dvt/contracts/graph/manifest.py +1904 -0
  112. dvt/contracts/graph/metrics.py +98 -0
  113. dvt/contracts/graph/model_config.py +71 -0
  114. dvt/contracts/graph/node_args.py +42 -0
  115. dvt/contracts/graph/nodes.py +1806 -0
  116. dvt/contracts/graph/semantic_manifest.py +233 -0
  117. dvt/contracts/graph/unparsed.py +812 -0
  118. dvt/contracts/project.py +417 -0
  119. dvt/contracts/results.py +53 -0
  120. dvt/contracts/selection.py +23 -0
  121. dvt/contracts/sql.py +86 -0
  122. dvt/contracts/state.py +69 -0
  123. dvt/contracts/util.py +46 -0
  124. dvt/deprecations.py +347 -0
  125. dvt/deps/__init__.py +0 -0
  126. dvt/deps/base.py +153 -0
  127. dvt/deps/git.py +196 -0
  128. dvt/deps/local.py +80 -0
  129. dvt/deps/registry.py +131 -0
  130. dvt/deps/resolver.py +149 -0
  131. dvt/deps/tarball.py +121 -0
  132. dvt/docs/source/_ext/dbt_click.py +118 -0
  133. dvt/docs/source/conf.py +32 -0
  134. dvt/env_vars.py +64 -0
  135. dvt/event_time/event_time.py +40 -0
  136. dvt/event_time/sample_window.py +60 -0
  137. dvt/events/__init__.py +16 -0
  138. dvt/events/base_types.py +37 -0
  139. dvt/events/core_types_pb2.py +2 -0
  140. dvt/events/logging.py +109 -0
  141. dvt/events/types.py +2534 -0
  142. dvt/exceptions.py +1487 -0
  143. dvt/flags.py +89 -0
  144. dvt/graph/__init__.py +11 -0
  145. dvt/graph/cli.py +248 -0
  146. dvt/graph/graph.py +172 -0
  147. dvt/graph/queue.py +213 -0
  148. dvt/graph/selector.py +375 -0
  149. dvt/graph/selector_methods.py +976 -0
  150. dvt/graph/selector_spec.py +223 -0
  151. dvt/graph/thread_pool.py +18 -0
  152. dvt/hooks.py +21 -0
  153. dvt/include/README.md +49 -0
  154. dvt/include/__init__.py +3 -0
  155. dvt/include/global_project.py +4 -0
  156. dvt/include/starter_project/.gitignore +4 -0
  157. dvt/include/starter_project/README.md +15 -0
  158. dvt/include/starter_project/__init__.py +3 -0
  159. dvt/include/starter_project/analyses/.gitkeep +0 -0
  160. dvt/include/starter_project/dvt_project.yml +36 -0
  161. dvt/include/starter_project/macros/.gitkeep +0 -0
  162. dvt/include/starter_project/models/example/my_first_dbt_model.sql +27 -0
  163. dvt/include/starter_project/models/example/my_second_dbt_model.sql +6 -0
  164. dvt/include/starter_project/models/example/schema.yml +21 -0
  165. dvt/include/starter_project/seeds/.gitkeep +0 -0
  166. dvt/include/starter_project/snapshots/.gitkeep +0 -0
  167. dvt/include/starter_project/tests/.gitkeep +0 -0
  168. dvt/internal_deprecations.py +27 -0
  169. dvt/jsonschemas/__init__.py +3 -0
  170. dvt/jsonschemas/jsonschemas.py +309 -0
  171. dvt/jsonschemas/project/0.0.110.json +4717 -0
  172. dvt/jsonschemas/project/0.0.85.json +2015 -0
  173. dvt/jsonschemas/resources/0.0.110.json +2636 -0
  174. dvt/jsonschemas/resources/0.0.85.json +2536 -0
  175. dvt/jsonschemas/resources/latest.json +6773 -0
  176. dvt/links.py +4 -0
  177. dvt/materializations/__init__.py +0 -0
  178. dvt/materializations/incremental/__init__.py +0 -0
  179. dvt/materializations/incremental/microbatch.py +235 -0
  180. dvt/mp_context.py +8 -0
  181. dvt/node_types.py +37 -0
  182. dvt/parser/__init__.py +23 -0
  183. dvt/parser/analysis.py +21 -0
  184. dvt/parser/base.py +549 -0
  185. dvt/parser/common.py +267 -0
  186. dvt/parser/docs.py +52 -0
  187. dvt/parser/fixtures.py +51 -0
  188. dvt/parser/functions.py +30 -0
  189. dvt/parser/generic_test.py +100 -0
  190. dvt/parser/generic_test_builders.py +334 -0
  191. dvt/parser/hooks.py +119 -0
  192. dvt/parser/macros.py +137 -0
  193. dvt/parser/manifest.py +2204 -0
  194. dvt/parser/models.py +574 -0
  195. dvt/parser/partial.py +1179 -0
  196. dvt/parser/read_files.py +445 -0
  197. dvt/parser/schema_generic_tests.py +423 -0
  198. dvt/parser/schema_renderer.py +111 -0
  199. dvt/parser/schema_yaml_readers.py +936 -0
  200. dvt/parser/schemas.py +1467 -0
  201. dvt/parser/search.py +149 -0
  202. dvt/parser/seeds.py +28 -0
  203. dvt/parser/singular_test.py +20 -0
  204. dvt/parser/snapshots.py +44 -0
  205. dvt/parser/sources.py +557 -0
  206. dvt/parser/sql.py +63 -0
  207. dvt/parser/unit_tests.py +622 -0
  208. dvt/plugins/__init__.py +20 -0
  209. dvt/plugins/contracts.py +10 -0
  210. dvt/plugins/exceptions.py +2 -0
  211. dvt/plugins/manager.py +164 -0
  212. dvt/plugins/manifest.py +21 -0
  213. dvt/profiler.py +20 -0
  214. dvt/py.typed +1 -0
  215. dvt/runners/__init__.py +2 -0
  216. dvt/runners/exposure_runner.py +7 -0
  217. dvt/runners/no_op_runner.py +46 -0
  218. dvt/runners/saved_query_runner.py +7 -0
  219. dvt/selected_resources.py +8 -0
  220. dvt/task/__init__.py +0 -0
  221. dvt/task/base.py +504 -0
  222. dvt/task/build.py +197 -0
  223. dvt/task/clean.py +57 -0
  224. dvt/task/clone.py +162 -0
  225. dvt/task/compile.py +151 -0
  226. dvt/task/compute.py +366 -0
  227. dvt/task/debug.py +650 -0
  228. dvt/task/deps.py +280 -0
  229. dvt/task/docs/__init__.py +3 -0
  230. dvt/task/docs/generate.py +408 -0
  231. dvt/task/docs/index.html +250 -0
  232. dvt/task/docs/serve.py +28 -0
  233. dvt/task/freshness.py +323 -0
  234. dvt/task/function.py +122 -0
  235. dvt/task/group_lookup.py +46 -0
  236. dvt/task/init.py +374 -0
  237. dvt/task/list.py +237 -0
  238. dvt/task/printer.py +176 -0
  239. dvt/task/profiles.py +256 -0
  240. dvt/task/retry.py +175 -0
  241. dvt/task/run.py +1146 -0
  242. dvt/task/run_operation.py +142 -0
  243. dvt/task/runnable.py +802 -0
  244. dvt/task/seed.py +104 -0
  245. dvt/task/show.py +150 -0
  246. dvt/task/snapshot.py +57 -0
  247. dvt/task/sql.py +111 -0
  248. dvt/task/test.py +464 -0
  249. dvt/tests/fixtures/__init__.py +1 -0
  250. dvt/tests/fixtures/project.py +620 -0
  251. dvt/tests/util.py +651 -0
  252. dvt/tracking.py +529 -0
  253. dvt/utils/__init__.py +3 -0
  254. dvt/utils/artifact_upload.py +151 -0
  255. dvt/utils/utils.py +408 -0
  256. dvt/version.py +249 -0
  257. dvt_core-1.11.0b4.dist-info/METADATA +252 -0
  258. dvt_core-1.11.0b4.dist-info/RECORD +261 -0
  259. dvt_core-1.11.0b4.dist-info/WHEEL +5 -0
  260. dvt_core-1.11.0b4.dist-info/entry_points.txt +2 -0
  261. dvt_core-1.11.0b4.dist-info/top_level.txt +1 -0
dvt/cli/main.py ADDED
@@ -0,0 +1,971 @@
1
+ import functools
2
+ from copy import copy
3
+ from dataclasses import dataclass
4
+ from typing import Callable, List, Optional, Union
5
+
6
+ import click
7
+ from click.exceptions import BadOptionUsage
8
+ from click.exceptions import Exit as ClickExit
9
+ from click.exceptions import NoSuchOption, UsageError
10
+ from dvt.artifacts.schemas.catalog import CatalogArtifact
11
+ from dvt.artifacts.schemas.run import RunExecutionResult
12
+ from dvt.cli import params as p
13
+ from dvt.cli import requires
14
+ from dvt.cli.exceptions import DbtInternalException, DbtUsageException
15
+ from dvt.cli.requires import setup_manifest
16
+ from dvt.contracts.graph.manifest import Manifest
17
+ from dvt.mp_context import get_mp_context
18
+
19
+ from dbt.adapters.factory import register_adapter
20
+ from dbt_common.events.base_types import EventMsg
21
+
22
+
23
+ @dataclass
24
+ class dbtRunnerResult:
25
+ """Contains the result of an invocation of the dbtRunner"""
26
+
27
+ success: bool
28
+
29
+ exception: Optional[BaseException] = None
30
+ result: Union[
31
+ bool, # debug
32
+ CatalogArtifact, # docs generate
33
+ List[str], # list/ls
34
+ Manifest, # parse
35
+ None, # clean, deps, init, source
36
+ RunExecutionResult, # build, compile, run, seed, snapshot, test, run-operation
37
+ ] = None
38
+
39
+
40
+ # Programmatic invocation
41
+ class dbtRunner:
42
+ def __init__(
43
+ self,
44
+ manifest: Optional[Manifest] = None,
45
+ callbacks: Optional[List[Callable[[EventMsg], None]]] = None,
46
+ ) -> None:
47
+ self.manifest = manifest
48
+
49
+ if callbacks is None:
50
+ callbacks = []
51
+ self.callbacks = callbacks
52
+
53
+ def invoke(self, args: List[str], **kwargs) -> dbtRunnerResult:
54
+ try:
55
+ dbt_ctx = cli.make_context(cli.name, args.copy())
56
+ dbt_ctx.obj = {
57
+ "manifest": self.manifest,
58
+ "callbacks": self.callbacks,
59
+ "dbt_runner_command_args": args,
60
+ }
61
+
62
+ for key, value in kwargs.items():
63
+ dbt_ctx.params[key] = value
64
+ # Hack to set parameter source to custom string
65
+ dbt_ctx.set_parameter_source(key, "kwargs") # type: ignore
66
+
67
+ result, success = cli.invoke(dbt_ctx)
68
+ return dbtRunnerResult(
69
+ result=result,
70
+ success=success,
71
+ )
72
+ except requires.ResultExit as e:
73
+ return dbtRunnerResult(
74
+ result=e.result,
75
+ success=False,
76
+ )
77
+ except requires.ExceptionExit as e:
78
+ return dbtRunnerResult(
79
+ exception=e.exception,
80
+ success=False,
81
+ )
82
+ except (BadOptionUsage, NoSuchOption, UsageError) as e:
83
+ return dbtRunnerResult(
84
+ exception=DbtUsageException(e.message),
85
+ success=False,
86
+ )
87
+ except ClickExit as e:
88
+ if e.exit_code == 0:
89
+ return dbtRunnerResult(success=True)
90
+ return dbtRunnerResult(
91
+ exception=DbtInternalException(f"unhandled exit code {e.exit_code}"),
92
+ success=False,
93
+ )
94
+ except BaseException as e:
95
+ return dbtRunnerResult(
96
+ exception=e,
97
+ success=False,
98
+ )
99
+
100
+
101
+ # approach from https://github.com/pallets/click/issues/108#issuecomment-280489786
102
+ def global_flags(func):
103
+ @p.cache_selected_only
104
+ @p.debug
105
+ @p.defer
106
+ @p.deprecated_defer
107
+ @p.defer_state
108
+ @p.deprecated_favor_state
109
+ @p.deprecated_print
110
+ @p.deprecated_state
111
+ @p.fail_fast
112
+ @p.favor_state
113
+ @p.indirect_selection
114
+ @p.log_cache_events
115
+ @p.log_file_max_bytes
116
+ @p.log_format
117
+ @p.log_format_file
118
+ @p.log_level
119
+ @p.log_level_file
120
+ @p.log_path
121
+ @p.macro_debugging
122
+ @p.partial_parse
123
+ @p.partial_parse_file_path
124
+ @p.partial_parse_file_diff
125
+ @p.populate_cache
126
+ @p.print
127
+ @p.printer_width
128
+ @p.profile
129
+ @p.quiet
130
+ @p.record_timing_info
131
+ @p.send_anonymous_usage_stats
132
+ @p.single_threaded
133
+ @p.show_all_deprecations
134
+ @p.state
135
+ @p.static_parser
136
+ @p.target
137
+ @p.use_colors
138
+ @p.use_colors_file
139
+ @p.use_experimental_parser
140
+ @p.version
141
+ @p.version_check
142
+ @p.warn_error
143
+ @p.warn_error_options
144
+ @p.write_json
145
+ @p.use_fast_test_edges
146
+ @p.upload_artifacts
147
+ @functools.wraps(func)
148
+ def wrapper(*args, **kwargs):
149
+ return func(*args, **kwargs)
150
+
151
+ return wrapper
152
+
153
+
154
+ # dbt
155
+ @click.group(
156
+ context_settings={"help_option_names": ["-h", "--help"]},
157
+ invoke_without_command=True,
158
+ no_args_is_help=True,
159
+ epilog="Specify one of these sub-commands and you can find more help from there.",
160
+ )
161
+ @click.pass_context
162
+ @global_flags
163
+ @p.show_resource_report
164
+ def cli(ctx, **kwargs):
165
+ """An ELT tool for managing your SQL transformations and data models.
166
+ For more documentation on these commands, visit: docs.getdbt.com
167
+ """
168
+
169
+
170
+ # dbt build
171
+ @cli.command("build")
172
+ @click.pass_context
173
+ @global_flags
174
+ @p.empty
175
+ @p.event_time_start
176
+ @p.event_time_end
177
+ @p.exclude
178
+ @p.export_saved_queries
179
+ @p.full_refresh
180
+ @p.deprecated_include_saved_query
181
+ @p.profiles_dir
182
+ @p.project_dir
183
+ @p.resource_type
184
+ @p.exclude_resource_type
185
+ @p.sample
186
+ @p.select
187
+ @p.selector
188
+ @p.show
189
+ @p.store_failures
190
+ @p.target_path
191
+ @p.threads
192
+ @p.vars
193
+ @requires.postflight
194
+ @requires.preflight
195
+ @requires.profile
196
+ @requires.project
197
+ @requires.catalogs
198
+ @requires.runtime_config
199
+ @requires.manifest
200
+ def build(ctx, **kwargs):
201
+ """Run all seeds, models, snapshots, and tests in DAG order"""
202
+ from dvt.task.build import BuildTask
203
+
204
+ task = BuildTask(
205
+ ctx.obj["flags"],
206
+ ctx.obj["runtime_config"],
207
+ ctx.obj["manifest"],
208
+ )
209
+
210
+ results = task.run()
211
+ success = task.interpret_results(results)
212
+ return results, success
213
+
214
+
215
+ # dbt clean
216
+ @cli.command("clean")
217
+ @click.pass_context
218
+ @global_flags
219
+ @p.clean_project_files_only
220
+ @p.profiles_dir
221
+ @p.project_dir
222
+ @p.target_path
223
+ @p.vars
224
+ @requires.postflight
225
+ @requires.preflight
226
+ @requires.unset_profile
227
+ @requires.project
228
+ def clean(ctx, **kwargs):
229
+ """Delete all folders in the clean-targets list (usually the dbt_packages and target directories.)"""
230
+ from dvt.task.clean import CleanTask
231
+
232
+ with CleanTask(ctx.obj["flags"], ctx.obj["project"]) as task:
233
+ results = task.run()
234
+ success = task.interpret_results(results)
235
+ return results, success
236
+
237
+
238
+ # dbt docs
239
+ @cli.group()
240
+ @click.pass_context
241
+ @global_flags
242
+ def docs(ctx, **kwargs):
243
+ """Generate or serve the documentation website for your project"""
244
+
245
+
246
+ # dbt docs generate
247
+ @docs.command("generate")
248
+ @click.pass_context
249
+ @global_flags
250
+ @p.compile_docs
251
+ @p.exclude
252
+ @p.profiles_dir
253
+ @p.project_dir
254
+ @p.select
255
+ @p.selector
256
+ @p.empty_catalog
257
+ @p.static
258
+ @p.target_path
259
+ @p.threads
260
+ @p.vars
261
+ @requires.postflight
262
+ @requires.preflight
263
+ @requires.profile
264
+ @requires.project
265
+ @requires.runtime_config
266
+ @requires.manifest(write=False)
267
+ def docs_generate(ctx, **kwargs):
268
+ """Generate the documentation website for your project"""
269
+ from dvt.task.docs.generate import GenerateTask
270
+
271
+ task = GenerateTask(
272
+ ctx.obj["flags"],
273
+ ctx.obj["runtime_config"],
274
+ ctx.obj["manifest"],
275
+ )
276
+
277
+ results = task.run()
278
+ success = task.interpret_results(results)
279
+ return results, success
280
+
281
+
282
+ # dbt docs serve
283
+ @docs.command("serve")
284
+ @click.pass_context
285
+ @global_flags
286
+ @p.browser
287
+ @p.host
288
+ @p.port
289
+ @p.profiles_dir
290
+ @p.project_dir
291
+ @p.target_path
292
+ @p.vars
293
+ @requires.postflight
294
+ @requires.preflight
295
+ @requires.profile
296
+ @requires.project
297
+ @requires.runtime_config
298
+ def docs_serve(ctx, **kwargs):
299
+ """Serve the documentation website for your project"""
300
+ from dvt.task.docs.serve import ServeTask
301
+
302
+ task = ServeTask(
303
+ ctx.obj["flags"],
304
+ ctx.obj["runtime_config"],
305
+ )
306
+
307
+ results = task.run()
308
+ success = task.interpret_results(results)
309
+ return results, success
310
+
311
+
312
+ # dbt compile
313
+ @cli.command("compile")
314
+ @click.pass_context
315
+ @global_flags
316
+ @p.exclude
317
+ @p.full_refresh
318
+ @p.show_output_format
319
+ @p.introspect
320
+ @p.profiles_dir
321
+ @p.project_dir
322
+ @p.empty
323
+ @p.select
324
+ @p.selector
325
+ @p.inline
326
+ @p.compile_inject_ephemeral_ctes
327
+ @p.target_path
328
+ @p.threads
329
+ @p.vars
330
+ @requires.postflight
331
+ @requires.preflight
332
+ @requires.profile
333
+ @requires.project
334
+ @requires.runtime_config
335
+ @requires.manifest
336
+ def compile(ctx, **kwargs):
337
+ """Generates executable SQL from source, model, test, and analysis files. Compiled SQL files are written to the
338
+ target/ directory."""
339
+ from dvt.task.compile import CompileTask
340
+
341
+ task = CompileTask(
342
+ ctx.obj["flags"],
343
+ ctx.obj["runtime_config"],
344
+ ctx.obj["manifest"],
345
+ )
346
+
347
+ results = task.run()
348
+ success = task.interpret_results(results)
349
+ return results, success
350
+
351
+
352
+ # dbt show
353
+ @cli.command("show")
354
+ @click.pass_context
355
+ @global_flags
356
+ @p.exclude
357
+ @p.full_refresh
358
+ @p.show_output_format
359
+ @p.show_limit
360
+ @p.introspect
361
+ @p.profiles_dir
362
+ @p.project_dir
363
+ @p.select
364
+ @p.selector
365
+ @p.inline
366
+ @p.inline_direct
367
+ @p.target_path
368
+ @p.threads
369
+ @p.vars
370
+ @requires.postflight
371
+ @requires.preflight
372
+ @requires.profile
373
+ @requires.project
374
+ @requires.runtime_config
375
+ def show(ctx, **kwargs):
376
+ """Generates executable SQL for a named resource or inline query, runs that SQL, and returns a preview of the
377
+ results. Does not materialize anything to the warehouse."""
378
+ from dvt.task.show import ShowTask, ShowTaskDirect
379
+
380
+ if ctx.obj["flags"].inline_direct:
381
+ # Issue the inline query directly, with no templating. Does not require
382
+ # loading the manifest.
383
+ register_adapter(ctx.obj["runtime_config"], get_mp_context())
384
+ task = ShowTaskDirect(
385
+ ctx.obj["flags"],
386
+ ctx.obj["runtime_config"],
387
+ )
388
+ else:
389
+ setup_manifest(ctx)
390
+ task = ShowTask(
391
+ ctx.obj["flags"],
392
+ ctx.obj["runtime_config"],
393
+ ctx.obj["manifest"],
394
+ )
395
+
396
+ results = task.run()
397
+ success = task.interpret_results(results)
398
+ return results, success
399
+
400
+
401
+ # dbt debug
402
+ @cli.command("debug")
403
+ @click.pass_context
404
+ @global_flags
405
+ @p.all_profiles
406
+ @p.debug_connection
407
+ @p.config_dir
408
+ @p.profiles_dir_exists_false
409
+ @p.project_dir
410
+ @p.vars
411
+ @requires.postflight
412
+ @requires.preflight
413
+ def debug(ctx, **kwargs):
414
+ """Show information on the current dbt environment and check dependencies, then test the database connection. Not to be confused with the --debug option which increases verbosity."""
415
+ from dvt.task.debug import DebugTask
416
+
417
+ task = DebugTask(
418
+ ctx.obj["flags"],
419
+ )
420
+
421
+ results = task.run()
422
+ success = task.interpret_results(results)
423
+ return results, success
424
+
425
+
426
+ # dvt profiles
427
+ @cli.group()
428
+ @click.pass_context
429
+ @global_flags
430
+ def profiles(ctx, **kwargs):
431
+ """DVT extension: Manage and test database profiles"""
432
+
433
+
434
+ # dvt profiles list
435
+ @profiles.command("list")
436
+ @click.pass_context
437
+ @global_flags
438
+ @p.profiles_dir
439
+ @p.project_dir
440
+ @requires.postflight
441
+ @requires.preflight
442
+ def profiles_list(ctx, **kwargs):
443
+ """List all configured profiles"""
444
+ from dvt.task.profiles import ProfilesListTask
445
+
446
+ task = ProfilesListTask(ctx.obj["flags"])
447
+ results = task.run()
448
+ success = task.interpret_results(results)
449
+ return results, success
450
+
451
+
452
+ # dvt profiles show
453
+ @profiles.command("show")
454
+ @click.argument("profile_name")
455
+ @click.pass_context
456
+ @global_flags
457
+ @p.profiles_dir
458
+ @p.project_dir
459
+ @requires.postflight
460
+ @requires.preflight
461
+ def profiles_show(ctx, profile_name, **kwargs):
462
+ """Show details of a specific profile"""
463
+ from dvt.task.profiles import ProfilesShowTask
464
+
465
+ task = ProfilesShowTask(ctx.obj["flags"], profile_name)
466
+ results = task.run()
467
+ success = task.interpret_results(results)
468
+ return results, success
469
+
470
+
471
+ # dvt profiles test
472
+ @profiles.command("test")
473
+ @click.argument("profile_name", required=False)
474
+ @click.pass_context
475
+ @global_flags
476
+ @p.profiles_dir
477
+ @p.project_dir
478
+ @requires.postflight
479
+ @requires.preflight
480
+ def profiles_test(ctx, profile_name, **kwargs):
481
+ """Test one or all profiles (if no profile_name provided, tests all)"""
482
+ from dvt.task.profiles import ProfilesTestTask
483
+
484
+ task = ProfilesTestTask(ctx.obj["flags"], profile_name)
485
+ results = task.run()
486
+ success = task.interpret_results(results)
487
+ return results, success
488
+
489
+
490
+ # dvt compute
491
+ @cli.group()
492
+ @click.pass_context
493
+ @global_flags
494
+ def compute(ctx, **kwargs):
495
+ """DVT extension: Manage compute layer configuration"""
496
+
497
+
498
+ # dvt compute show
499
+ @compute.command("show")
500
+ @click.pass_context
501
+ @global_flags
502
+ @p.project_dir
503
+ @requires.postflight
504
+ @requires.preflight
505
+ def compute_show(ctx, **kwargs):
506
+ """Show compute layer configuration"""
507
+ from dvt.task.compute import ComputeShowTask
508
+
509
+ task = ComputeShowTask(ctx.obj["flags"])
510
+ results = task.run()
511
+ success = task.interpret_results(results)
512
+ return results, success
513
+
514
+
515
+ # dvt compute engines
516
+ @compute.command("engines")
517
+ @click.pass_context
518
+ @global_flags
519
+ @p.project_dir
520
+ @requires.postflight
521
+ @requires.preflight
522
+ def compute_engines(ctx, **kwargs):
523
+ """List available compute engines"""
524
+ from dvt.task.compute import ComputeEnginesTask
525
+
526
+ task = ComputeEnginesTask(ctx.obj["flags"])
527
+ results = task.run()
528
+ success = task.interpret_results(results)
529
+ return results, success
530
+
531
+
532
+ # dvt compute test
533
+ @compute.command("test")
534
+ @click.argument("engine_name", required=False)
535
+ @click.pass_context
536
+ @global_flags
537
+ @p.project_dir
538
+ @requires.postflight
539
+ @requires.preflight
540
+ def compute_test(ctx, engine_name, **kwargs):
541
+ """Test compute engines (if no engine_name provided, tests all)"""
542
+ from dvt.task.compute import ComputeTestTask
543
+
544
+ task = ComputeTestTask(ctx.obj["flags"], engine_name)
545
+ results = task.run()
546
+ success = task.interpret_results(results)
547
+ return results, success
548
+
549
+
550
+ # dbt deps
551
+ @cli.command("deps")
552
+ @click.pass_context
553
+ @global_flags
554
+ @p.profiles_dir_exists_false
555
+ @p.project_dir
556
+ @p.vars
557
+ @p.source
558
+ @p.lock
559
+ @p.upgrade
560
+ @p.add_package
561
+ @requires.postflight
562
+ @requires.preflight
563
+ @requires.unset_profile
564
+ @requires.project
565
+ def deps(ctx, **kwargs):
566
+ """Install dbt packages specified.
567
+ In the following case, a new `package-lock.yml` will be generated and the packages are installed:
568
+ - user updated the packages.yml
569
+ - user specify the flag --update, which means for packages that are specified as a
570
+ range, dbt-core will try to install the newer version
571
+ Otherwise, deps will use `package-lock.yml` as source of truth to install packages.
572
+
573
+ There is a way to add new packages by providing an `--add-package` flag to deps command
574
+ which will allow user to specify a package they want to add in the format of packagename@version.
575
+ """
576
+ from dvt.task.deps import DepsTask
577
+
578
+ flags = ctx.obj["flags"]
579
+ if flags.ADD_PACKAGE:
580
+ if not flags.ADD_PACKAGE["version"] and flags.SOURCE != "local":
581
+ raise BadOptionUsage(
582
+ message=f"Version is required in --add-package when a package when source is {flags.SOURCE}",
583
+ option_name="--add-package",
584
+ )
585
+ with DepsTask(flags, ctx.obj["project"]) as task:
586
+ results = task.run()
587
+ success = task.interpret_results(results)
588
+ return results, success
589
+
590
+
591
+ # dbt init
592
+ @cli.command("init")
593
+ @click.pass_context
594
+ @global_flags
595
+ # for backwards compatibility, accept 'project_name' as an optional positional argument
596
+ @click.argument("project_name", required=False)
597
+ @p.profiles_dir_exists_false
598
+ @p.project_dir
599
+ @p.skip_profile_setup
600
+ @p.vars
601
+ @requires.postflight
602
+ @requires.preflight
603
+ def init(ctx, **kwargs):
604
+ """Initialize a new dbt project."""
605
+ from dvt.task.init import InitTask
606
+
607
+ with InitTask(ctx.obj["flags"]) as task:
608
+ results = task.run()
609
+ success = task.interpret_results(results)
610
+ return results, success
611
+
612
+
613
+ # dbt list
614
+ @cli.command("list")
615
+ @click.pass_context
616
+ @global_flags
617
+ @p.exclude
618
+ @p.models
619
+ @p.output
620
+ @p.output_keys
621
+ @p.profiles_dir
622
+ @p.project_dir
623
+ @p.resource_type
624
+ @p.exclude_resource_type
625
+ @p.raw_select
626
+ @p.selector
627
+ @p.target_path
628
+ @p.vars
629
+ @requires.postflight
630
+ @requires.preflight
631
+ @requires.profile
632
+ @requires.project
633
+ @requires.runtime_config
634
+ @requires.manifest
635
+ def list(ctx, **kwargs):
636
+ """List the resources in your project"""
637
+ from dvt.task.list import ListTask
638
+
639
+ task = ListTask(
640
+ ctx.obj["flags"],
641
+ ctx.obj["runtime_config"],
642
+ ctx.obj["manifest"],
643
+ )
644
+
645
+ results = task.run()
646
+ success = task.interpret_results(results)
647
+ return results, success
648
+
649
+
650
+ # Alias "list" to "ls"
651
+ ls = copy(cli.commands["list"])
652
+ ls.hidden = True
653
+ cli.add_command(ls, "ls")
654
+
655
+
656
+ # dbt parse
657
+ @cli.command("parse")
658
+ @click.pass_context
659
+ @global_flags
660
+ @p.profiles_dir
661
+ @p.project_dir
662
+ @p.target_path
663
+ @p.threads
664
+ @p.vars
665
+ @requires.postflight
666
+ @requires.preflight
667
+ @requires.profile
668
+ @requires.project
669
+ @requires.catalogs
670
+ @requires.runtime_config
671
+ @requires.manifest(write_perf_info=True)
672
+ def parse(ctx, **kwargs):
673
+ """Parses the project and provides information on performance"""
674
+ # manifest generation and writing happens in @requires.manifest
675
+ return ctx.obj["manifest"], True
676
+
677
+
678
+ # dbt run
679
+ @cli.command("run")
680
+ @click.pass_context
681
+ @global_flags
682
+ @p.exclude
683
+ @p.full_refresh
684
+ @p.profiles_dir
685
+ @p.project_dir
686
+ @p.empty
687
+ @p.event_time_start
688
+ @p.event_time_end
689
+ @p.sample
690
+ @p.select
691
+ @p.selector
692
+ @p.target_path
693
+ @p.threads
694
+ @p.vars
695
+ @requires.postflight
696
+ @requires.preflight
697
+ @requires.profile
698
+ @requires.project
699
+ @requires.catalogs
700
+ @requires.runtime_config
701
+ @requires.manifest
702
+ def run(ctx, **kwargs):
703
+ """Compile SQL and execute against the current target database."""
704
+ from dvt.task.run import RunTask
705
+
706
+ task = RunTask(
707
+ ctx.obj["flags"],
708
+ ctx.obj["runtime_config"],
709
+ ctx.obj["manifest"],
710
+ )
711
+
712
+ results = task.run()
713
+ success = task.interpret_results(results)
714
+ return results, success
715
+
716
+
717
+ # dbt retry
718
+ @cli.command("retry")
719
+ @click.pass_context
720
+ @global_flags
721
+ @p.project_dir
722
+ @p.profiles_dir
723
+ @p.vars
724
+ @p.target_path
725
+ @p.threads
726
+ @p.full_refresh
727
+ @requires.postflight
728
+ @requires.preflight
729
+ @requires.profile
730
+ @requires.project
731
+ @requires.runtime_config
732
+ def retry(ctx, **kwargs):
733
+ """Retry the nodes that failed in the previous run."""
734
+ from dvt.task.retry import RetryTask
735
+
736
+ # Retry will parse manifest inside the task after we consolidate the flags
737
+ task = RetryTask(
738
+ ctx.obj["flags"],
739
+ ctx.obj["runtime_config"],
740
+ )
741
+
742
+ results = task.run()
743
+ success = task.interpret_results(results)
744
+ return results, success
745
+
746
+
747
+ # dbt clone
748
+ @cli.command("clone")
749
+ @click.pass_context
750
+ @global_flags
751
+ @p.exclude
752
+ @p.full_refresh
753
+ @p.profiles_dir
754
+ @p.project_dir
755
+ @p.resource_type
756
+ @p.exclude_resource_type
757
+ @p.select
758
+ @p.selector
759
+ @p.target_path
760
+ @p.threads
761
+ @p.vars
762
+ @requires.preflight
763
+ @requires.profile
764
+ @requires.project
765
+ @requires.runtime_config
766
+ @requires.manifest
767
+ @requires.postflight
768
+ def clone(ctx, **kwargs):
769
+ """Create clones of selected nodes based on their location in the manifest provided to --state."""
770
+ from dvt.task.clone import CloneTask
771
+
772
+ task = CloneTask(
773
+ ctx.obj["flags"],
774
+ ctx.obj["runtime_config"],
775
+ ctx.obj["manifest"],
776
+ )
777
+
778
+ results = task.run()
779
+ success = task.interpret_results(results)
780
+ return results, success
781
+
782
+
783
+ # dbt run operation
784
+ @cli.command("run-operation")
785
+ @click.pass_context
786
+ @global_flags
787
+ @click.argument("macro")
788
+ @p.args
789
+ @p.profiles_dir
790
+ @p.project_dir
791
+ @p.target_path
792
+ @p.threads
793
+ @p.vars
794
+ @requires.postflight
795
+ @requires.preflight
796
+ @requires.profile
797
+ @requires.project
798
+ @requires.runtime_config
799
+ @requires.manifest
800
+ def run_operation(ctx, **kwargs):
801
+ """Run the named macro with any supplied arguments."""
802
+ from dvt.task.run_operation import RunOperationTask
803
+
804
+ task = RunOperationTask(
805
+ ctx.obj["flags"],
806
+ ctx.obj["runtime_config"],
807
+ ctx.obj["manifest"],
808
+ )
809
+
810
+ results = task.run()
811
+ success = task.interpret_results(results)
812
+ return results, success
813
+
814
+
815
+ # dbt seed
816
+ @cli.command("seed")
817
+ @click.pass_context
818
+ @global_flags
819
+ @p.exclude
820
+ @p.full_refresh
821
+ @p.profiles_dir
822
+ @p.project_dir
823
+ @p.select
824
+ @p.selector
825
+ @p.show
826
+ @p.target_path
827
+ @p.threads
828
+ @p.vars
829
+ @requires.postflight
830
+ @requires.preflight
831
+ @requires.profile
832
+ @requires.project
833
+ @requires.catalogs
834
+ @requires.runtime_config
835
+ @requires.manifest
836
+ def seed(ctx, **kwargs):
837
+ """Load data from csv files into your data warehouse."""
838
+ from dvt.task.seed import SeedTask
839
+
840
+ task = SeedTask(
841
+ ctx.obj["flags"],
842
+ ctx.obj["runtime_config"],
843
+ ctx.obj["manifest"],
844
+ )
845
+ results = task.run()
846
+ success = task.interpret_results(results)
847
+ return results, success
848
+
849
+
850
+ # dbt snapshot
851
+ @cli.command("snapshot")
852
+ @click.pass_context
853
+ @global_flags
854
+ @p.empty
855
+ @p.exclude
856
+ @p.profiles_dir
857
+ @p.project_dir
858
+ @p.select
859
+ @p.selector
860
+ @p.target_path
861
+ @p.threads
862
+ @p.vars
863
+ @requires.postflight
864
+ @requires.preflight
865
+ @requires.profile
866
+ @requires.project
867
+ @requires.catalogs
868
+ @requires.runtime_config
869
+ @requires.manifest
870
+ def snapshot(ctx, **kwargs):
871
+ """Execute snapshots defined in your project"""
872
+ from dvt.task.snapshot import SnapshotTask
873
+
874
+ task = SnapshotTask(
875
+ ctx.obj["flags"],
876
+ ctx.obj["runtime_config"],
877
+ ctx.obj["manifest"],
878
+ )
879
+
880
+ results = task.run()
881
+ success = task.interpret_results(results)
882
+ return results, success
883
+
884
+
885
+ # dbt source
886
+ @cli.group()
887
+ @click.pass_context
888
+ @global_flags
889
+ def source(ctx, **kwargs):
890
+ """Manage your project's sources"""
891
+
892
+
893
+ # dbt source freshness
894
+ @source.command("freshness")
895
+ @click.pass_context
896
+ @global_flags
897
+ @p.exclude
898
+ @p.output_path # TODO: Is this ok to re-use? We have three different output params, how much can we consolidate?
899
+ @p.profiles_dir
900
+ @p.project_dir
901
+ @p.select
902
+ @p.selector
903
+ @p.target_path
904
+ @p.threads
905
+ @p.vars
906
+ @requires.postflight
907
+ @requires.preflight
908
+ @requires.profile
909
+ @requires.project
910
+ @requires.runtime_config
911
+ @requires.manifest
912
+ def freshness(ctx, **kwargs):
913
+ """check the current freshness of the project's sources"""
914
+ from dvt.task.freshness import FreshnessTask
915
+
916
+ task = FreshnessTask(
917
+ ctx.obj["flags"],
918
+ ctx.obj["runtime_config"],
919
+ ctx.obj["manifest"],
920
+ )
921
+
922
+ results = task.run()
923
+ success = task.interpret_results(results)
924
+ return results, success
925
+
926
+
927
+ # Alias "source freshness" to "snapshot-freshness"
928
+ snapshot_freshness = copy(cli.commands["source"].commands["freshness"]) # type: ignore
929
+ snapshot_freshness.hidden = True
930
+ cli.commands["source"].add_command(snapshot_freshness, "snapshot-freshness") # type: ignore
931
+
932
+
933
+ # dbt test
934
+ @cli.command("test")
935
+ @click.pass_context
936
+ @global_flags
937
+ @p.exclude
938
+ @p.resource_type
939
+ @p.exclude_resource_type
940
+ @p.profiles_dir
941
+ @p.project_dir
942
+ @p.select
943
+ @p.selector
944
+ @p.store_failures
945
+ @p.target_path
946
+ @p.threads
947
+ @p.vars
948
+ @requires.postflight
949
+ @requires.preflight
950
+ @requires.profile
951
+ @requires.project
952
+ @requires.runtime_config
953
+ @requires.manifest
954
+ def test(ctx, **kwargs):
955
+ """Runs tests on data in deployed models. Run this after `dbt run`"""
956
+ from dvt.task.test import TestTask
957
+
958
+ task = TestTask(
959
+ ctx.obj["flags"],
960
+ ctx.obj["runtime_config"],
961
+ ctx.obj["manifest"],
962
+ )
963
+
964
+ results = task.run()
965
+ success = task.interpret_results(results)
966
+ return results, success
967
+
968
+
969
+ # Support running as a module
970
+ if __name__ == "__main__":
971
+ cli()