dagster-dbt 0.27.16__py3-none-any.whl → 0.28.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dagster_dbt/asset_utils.py +48 -22
- dagster_dbt/cloud/asset_defs.py +1 -21
- dagster_dbt/components/dbt_project/component.py +136 -43
- dagster_dbt/components/dbt_project/scaffolder.py +37 -22
- dagster_dbt/core/resource.py +8 -5
- dagster_dbt/dagster_dbt_translator.py +30 -59
- dagster_dbt/dbt_project_manager.py +170 -0
- dagster_dbt/freshness_builder.py +6 -2
- dagster_dbt/version.py +1 -1
- {dagster_dbt-0.27.16.dist-info → dagster_dbt-0.28.1.dist-info}/METADATA +3 -2
- {dagster_dbt-0.27.16.dist-info → dagster_dbt-0.28.1.dist-info}/RECORD +15 -14
- {dagster_dbt-0.27.16.dist-info → dagster_dbt-0.28.1.dist-info}/WHEEL +0 -0
- {dagster_dbt-0.27.16.dist-info → dagster_dbt-0.28.1.dist-info}/entry_points.txt +0 -0
- {dagster_dbt-0.27.16.dist-info → dagster_dbt-0.28.1.dist-info}/licenses/LICENSE +0 -0
- {dagster_dbt-0.27.16.dist-info → dagster_dbt-0.28.1.dist-info}/top_level.txt +0 -0
dagster_dbt/asset_utils.py
CHANGED
|
@@ -1,11 +1,14 @@
|
|
|
1
1
|
import hashlib
|
|
2
2
|
import os
|
|
3
|
+
import shutil
|
|
4
|
+
import tempfile
|
|
3
5
|
import textwrap
|
|
4
6
|
from collections import defaultdict
|
|
5
7
|
from collections.abc import Iterable, Mapping, Sequence
|
|
6
8
|
from pathlib import Path
|
|
7
9
|
from typing import TYPE_CHECKING, AbstractSet, Annotated, Any, Final, Optional, Union # noqa: UP035
|
|
8
10
|
|
|
11
|
+
import yaml
|
|
9
12
|
from dagster import (
|
|
10
13
|
AssetCheckKey,
|
|
11
14
|
AssetCheckSpec,
|
|
@@ -18,7 +21,6 @@ from dagster import (
|
|
|
18
21
|
DagsterInvalidDefinitionError,
|
|
19
22
|
DagsterInvariantViolationError,
|
|
20
23
|
DefaultScheduleStatus,
|
|
21
|
-
LegacyFreshnessPolicy,
|
|
22
24
|
OpExecutionContext,
|
|
23
25
|
RunConfig,
|
|
24
26
|
ScheduleDefinition,
|
|
@@ -33,6 +35,7 @@ from dagster._core.definitions.metadata import TableMetadataSet
|
|
|
33
35
|
from dagster._core.errors import DagsterInvalidPropertyError
|
|
34
36
|
from dagster._core.types.dagster_type import Nothing
|
|
35
37
|
from dagster._record import ImportFrom, record
|
|
38
|
+
from dagster_shared.record import replace
|
|
36
39
|
|
|
37
40
|
from dagster_dbt.dbt_project import DbtProject
|
|
38
41
|
from dagster_dbt.metadata_set import DbtMetadataSet
|
|
@@ -56,6 +59,10 @@ DBT_DEFAULT_SELECTOR = ""
|
|
|
56
59
|
DBT_INDIRECT_SELECTION_ENV: Final[str] = "DBT_INDIRECT_SELECTION"
|
|
57
60
|
DBT_EMPTY_INDIRECT_SELECTION: Final[str] = "empty"
|
|
58
61
|
|
|
62
|
+
# Threshold for switching to selector file to avoid CLI argument length limits
|
|
63
|
+
# https://github.com/dagster-io/dagster/issues/16997
|
|
64
|
+
_SELECTION_ARGS_THRESHOLD: Final[int] = 200
|
|
65
|
+
|
|
59
66
|
DUPLICATE_ASSET_KEY_ERROR_MESSAGE = (
|
|
60
67
|
"The following dbt resources are configured with identical Dagster asset keys."
|
|
61
68
|
" Please ensure that each dbt resource generates a unique Dagster asset key."
|
|
@@ -168,7 +175,9 @@ def get_asset_keys_by_output_name_for_source(
|
|
|
168
175
|
raise KeyError(f"Could not find a dbt source with name: {source_name}")
|
|
169
176
|
|
|
170
177
|
return {
|
|
171
|
-
dagster_name_fn(value): dagster_dbt_translator.get_asset_spec(
|
|
178
|
+
dagster_name_fn(value): dagster_dbt_translator.get_asset_spec(
|
|
179
|
+
manifest, unique_id, dbt_project
|
|
180
|
+
).key
|
|
172
181
|
for unique_id, value in matching.items()
|
|
173
182
|
}
|
|
174
183
|
|
|
@@ -442,6 +451,10 @@ def get_updated_cli_invocation_params_for_context(
|
|
|
442
451
|
[assets_def]
|
|
443
452
|
)
|
|
444
453
|
|
|
454
|
+
# Get project_dir from dbt_project if available
|
|
455
|
+
project_dir = Path(dbt_project.project_dir) if dbt_project else None
|
|
456
|
+
target_project = dbt_project
|
|
457
|
+
|
|
445
458
|
selection_args, indirect_selection_override = get_subset_selection_for_context(
|
|
446
459
|
context=context,
|
|
447
460
|
manifest=manifest,
|
|
@@ -451,17 +464,49 @@ def get_updated_cli_invocation_params_for_context(
|
|
|
451
464
|
dagster_dbt_translator=dagster_dbt_translator,
|
|
452
465
|
current_dbt_indirect_selection_env=indirect_selection,
|
|
453
466
|
)
|
|
467
|
+
if (
|
|
468
|
+
selection_args[0] == "--select"
|
|
469
|
+
and project_dir
|
|
470
|
+
and len(resources := selection_args[1].split(" ")) > _SELECTION_ARGS_THRESHOLD
|
|
471
|
+
):
|
|
472
|
+
temp_project_dir = tempfile.mkdtemp()
|
|
473
|
+
shutil.copytree(project_dir, temp_project_dir, dirs_exist_ok=True)
|
|
474
|
+
selectors_path = Path(temp_project_dir) / "selectors.yml"
|
|
475
|
+
|
|
476
|
+
# Delete any existing selectors, we need to create our own
|
|
477
|
+
if selectors_path.exists():
|
|
478
|
+
selectors_path.unlink()
|
|
479
|
+
|
|
480
|
+
selector_name = f"dagster_run_{context.run_id}"
|
|
481
|
+
temp_selectors = {
|
|
482
|
+
"selectors": [
|
|
483
|
+
{
|
|
484
|
+
"name": selector_name,
|
|
485
|
+
"definition": {"union": list(resources)},
|
|
486
|
+
}
|
|
487
|
+
]
|
|
488
|
+
}
|
|
489
|
+
selectors_path.write_text(yaml.safe_dump(temp_selectors))
|
|
490
|
+
logger.info(
|
|
491
|
+
f"DBT selection of {len(resources)} resources exceeds threshold of {_SELECTION_ARGS_THRESHOLD}. "
|
|
492
|
+
"This may exceed system argument length limits. "
|
|
493
|
+
f"Executing materialization against temporary copy of DBT project at {temp_project_dir} with ephemeral selector."
|
|
494
|
+
)
|
|
495
|
+
selection_args = ["--selector", selector_name]
|
|
496
|
+
target_project = replace(dbt_project, project_dir=temp_project_dir)
|
|
454
497
|
|
|
455
498
|
indirect_selection = (
|
|
456
499
|
indirect_selection_override if indirect_selection_override else indirect_selection
|
|
457
500
|
)
|
|
501
|
+
else:
|
|
502
|
+
target_project = dbt_project
|
|
458
503
|
|
|
459
504
|
return DbtCliInvocationPartialParams(
|
|
460
505
|
manifest=manifest,
|
|
461
506
|
dagster_dbt_translator=dagster_dbt_translator,
|
|
462
507
|
selection_args=selection_args,
|
|
463
508
|
indirect_selection=indirect_selection,
|
|
464
|
-
dbt_project=
|
|
509
|
+
dbt_project=target_project,
|
|
465
510
|
)
|
|
466
511
|
|
|
467
512
|
|
|
@@ -604,25 +649,6 @@ def default_owners_from_dbt_resource_props(
|
|
|
604
649
|
return [owner] if isinstance(owner, str) else owner
|
|
605
650
|
|
|
606
651
|
|
|
607
|
-
def default_freshness_policy_fn(
|
|
608
|
-
dbt_resource_props: Mapping[str, Any],
|
|
609
|
-
) -> Optional[LegacyFreshnessPolicy]:
|
|
610
|
-
dagster_metadata = dbt_resource_props.get("meta", {}).get("dagster", {})
|
|
611
|
-
freshness_policy_config = dagster_metadata.get("freshness_policy", {})
|
|
612
|
-
|
|
613
|
-
freshness_policy = (
|
|
614
|
-
LegacyFreshnessPolicy(
|
|
615
|
-
maximum_lag_minutes=float(freshness_policy_config["maximum_lag_minutes"]),
|
|
616
|
-
cron_schedule=freshness_policy_config.get("cron_schedule"),
|
|
617
|
-
cron_schedule_timezone=freshness_policy_config.get("cron_schedule_timezone"),
|
|
618
|
-
)
|
|
619
|
-
if freshness_policy_config
|
|
620
|
-
else None
|
|
621
|
-
)
|
|
622
|
-
|
|
623
|
-
return freshness_policy
|
|
624
|
-
|
|
625
|
-
|
|
626
652
|
def default_auto_materialize_policy_fn(
|
|
627
653
|
dbt_resource_props: Mapping[str, Any],
|
|
628
654
|
) -> Optional[AutoMaterializePolicy]:
|
dagster_dbt/cloud/asset_defs.py
CHANGED
|
@@ -33,7 +33,6 @@ from dagster_dbt.asset_utils import (
|
|
|
33
33
|
default_asset_key_fn,
|
|
34
34
|
default_auto_materialize_policy_fn,
|
|
35
35
|
default_description_fn,
|
|
36
|
-
default_freshness_policy_fn,
|
|
37
36
|
default_group_from_dbt_resource_props,
|
|
38
37
|
get_node,
|
|
39
38
|
)
|
|
@@ -326,10 +325,6 @@ class DbtCloudCacheableAssetsDefinition(CacheableAssetsDefinition):
|
|
|
326
325
|
def get_group_name(cls, dbt_resource_props): # pyright: ignore[reportIncompatibleMethodOverride]
|
|
327
326
|
return self._node_info_to_group_fn(dbt_resource_props)
|
|
328
327
|
|
|
329
|
-
@classmethod
|
|
330
|
-
def get_freshness_policy(cls, dbt_resource_props): # pyright: ignore[reportIncompatibleMethodOverride]
|
|
331
|
-
return self._node_info_to_freshness_policy_fn(dbt_resource_props)
|
|
332
|
-
|
|
333
328
|
@classmethod
|
|
334
329
|
def get_auto_materialize_policy(cls, dbt_resource_props): # pyright: ignore[reportIncompatibleMethodOverride]
|
|
335
330
|
return self._node_info_to_auto_materialize_policy_fn(dbt_resource_props)
|
|
@@ -375,11 +370,6 @@ class DbtCloudCacheableAssetsDefinition(CacheableAssetsDefinition):
|
|
|
375
370
|
for spec in specs
|
|
376
371
|
},
|
|
377
372
|
},
|
|
378
|
-
legacy_freshness_policies_by_output_name={
|
|
379
|
-
spec.key.to_python_identifier(): spec.legacy_freshness_policy
|
|
380
|
-
for spec in specs
|
|
381
|
-
if spec.legacy_freshness_policy
|
|
382
|
-
},
|
|
383
373
|
auto_materialize_policies_by_output_name={
|
|
384
374
|
spec.key.to_python_identifier(): spec.auto_materialize_policy
|
|
385
375
|
for spec in specs
|
|
@@ -546,9 +536,6 @@ def load_assets_from_dbt_cloud_job(
|
|
|
546
536
|
node_info_to_group_fn: Callable[
|
|
547
537
|
[Mapping[str, Any]], Optional[str]
|
|
548
538
|
] = default_group_from_dbt_resource_props,
|
|
549
|
-
node_info_to_freshness_policy_fn: Callable[
|
|
550
|
-
[Mapping[str, Any]], Optional[LegacyFreshnessPolicy]
|
|
551
|
-
] = default_freshness_policy_fn,
|
|
552
539
|
node_info_to_auto_materialize_policy_fn: Callable[
|
|
553
540
|
[Mapping[str, Any]], Optional[AutoMaterializePolicy]
|
|
554
541
|
] = default_auto_materialize_policy_fn,
|
|
@@ -570,13 +557,6 @@ def load_assets_from_dbt_cloud_job(
|
|
|
570
557
|
dbt source -> AssetKey([source_name, table_name])
|
|
571
558
|
node_info_to_group_fn (Dict[str, Any] -> Optional[str]): A function that takes a
|
|
572
559
|
dictionary of dbt node info and returns the group that this node should be assigned to.
|
|
573
|
-
node_info_to_freshness_policy_fn (Dict[str, Any] -> Optional[FreshnessPolicy]): A function
|
|
574
|
-
that takes a dictionary of dbt node info and optionally returns a FreshnessPolicy that
|
|
575
|
-
should be applied to this node. By default, freshness policies will be created from
|
|
576
|
-
config applied to dbt models, i.e.:
|
|
577
|
-
`dagster_freshness_policy={"maximum_lag_minutes": 60, "cron_schedule": "0 9 * * *"}`
|
|
578
|
-
will result in that model being assigned
|
|
579
|
-
`FreshnessPolicy(maximum_lag_minutes=60, cron_schedule="0 9 * * *")`
|
|
580
560
|
node_info_to_auto_materialize_policy_fn (Dict[str, Any] -> Optional[AutoMaterializePolicy]):
|
|
581
561
|
A function that takes a dictionary of dbt node info and optionally returns a AutoMaterializePolicy
|
|
582
562
|
that should be applied to this node. By default, AutoMaterializePolicies will be created from
|
|
@@ -631,7 +611,7 @@ def load_assets_from_dbt_cloud_job(
|
|
|
631
611
|
job_id=job_id,
|
|
632
612
|
node_info_to_asset_key=node_info_to_asset_key,
|
|
633
613
|
node_info_to_group_fn=node_info_to_group_fn,
|
|
634
|
-
node_info_to_freshness_policy_fn=
|
|
614
|
+
node_info_to_freshness_policy_fn=lambda _: None,
|
|
635
615
|
node_info_to_auto_materialize_policy_fn=node_info_to_auto_materialize_policy_fn,
|
|
636
616
|
partitions_def=partitions_def,
|
|
637
617
|
partition_key_to_vars_fn=partition_key_to_vars_fn,
|
|
@@ -1,10 +1,9 @@
|
|
|
1
1
|
import itertools
|
|
2
2
|
import json
|
|
3
|
-
import shutil
|
|
4
3
|
from collections.abc import Iterator, Mapping
|
|
5
4
|
from contextlib import contextmanager
|
|
6
5
|
from contextvars import ContextVar
|
|
7
|
-
from dataclasses import dataclass, field
|
|
6
|
+
from dataclasses import dataclass, field, replace
|
|
8
7
|
from functools import cached_property
|
|
9
8
|
from pathlib import Path
|
|
10
9
|
from typing import Annotated, Any, Literal, Optional, Union
|
|
@@ -40,6 +39,12 @@ from dagster_dbt.dagster_dbt_translator import (
|
|
|
40
39
|
from dagster_dbt.dbt_manifest import validate_manifest
|
|
41
40
|
from dagster_dbt.dbt_manifest_asset_selection import DbtManifestAssetSelection
|
|
42
41
|
from dagster_dbt.dbt_project import DbtProject
|
|
42
|
+
from dagster_dbt.dbt_project_manager import (
|
|
43
|
+
DbtProjectArgsManager,
|
|
44
|
+
DbtProjectManager,
|
|
45
|
+
NoopDbtProjectManager,
|
|
46
|
+
RemoteGitDbtProjectManager,
|
|
47
|
+
)
|
|
43
48
|
from dagster_dbt.utils import ASSET_RESOURCE_TYPES
|
|
44
49
|
|
|
45
50
|
|
|
@@ -63,29 +68,18 @@ class DbtProjectArgs(dg.Resolvable):
|
|
|
63
68
|
state_path: Optional[str] = None
|
|
64
69
|
|
|
65
70
|
|
|
66
|
-
def resolve_dbt_project(context: ResolutionContext, model) ->
|
|
67
|
-
if isinstance(model,
|
|
68
|
-
return
|
|
69
|
-
context.resolve_source_relative_path(
|
|
70
|
-
context.resolve_value(model, as_type=str),
|
|
71
|
-
)
|
|
72
|
-
)
|
|
73
|
-
|
|
74
|
-
args = DbtProjectArgs.resolve_from_model(context, model)
|
|
75
|
-
|
|
76
|
-
kwargs = {} # use optionally splatted kwargs to avoid redefining default value
|
|
77
|
-
if args.target_path:
|
|
78
|
-
kwargs["target_path"] = args.target_path
|
|
71
|
+
def resolve_dbt_project(context: ResolutionContext, model) -> DbtProjectManager:
|
|
72
|
+
if isinstance(model, RemoteGitDbtProjectManager.model()):
|
|
73
|
+
return RemoteGitDbtProjectManager.resolve_from_model(context, model)
|
|
79
74
|
|
|
80
|
-
|
|
81
|
-
project_dir=context.
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
state_path=args.state_path,
|
|
85
|
-
packaged_project_dir=args.packaged_project_dir,
|
|
86
|
-
profile=args.profile,
|
|
87
|
-
**kwargs,
|
|
75
|
+
args = (
|
|
76
|
+
DbtProjectArgs(project_dir=context.resolve_value(model, as_type=str))
|
|
77
|
+
if isinstance(model, str)
|
|
78
|
+
else DbtProjectArgs.resolve_from_model(context, model)
|
|
88
79
|
)
|
|
80
|
+
# resolve the project_dir relative to where this component is defined
|
|
81
|
+
args = replace(args, project_dir=context.resolve_source_relative_path(args.project_dir))
|
|
82
|
+
return DbtProjectArgsManager(args)
|
|
89
83
|
|
|
90
84
|
|
|
91
85
|
DbtMetadataAddons: TypeAlias = Literal["column_metadata", "row_count"]
|
|
@@ -113,13 +107,25 @@ class DbtProjectComponent(StateBackedComponent, dg.Resolvable):
|
|
|
113
107
|
|
|
114
108
|
Scaffold a DbtProjectComponent definition by running `dg scaffold defs dagster_dbt.DbtProjectComponent --project-path path/to/your/existing/dbt_project`
|
|
115
109
|
in the Dagster project directory.
|
|
110
|
+
|
|
111
|
+
Example:
|
|
112
|
+
|
|
113
|
+
.. code-block:: yaml
|
|
114
|
+
|
|
115
|
+
# defs.yaml
|
|
116
|
+
|
|
117
|
+
type: dagster_dbt.DbtProjectComponent
|
|
118
|
+
attributes:
|
|
119
|
+
project: "{{ project_root }}/path/to/dbt_project"
|
|
120
|
+
cli_args:
|
|
121
|
+
- build
|
|
116
122
|
"""
|
|
117
123
|
|
|
118
124
|
project: Annotated[
|
|
119
|
-
DbtProject,
|
|
125
|
+
Union[DbtProject, DbtProjectManager],
|
|
120
126
|
Resolver(
|
|
121
127
|
resolve_dbt_project,
|
|
122
|
-
model_field_type=Union[str, DbtProjectArgs.model()],
|
|
128
|
+
model_field_type=Union[str, DbtProjectArgs.model(), RemoteGitDbtProjectManager.model()],
|
|
123
129
|
description="The path to the dbt project or a mapping defining a DbtProject",
|
|
124
130
|
examples=[
|
|
125
131
|
"{{ project_root }}/path/to/dbt_project",
|
|
@@ -212,8 +218,8 @@ class DbtProjectComponent(StateBackedComponent, dg.Resolvable):
|
|
|
212
218
|
@property
|
|
213
219
|
def defs_state_config(self) -> DefsStateConfig:
|
|
214
220
|
return DefsStateConfig(
|
|
215
|
-
key=f"{self.__class__.__name__}[{self.
|
|
216
|
-
|
|
221
|
+
key=f"{self.__class__.__name__}[{self._project_manager.defs_state_discriminator}]",
|
|
222
|
+
management_type=DefsStateManagementType.LOCAL_FILESYSTEM,
|
|
217
223
|
refresh_if_dev=self.prepare_if_dev,
|
|
218
224
|
)
|
|
219
225
|
|
|
@@ -225,9 +231,66 @@ class DbtProjectComponent(StateBackedComponent, dg.Resolvable):
|
|
|
225
231
|
def _base_translator(self) -> "DagsterDbtTranslator":
|
|
226
232
|
return DagsterDbtTranslator(self.translation_settings)
|
|
227
233
|
|
|
234
|
+
def get_resource_props(self, manifest: Mapping[str, Any], unique_id: str) -> Mapping[str, Any]:
|
|
235
|
+
"""Given a parsed manifest and a dbt unique_id, returns the dictionary of properties
|
|
236
|
+
for the corresponding dbt resource (e.g. model, seed, snapshot, source) as defined
|
|
237
|
+
in your dbt project. This can be used as a convenience method when overriding the
|
|
238
|
+
`get_asset_spec` method.
|
|
239
|
+
|
|
240
|
+
Args:
|
|
241
|
+
manifest (Mapping[str, Any]): The parsed manifest of the dbt project.
|
|
242
|
+
unique_id (str): The unique_id of the dbt resource.
|
|
243
|
+
|
|
244
|
+
Returns:
|
|
245
|
+
Mapping[str, Any]: The dictionary of properties for the corresponding dbt resource.
|
|
246
|
+
|
|
247
|
+
Examples:
|
|
248
|
+
.. code-block:: python
|
|
249
|
+
|
|
250
|
+
class CustomDbtProjectComponent(DbtProjectComponent):
|
|
251
|
+
|
|
252
|
+
def get_asset_spec(self, manifest: Mapping[str, Any], unique_id: str, project: Optional[DbtProject]) -> dg.AssetSpec:
|
|
253
|
+
base_spec = super().get_asset_spec(manifest, unique_id, project)
|
|
254
|
+
resource_props = self.get_resource_props(manifest, unique_id)
|
|
255
|
+
if resource_props["meta"].get("use_custom_group"):
|
|
256
|
+
return base_spec.replace_attributes(group_name="custom_group")
|
|
257
|
+
else:
|
|
258
|
+
return base_spec
|
|
259
|
+
"""
|
|
260
|
+
return get_node(manifest, unique_id)
|
|
261
|
+
|
|
262
|
+
@public
|
|
228
263
|
def get_asset_spec(
|
|
229
264
|
self, manifest: Mapping[str, Any], unique_id: str, project: Optional[DbtProject]
|
|
230
265
|
) -> dg.AssetSpec:
|
|
266
|
+
"""Generates an AssetSpec for a given dbt node.
|
|
267
|
+
|
|
268
|
+
This method can be overridden in a subclass to customize how dbt nodes are converted
|
|
269
|
+
to Dagster asset specs. By default, it delegates to the configured DagsterDbtTranslator.
|
|
270
|
+
|
|
271
|
+
Args:
|
|
272
|
+
manifest: The dbt manifest dictionary containing information about all dbt nodes
|
|
273
|
+
unique_id: The unique identifier for the dbt node (e.g., "model.my_project.my_model")
|
|
274
|
+
project: The DbtProject object, if available
|
|
275
|
+
|
|
276
|
+
Returns:
|
|
277
|
+
An AssetSpec that represents the dbt node as a Dagster asset
|
|
278
|
+
|
|
279
|
+
Example:
|
|
280
|
+
Override this method to add custom tags to all dbt models:
|
|
281
|
+
|
|
282
|
+
.. code-block:: python
|
|
283
|
+
|
|
284
|
+
from dagster_dbt import DbtProjectComponent
|
|
285
|
+
import dagster as dg
|
|
286
|
+
|
|
287
|
+
class CustomDbtProjectComponent(DbtProjectComponent):
|
|
288
|
+
def get_asset_spec(self, manifest, unique_id, project):
|
|
289
|
+
base_spec = super().get_asset_spec(manifest, unique_id, project)
|
|
290
|
+
return base_spec.replace_attributes(
|
|
291
|
+
tags={**base_spec.tags, "custom_tag": "my_value"}
|
|
292
|
+
)
|
|
293
|
+
"""
|
|
231
294
|
return self._base_translator.get_asset_spec(manifest, unique_id, project)
|
|
232
295
|
|
|
233
296
|
def get_asset_check_spec(
|
|
@@ -240,37 +303,40 @@ class DbtProjectComponent(StateBackedComponent, dg.Resolvable):
|
|
|
240
303
|
return self._base_translator.get_asset_check_spec(asset_spec, manifest, unique_id, project)
|
|
241
304
|
|
|
242
305
|
@cached_property
|
|
243
|
-
def
|
|
244
|
-
|
|
306
|
+
def _project_manager(self) -> DbtProjectManager:
|
|
307
|
+
if isinstance(self.project, DbtProject):
|
|
308
|
+
return NoopDbtProjectManager(self.project)
|
|
309
|
+
else:
|
|
310
|
+
return self.project
|
|
311
|
+
|
|
312
|
+
@cached_property
|
|
313
|
+
def dbt_project(self) -> DbtProject:
|
|
314
|
+
return self._project_manager.get_project(None)
|
|
245
315
|
|
|
246
316
|
def get_asset_selection(
|
|
247
317
|
self, select: str, exclude: str = DBT_DEFAULT_EXCLUDE
|
|
248
318
|
) -> DbtManifestAssetSelection:
|
|
249
319
|
return DbtManifestAssetSelection.build(
|
|
250
|
-
manifest=self.
|
|
320
|
+
manifest=self.dbt_project.manifest_path,
|
|
251
321
|
dagster_dbt_translator=self.translator,
|
|
252
322
|
select=select,
|
|
253
323
|
exclude=exclude,
|
|
254
324
|
)
|
|
255
325
|
|
|
256
326
|
def write_state_to_path(self, state_path: Path) -> None:
|
|
257
|
-
|
|
258
|
-
self.project.preparer.prepare(self.project)
|
|
259
|
-
# move the manifest to the correct path
|
|
260
|
-
shutil.copyfile(self.project.manifest_path, state_path)
|
|
327
|
+
self._project_manager.prepare(state_path)
|
|
261
328
|
|
|
262
329
|
def build_defs_from_state(
|
|
263
330
|
self, context: dg.ComponentLoadContext, state_path: Optional[Path]
|
|
264
331
|
) -> dg.Definitions:
|
|
265
|
-
|
|
266
|
-
shutil.copyfile(state_path, self.project.manifest_path)
|
|
332
|
+
project = self._project_manager.get_project(state_path)
|
|
267
333
|
|
|
268
334
|
res_ctx = context.resolution_context
|
|
269
335
|
|
|
270
336
|
@dbt_assets(
|
|
271
|
-
manifest=
|
|
272
|
-
project=
|
|
273
|
-
name=self.op.name if self.op else
|
|
337
|
+
manifest=project.manifest_path,
|
|
338
|
+
project=project,
|
|
339
|
+
name=self.op.name if self.op else project.name,
|
|
274
340
|
op_tags=self.op.tags if self.op else None,
|
|
275
341
|
dagster_dbt_translator=self.translator,
|
|
276
342
|
select=self.select,
|
|
@@ -279,7 +345,7 @@ class DbtProjectComponent(StateBackedComponent, dg.Resolvable):
|
|
|
279
345
|
)
|
|
280
346
|
def _fn(context: dg.AssetExecutionContext):
|
|
281
347
|
with _set_resolution_context(res_ctx):
|
|
282
|
-
yield from self.execute(context=context, dbt=
|
|
348
|
+
yield from self.execute(context=context, dbt=DbtCliResource(project))
|
|
283
349
|
|
|
284
350
|
return dg.Definitions(assets=[_fn])
|
|
285
351
|
|
|
@@ -336,12 +402,39 @@ class DbtProjectComponent(StateBackedComponent, dg.Resolvable):
|
|
|
336
402
|
iterator = iterator.fetch_row_counts()
|
|
337
403
|
return iterator
|
|
338
404
|
|
|
405
|
+
@public
|
|
339
406
|
def execute(self, context: dg.AssetExecutionContext, dbt: DbtCliResource) -> Iterator:
|
|
407
|
+
"""Executes the dbt command for the selected assets.
|
|
408
|
+
|
|
409
|
+
This method can be overridden in a subclass to customize the execution behavior,
|
|
410
|
+
such as adding custom logging, modifying CLI arguments, or handling events differently.
|
|
411
|
+
|
|
412
|
+
Args:
|
|
413
|
+
context: The asset execution context provided by Dagster
|
|
414
|
+
dbt: The DbtCliResource used to execute dbt commands
|
|
415
|
+
|
|
416
|
+
Yields:
|
|
417
|
+
Events from the dbt CLI execution (e.g., AssetMaterialization, AssetObservation)
|
|
418
|
+
|
|
419
|
+
Example:
|
|
420
|
+
Override this method to add custom logging before and after execution:
|
|
421
|
+
|
|
422
|
+
.. code-block:: python
|
|
423
|
+
|
|
424
|
+
from dagster_dbt import DbtProjectComponent
|
|
425
|
+
import dagster as dg
|
|
426
|
+
|
|
427
|
+
class CustomDbtProjectComponent(DbtProjectComponent):
|
|
428
|
+
def execute(self, context, dbt):
|
|
429
|
+
context.log.info("Starting custom dbt execution")
|
|
430
|
+
yield from super().execute(context, dbt)
|
|
431
|
+
context.log.info("Completed custom dbt execution")
|
|
432
|
+
"""
|
|
340
433
|
yield from self._get_dbt_event_iterator(context, dbt)
|
|
341
434
|
|
|
342
435
|
@cached_property
|
|
343
436
|
def _validated_manifest(self):
|
|
344
|
-
return validate_manifest(self.
|
|
437
|
+
return validate_manifest(self.dbt_project.manifest_path)
|
|
345
438
|
|
|
346
439
|
@cached_property
|
|
347
440
|
def _validated_translator(self):
|
|
@@ -364,7 +457,7 @@ class DbtProjectComponent(StateBackedComponent, dg.Resolvable):
|
|
|
364
457
|
return dagster_dbt_translator.get_asset_spec(
|
|
365
458
|
manifest,
|
|
366
459
|
next(iter(matching_model_ids)),
|
|
367
|
-
self.
|
|
460
|
+
self.dbt_project,
|
|
368
461
|
).key
|
|
369
462
|
|
|
370
463
|
|
|
@@ -396,4 +489,4 @@ def get_projects_from_dbt_component(components: Path) -> list[DbtProject]:
|
|
|
396
489
|
of_type=DbtProjectComponent
|
|
397
490
|
)
|
|
398
491
|
|
|
399
|
-
return [component.
|
|
492
|
+
return [component.dbt_project for component in project_components]
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import os
|
|
2
2
|
from pathlib import Path
|
|
3
|
-
from typing import Optional
|
|
3
|
+
from typing import Any, Optional
|
|
4
4
|
|
|
5
5
|
import dagster._check as check
|
|
6
6
|
from dagster._core.errors import DagsterInvalidInvocationError
|
|
@@ -13,6 +13,7 @@ from pydantic import BaseModel, Field
|
|
|
13
13
|
class DbtScaffoldParams(BaseModel):
|
|
14
14
|
init: bool = Field(default=False)
|
|
15
15
|
project_path: Optional[str] = None
|
|
16
|
+
git_url: Optional[str] = None
|
|
16
17
|
|
|
17
18
|
|
|
18
19
|
class DbtProjectComponentScaffolder(Scaffolder[DbtScaffoldParams]):
|
|
@@ -20,31 +21,45 @@ class DbtProjectComponentScaffolder(Scaffolder[DbtScaffoldParams]):
|
|
|
20
21
|
def get_scaffold_params(cls) -> type[DbtScaffoldParams]:
|
|
21
22
|
return DbtScaffoldParams
|
|
22
23
|
|
|
23
|
-
def
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
24
|
+
def _init_dbt_project(self) -> None:
|
|
25
|
+
try:
|
|
26
|
+
from dbt.cli.main import dbtRunner
|
|
27
|
+
except ImportError:
|
|
28
|
+
raise DagsterInvalidInvocationError(
|
|
29
|
+
"dbt-core is not installed. Please install dbt to scaffold this component."
|
|
30
|
+
)
|
|
31
|
+
|
|
32
|
+
dbtRunner().invoke(["init"])
|
|
29
33
|
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
except ImportError:
|
|
34
|
-
raise DagsterInvalidInvocationError(
|
|
35
|
-
"dbt-core is not installed. Please install dbt to scaffold this component."
|
|
36
|
-
)
|
|
34
|
+
def _git_url_params(self, git_url: str, project_path: Optional[str]) -> dict[str, Any]:
|
|
35
|
+
repo_relative_path = {"repo_relative_path": project_path} if project_path else {}
|
|
36
|
+
return {"project": {"repo_url": git_url, **repo_relative_path}}
|
|
37
37
|
|
|
38
|
-
|
|
38
|
+
def scaffold(self, request: ScaffoldRequest[DbtScaffoldParams]) -> None:
|
|
39
|
+
project_root = request.project_root or Path(os.getcwd())
|
|
40
|
+
check.param_invariant(
|
|
41
|
+
not (request.params.init and (request.params.git_url or request.params.project_path)),
|
|
42
|
+
"init and git_url/project_path cannot be set at the same time.",
|
|
43
|
+
)
|
|
44
|
+
if request.params.init:
|
|
45
|
+
self._init_dbt_project()
|
|
39
46
|
subpaths = [
|
|
40
|
-
path
|
|
41
|
-
for path in Path(project_root).iterdir()
|
|
42
|
-
if path.is_dir() and path.name != "logs"
|
|
47
|
+
path for path in project_root.iterdir() if path.is_dir() and path.name != "logs"
|
|
43
48
|
]
|
|
44
49
|
check.invariant(len(subpaths) == 1, "Expected exactly one subpath to be created.")
|
|
45
50
|
# this path should be relative to this directory
|
|
46
|
-
|
|
51
|
+
params = {"project": subpaths[0].name}
|
|
52
|
+
elif request.params.git_url is not None:
|
|
53
|
+
# project_path is the path to the dbt project within the git repository
|
|
54
|
+
project_path = request.params.project_path
|
|
55
|
+
repo_relative_path = {"repo_relative_path": project_path} if project_path else {}
|
|
56
|
+
params = {"project": {"repo_url": request.params.git_url, **repo_relative_path}}
|
|
57
|
+
elif request.params.project_path is not None:
|
|
58
|
+
# project_path represents a local directory
|
|
59
|
+
project_root_tmpl = "{{ project_root }}"
|
|
60
|
+
rel_path = os.path.relpath(request.params.project_path, start=project_root)
|
|
61
|
+
path_str = f"{project_root_tmpl}/{rel_path}"
|
|
62
|
+
params = {"project": path_str}
|
|
47
63
|
else:
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
scaffold_component(request, {"project": path_str})
|
|
64
|
+
params = {"project": None}
|
|
65
|
+
scaffold_component(request, params)
|
dagster_dbt/core/resource.py
CHANGED
|
@@ -428,7 +428,7 @@ class DbtCliResource(ConfigurableResource):
|
|
|
428
428
|
adapter = get_adapter(config)
|
|
429
429
|
manifest = ManifestLoader.load_macros(
|
|
430
430
|
config,
|
|
431
|
-
adapter.connections.set_query_header,
|
|
431
|
+
adapter.connections.set_query_header,
|
|
432
432
|
base_macros_only=True,
|
|
433
433
|
)
|
|
434
434
|
adapter.set_macro_resolver(manifest)
|
|
@@ -627,8 +627,12 @@ class DbtCliResource(ConfigurableResource):
|
|
|
627
627
|
dagster_dbt_translator = updated_params.dagster_dbt_translator
|
|
628
628
|
selection_args = updated_params.selection_args
|
|
629
629
|
indirect_selection = updated_params.indirect_selection
|
|
630
|
-
|
|
631
630
|
target_path = target_path or self._get_unique_target_path(context=context)
|
|
631
|
+
project_dir = Path(
|
|
632
|
+
updated_params.dbt_project.project_dir
|
|
633
|
+
if updated_params.dbt_project
|
|
634
|
+
else self.project_dir
|
|
635
|
+
)
|
|
632
636
|
env = {
|
|
633
637
|
# Allow IO streaming when running in Windows.
|
|
634
638
|
# Also, allow it to be overriden by the current environment.
|
|
@@ -659,7 +663,7 @@ class DbtCliResource(ConfigurableResource):
|
|
|
659
663
|
**({"DBT_PROFILES_DIR": self.profiles_dir} if self.profiles_dir else {}),
|
|
660
664
|
# The DBT_PROJECT_DIR environment variable is set to the path containing the dbt project
|
|
661
665
|
# See https://docs.getdbt.com/reference/dbt_project.yml for more information.
|
|
662
|
-
|
|
666
|
+
"DBT_PROJECT_DIR": str(project_dir),
|
|
663
667
|
}
|
|
664
668
|
|
|
665
669
|
# set dbt indirect selection if needed to execute specific dbt tests due to asset check
|
|
@@ -683,14 +687,13 @@ class DbtCliResource(ConfigurableResource):
|
|
|
683
687
|
*profile_args,
|
|
684
688
|
*selection_args,
|
|
685
689
|
]
|
|
686
|
-
project_dir = Path(self.project_dir)
|
|
687
690
|
|
|
688
691
|
if not target_path.is_absolute():
|
|
689
692
|
target_path = project_dir.joinpath(target_path)
|
|
690
693
|
|
|
691
694
|
# run dbt --version to get the dbt core version
|
|
692
695
|
adapter: Optional[BaseAdapter] = None
|
|
693
|
-
with pushd(
|
|
696
|
+
with pushd(str(project_dir)):
|
|
694
697
|
# we do not need to initialize the adapter if we are using the fusion engine
|
|
695
698
|
if self._cli_version.major < 2:
|
|
696
699
|
try:
|
|
@@ -12,7 +12,6 @@ from dagster import (
|
|
|
12
12
|
AutoMaterializePolicy,
|
|
13
13
|
AutomationCondition,
|
|
14
14
|
DagsterInvalidDefinitionError,
|
|
15
|
-
LegacyFreshnessPolicy,
|
|
16
15
|
PartitionMapping,
|
|
17
16
|
)
|
|
18
17
|
from dagster._annotations import beta, public
|
|
@@ -36,7 +35,6 @@ from dagster_dbt.asset_utils import (
|
|
|
36
35
|
default_auto_materialize_policy_fn,
|
|
37
36
|
default_code_version_fn,
|
|
38
37
|
default_description_fn,
|
|
39
|
-
default_freshness_policy_fn,
|
|
40
38
|
default_group_from_dbt_resource_props,
|
|
41
39
|
default_metadata_from_dbt_resource_props,
|
|
42
40
|
default_owners_from_dbt_resource_props,
|
|
@@ -96,6 +94,34 @@ class DagsterDbtTranslator:
|
|
|
96
94
|
|
|
97
95
|
return self._settings
|
|
98
96
|
|
|
97
|
+
def get_resource_props(self, manifest: Mapping[str, Any], unique_id: str) -> Mapping[str, Any]:
|
|
98
|
+
"""Given a parsed manifest and a dbt unique_id, returns the dictionary of properties
|
|
99
|
+
for the corresponding dbt resource (e.g. model, seed, snapshot, source) as defined
|
|
100
|
+
in your dbt project. This can be used as a convenience method when overriding the
|
|
101
|
+
`get_asset_spec` method.
|
|
102
|
+
|
|
103
|
+
Args:
|
|
104
|
+
manifest (Mapping[str, Any]): The parsed manifest of the dbt project.
|
|
105
|
+
unique_id (str): The unique_id of the dbt resource.
|
|
106
|
+
|
|
107
|
+
Returns:
|
|
108
|
+
Mapping[str, Any]: The dictionary of properties for the corresponding dbt resource.
|
|
109
|
+
|
|
110
|
+
Examples:
|
|
111
|
+
.. code-block:: python
|
|
112
|
+
|
|
113
|
+
class CustomDagsterDbtTranslator(DagsterDbtTranslator):
|
|
114
|
+
|
|
115
|
+
def get_asset_spec(self, manifest: Mapping[str, Any], unique_id: str, project: Optional[DbtProject]) -> dg.AssetSpec:
|
|
116
|
+
base_spec = super().get_asset_spec(manifest, unique_id, project)
|
|
117
|
+
resource_props = self.get_resource_props(manifest, unique_id)
|
|
118
|
+
if resource_props["meta"].get("use_custom_group"):
|
|
119
|
+
return base_spec.replace_attributes(group_name="custom_group")
|
|
120
|
+
else:
|
|
121
|
+
return base_spec
|
|
122
|
+
"""
|
|
123
|
+
return get_node(manifest, unique_id)
|
|
124
|
+
|
|
99
125
|
def get_asset_spec(
|
|
100
126
|
self,
|
|
101
127
|
manifest: Mapping[str, Any],
|
|
@@ -115,7 +141,7 @@ class DagsterDbtTranslator:
|
|
|
115
141
|
return self._resolved_specs[memo_id]
|
|
116
142
|
|
|
117
143
|
group_props = {group["name"]: group for group in manifest.get("groups", {}).values()}
|
|
118
|
-
resource_props =
|
|
144
|
+
resource_props = self.get_resource_props(manifest, unique_id)
|
|
119
145
|
|
|
120
146
|
# calculate the dependencies for the asset
|
|
121
147
|
upstream_ids = get_upstream_unique_ids(manifest, resource_props)
|
|
@@ -123,7 +149,7 @@ class DagsterDbtTranslator:
|
|
|
123
149
|
AssetDep(
|
|
124
150
|
asset=self.get_asset_spec(manifest, upstream_id, project).key,
|
|
125
151
|
partition_mapping=self.get_partition_mapping(
|
|
126
|
-
resource_props,
|
|
152
|
+
resource_props, self.get_resource_props(manifest, upstream_id)
|
|
127
153
|
),
|
|
128
154
|
)
|
|
129
155
|
for upstream_id in upstream_ids
|
|
@@ -158,7 +184,6 @@ class DagsterDbtTranslator:
|
|
|
158
184
|
group_name=self.get_group_name(resource_props),
|
|
159
185
|
code_version=self.get_code_version(resource_props),
|
|
160
186
|
automation_condition=self.get_automation_condition(resource_props),
|
|
161
|
-
legacy_freshness_policy=self.get_freshness_policy(resource_props),
|
|
162
187
|
owners=self.get_owners(owners_resource_props),
|
|
163
188
|
tags=self.get_tags(resource_props),
|
|
164
189
|
kinds={"dbt", manifest.get("metadata", {}).get("adapter_type", "dbt")},
|
|
@@ -493,60 +518,6 @@ class DagsterDbtTranslator:
|
|
|
493
518
|
"""
|
|
494
519
|
return default_owners_from_dbt_resource_props(dbt_resource_props)
|
|
495
520
|
|
|
496
|
-
@public
|
|
497
|
-
@beta(emit_runtime_warning=False)
|
|
498
|
-
def get_freshness_policy(
|
|
499
|
-
self, dbt_resource_props: Mapping[str, Any]
|
|
500
|
-
) -> Optional[LegacyFreshnessPolicy]:
|
|
501
|
-
"""A function that takes a dictionary representing properties of a dbt resource, and
|
|
502
|
-
returns the Dagster :py:class:`dagster.FreshnessPolicy` for that resource.
|
|
503
|
-
|
|
504
|
-
Note that a dbt resource is unrelated to Dagster's resource concept, and simply represents
|
|
505
|
-
a model, seed, snapshot or source in a given dbt project. You can learn more about dbt
|
|
506
|
-
resources and the properties available in this dictionary here:
|
|
507
|
-
https://docs.getdbt.com/reference/artifacts/manifest-json#resource-details
|
|
508
|
-
|
|
509
|
-
This method can be overridden to provide a custom freshness policy for a dbt resource.
|
|
510
|
-
|
|
511
|
-
Args:
|
|
512
|
-
dbt_resource_props (Mapping[str, Any]): A dictionary representing the dbt resource.
|
|
513
|
-
|
|
514
|
-
Returns:
|
|
515
|
-
Optional[FreshnessPolicy]: A Dagster freshness policy.
|
|
516
|
-
|
|
517
|
-
Examples:
|
|
518
|
-
Set a custom freshness policy for all dbt resources:
|
|
519
|
-
|
|
520
|
-
.. code-block:: python
|
|
521
|
-
|
|
522
|
-
from typing import Any, Mapping
|
|
523
|
-
|
|
524
|
-
from dagster_dbt import DagsterDbtTranslator
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
class CustomDagsterDbtTranslator(DagsterDbtTranslator):
|
|
528
|
-
def get_freshness_policy(self, dbt_resource_props: Mapping[str, Any]) -> Optional[FreshnessPolicy]:
|
|
529
|
-
return FreshnessPolicy(maximum_lag_minutes=60)
|
|
530
|
-
|
|
531
|
-
Set a custom freshness policy for dbt resources with a specific tag:
|
|
532
|
-
|
|
533
|
-
.. code-block:: python
|
|
534
|
-
|
|
535
|
-
from typing import Any, Mapping
|
|
536
|
-
|
|
537
|
-
from dagster_dbt import DagsterDbtTranslator
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
class CustomDagsterDbtTranslator(DagsterDbtTranslator):
|
|
541
|
-
def get_freshness_policy(self, dbt_resource_props: Mapping[str, Any]) -> Optional[FreshnessPolicy]:
|
|
542
|
-
freshness_policy = None
|
|
543
|
-
if "my_custom_tag" in dbt_resource_props.get("tags", []):
|
|
544
|
-
freshness_policy = FreshnessPolicy(maximum_lag_minutes=60)
|
|
545
|
-
|
|
546
|
-
return freshness_policy
|
|
547
|
-
"""
|
|
548
|
-
return default_freshness_policy_fn(dbt_resource_props)
|
|
549
|
-
|
|
550
521
|
@public
|
|
551
522
|
@beta(emit_runtime_warning=False)
|
|
552
523
|
def get_auto_materialize_policy(
|
|
@@ -0,0 +1,170 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import shutil
|
|
3
|
+
from abc import ABC, abstractmethod
|
|
4
|
+
from dataclasses import asdict, dataclass
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from typing import TYPE_CHECKING, Annotated, Optional
|
|
7
|
+
from urllib.parse import quote, urlparse, urlunparse
|
|
8
|
+
|
|
9
|
+
from dagster._core.errors import DagsterInvalidDefinitionError
|
|
10
|
+
from dagster.components.resolved.base import Resolvable
|
|
11
|
+
from dagster.components.resolved.model import Resolver
|
|
12
|
+
from git import Repo
|
|
13
|
+
|
|
14
|
+
from dagster_dbt.dbt_project import DbtProject
|
|
15
|
+
|
|
16
|
+
if TYPE_CHECKING:
|
|
17
|
+
from dagster_dbt.components.dbt_project.component import DbtProjectArgs
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class DbtProjectManager(ABC):
|
|
21
|
+
"""Helper class that wraps a dbt project that may or may not be available on local disk at the time
|
|
22
|
+
it is instantiated. Provides methods for syncing the project to a local path and for instantiating the
|
|
23
|
+
final DbtProject object when ready.
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
@property
|
|
27
|
+
@abstractmethod
|
|
28
|
+
def defs_state_discriminator(self) -> str: ...
|
|
29
|
+
|
|
30
|
+
@abstractmethod
|
|
31
|
+
def sync(self, state_path: Path) -> None: ...
|
|
32
|
+
|
|
33
|
+
def _local_project_dir(self, state_path: Path) -> Path:
|
|
34
|
+
return state_path.parent / "project"
|
|
35
|
+
|
|
36
|
+
def prepare(self, state_path: Path) -> None:
|
|
37
|
+
"""Syncs the project to the given local path and ensures the manifest is generated
|
|
38
|
+
and the dependencies are installed.
|
|
39
|
+
"""
|
|
40
|
+
# ensure local dir is empty
|
|
41
|
+
local_dir = self._local_project_dir(state_path)
|
|
42
|
+
shutil.rmtree(local_dir, ignore_errors=True)
|
|
43
|
+
local_dir.mkdir()
|
|
44
|
+
|
|
45
|
+
# ensure project exists in the dir and is compiled
|
|
46
|
+
self.sync(state_path)
|
|
47
|
+
|
|
48
|
+
# indicate that project has been prepared
|
|
49
|
+
state_path.touch()
|
|
50
|
+
|
|
51
|
+
@abstractmethod
|
|
52
|
+
def get_project(self, state_path: Optional[Path]) -> "DbtProject": ...
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
@dataclass
|
|
56
|
+
class NoopDbtProjectManager(DbtProjectManager):
|
|
57
|
+
"""Wraps a DbtProject that has already been fully instantiated. Used for cases where a
|
|
58
|
+
user directly provides a DbtProject to the DbtProjectComponent.
|
|
59
|
+
"""
|
|
60
|
+
|
|
61
|
+
project: "DbtProject"
|
|
62
|
+
|
|
63
|
+
@property
|
|
64
|
+
def defs_state_discriminator(self) -> str:
|
|
65
|
+
return self.project.name
|
|
66
|
+
|
|
67
|
+
def sync(self, state_path: Path) -> None:
|
|
68
|
+
pass
|
|
69
|
+
|
|
70
|
+
def get_project(self, state_path: Optional[Path]) -> DbtProject:
|
|
71
|
+
return self.project
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
@dataclass
|
|
75
|
+
class DbtProjectArgsManager(DbtProjectManager):
|
|
76
|
+
"""Wraps DbtProjectArgs provided to the DbtProjectComponent. Avoids instantiating the DbtProject object
|
|
77
|
+
immediately as this would cause errors in cases where the project_dir has not yet been synced.
|
|
78
|
+
"""
|
|
79
|
+
|
|
80
|
+
args: "DbtProjectArgs"
|
|
81
|
+
|
|
82
|
+
@property
|
|
83
|
+
def defs_state_discriminator(self) -> str:
|
|
84
|
+
return Path(self.args.project_dir).stem
|
|
85
|
+
|
|
86
|
+
def sync(self, state_path: Path) -> None:
|
|
87
|
+
# we prepare the project in the original project directory rather than the new one
|
|
88
|
+
# so that code that does not have access to the local state path can still access
|
|
89
|
+
# the manifest.json file.
|
|
90
|
+
project = self.get_project(None)
|
|
91
|
+
project.preparer.prepare(project)
|
|
92
|
+
shutil.copytree(
|
|
93
|
+
self.args.project_dir, self._local_project_dir(state_path), dirs_exist_ok=True
|
|
94
|
+
)
|
|
95
|
+
|
|
96
|
+
def get_project(self, state_path: Optional[Path]) -> "DbtProject":
|
|
97
|
+
kwargs = asdict(self.args)
|
|
98
|
+
|
|
99
|
+
project_dir = self._local_project_dir(state_path) if state_path else self.args.project_dir
|
|
100
|
+
return DbtProject(
|
|
101
|
+
project_dir=project_dir,
|
|
102
|
+
# allow default values on DbtProject to take precedence
|
|
103
|
+
**{k: v for k, v in kwargs.items() if v is not None and k != "project_dir"},
|
|
104
|
+
)
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
@dataclass
|
|
108
|
+
class RemoteGitDbtProjectManager(DbtProjectManager, Resolvable):
|
|
109
|
+
"""Wraps a remote git repository containing a dbt project."""
|
|
110
|
+
|
|
111
|
+
repo_url: str
|
|
112
|
+
repo_relative_path: Annotated[
|
|
113
|
+
str,
|
|
114
|
+
Resolver.default(
|
|
115
|
+
description="The relative path to the dbt project within the repository.",
|
|
116
|
+
examples=["dbt/my_dbt_project"],
|
|
117
|
+
),
|
|
118
|
+
] = "."
|
|
119
|
+
token: Annotated[
|
|
120
|
+
Optional[str],
|
|
121
|
+
Resolver.default(
|
|
122
|
+
description="Token for authenticating to the provided repository.",
|
|
123
|
+
examples=["'{{ env.GITHUB_TOKEN }}'"],
|
|
124
|
+
),
|
|
125
|
+
] = None
|
|
126
|
+
profile: Annotated[
|
|
127
|
+
Optional[str],
|
|
128
|
+
Resolver.default(description="The profile to use for the dbt project."),
|
|
129
|
+
] = None
|
|
130
|
+
target: Annotated[
|
|
131
|
+
Optional[str],
|
|
132
|
+
Resolver.default(
|
|
133
|
+
description="The target to use for the dbt project.",
|
|
134
|
+
examples=["dev", "prod"],
|
|
135
|
+
),
|
|
136
|
+
] = None
|
|
137
|
+
|
|
138
|
+
@property
|
|
139
|
+
def defs_state_discriminator(self) -> str:
|
|
140
|
+
return self.repo_url
|
|
141
|
+
|
|
142
|
+
def _get_clone_url(self) -> str:
|
|
143
|
+
# in github action environments, the token will be available automatically as an env var
|
|
144
|
+
raw_token = self.token or os.getenv("GITHUB_TOKEN")
|
|
145
|
+
if raw_token is None:
|
|
146
|
+
return self.repo_url
|
|
147
|
+
else:
|
|
148
|
+
# insert the token into the url
|
|
149
|
+
token = quote(raw_token, safe="")
|
|
150
|
+
parts = urlparse(self.repo_url)
|
|
151
|
+
return urlunparse(parts._replace(netloc=f"{token}@{parts.netloc}"))
|
|
152
|
+
|
|
153
|
+
def sync(self, state_path: Path) -> None:
|
|
154
|
+
Repo.clone_from(self._get_clone_url(), self._local_project_dir(state_path), depth=1)
|
|
155
|
+
project = self.get_project(state_path)
|
|
156
|
+
project.preparer.prepare(project)
|
|
157
|
+
|
|
158
|
+
def get_project(self, state_path: Optional[Path]) -> "DbtProject":
|
|
159
|
+
if state_path is None:
|
|
160
|
+
raise DagsterInvalidDefinitionError(
|
|
161
|
+
"Attempted to `get_project()` on a `RemoteGitDbtProjectWrapper` without a path. "
|
|
162
|
+
"This can happen when calling unsupported methods on the `DbtProjectComponent`."
|
|
163
|
+
)
|
|
164
|
+
|
|
165
|
+
local_dir = self._local_project_dir(state_path)
|
|
166
|
+
return DbtProject(
|
|
167
|
+
project_dir=local_dir / self.repo_relative_path,
|
|
168
|
+
profile=self.profile,
|
|
169
|
+
target=self.target,
|
|
170
|
+
)
|
dagster_dbt/freshness_builder.py
CHANGED
|
@@ -6,7 +6,6 @@ from dagster import (
|
|
|
6
6
|
AssetsDefinition,
|
|
7
7
|
_check as check,
|
|
8
8
|
)
|
|
9
|
-
from dagster._annotations import beta
|
|
10
9
|
from dagster._core.definitions.asset_checks.asset_check_factories.freshness_checks.last_update import (
|
|
11
10
|
build_last_update_freshness_checks,
|
|
12
11
|
)
|
|
@@ -27,13 +26,18 @@ from dagster._core.errors import DagsterInvariantViolationError
|
|
|
27
26
|
if TYPE_CHECKING:
|
|
28
27
|
from dagster import AssetKey
|
|
29
28
|
|
|
29
|
+
from dagster._symbol_annotations.lifecycle import superseded
|
|
30
|
+
|
|
30
31
|
from dagster_dbt.asset_utils import (
|
|
31
32
|
get_asset_keys_to_resource_props,
|
|
32
33
|
get_manifest_and_translator_from_dbt_assets,
|
|
33
34
|
)
|
|
34
35
|
|
|
35
36
|
|
|
36
|
-
@
|
|
37
|
+
@superseded(
|
|
38
|
+
additional_warn_text="Create `FreshnessPolicy` objects for your dbt models by overriding `get_asset_spec` "
|
|
39
|
+
"in your `DagsterDbtTranslator`, or by updating the `translation` configuration of your `DbtProjectComponent` instead."
|
|
40
|
+
)
|
|
37
41
|
def build_freshness_checks_from_dbt_assets(
|
|
38
42
|
*,
|
|
39
43
|
dbt_assets: Sequence[AssetsDefinition],
|
dagster_dbt/version.py
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
__version__ = "0.
|
|
1
|
+
__version__ = "0.28.1"
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: dagster-dbt
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.28.1
|
|
4
4
|
Summary: A Dagster integration for dbt
|
|
5
5
|
Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-dbt
|
|
6
6
|
Author: Dagster Labs
|
|
@@ -15,8 +15,9 @@ Classifier: License :: OSI Approved :: Apache Software License
|
|
|
15
15
|
Classifier: Operating System :: OS Independent
|
|
16
16
|
Requires-Python: >=3.9,<3.14
|
|
17
17
|
License-File: LICENSE
|
|
18
|
-
Requires-Dist: dagster==1.
|
|
18
|
+
Requires-Dist: dagster==1.12.1
|
|
19
19
|
Requires-Dist: dbt-core<1.11,>=1.7
|
|
20
|
+
Requires-Dist: gitpython
|
|
20
21
|
Requires-Dist: Jinja2
|
|
21
22
|
Requires-Dist: networkx
|
|
22
23
|
Requires-Dist: orjson
|
|
@@ -1,24 +1,25 @@
|
|
|
1
1
|
dagster_dbt/__init__.py,sha256=rlPCxCzovXNDiqXBUEcgNVHjZrXQXwcX3kwUhe_69C4,4708
|
|
2
2
|
dagster_dbt/asset_decorator.py,sha256=8OqlypoTwKCs6dAxOuBAO32krbHBQRLVGe6uDzpTSXE,14892
|
|
3
3
|
dagster_dbt/asset_specs.py,sha256=2EdWIhY2QZhtGXM7N-kkkeK3ClnGgYX7ayRi_X11cLg,2741
|
|
4
|
-
dagster_dbt/asset_utils.py,sha256=
|
|
4
|
+
dagster_dbt/asset_utils.py,sha256=6M3i5_quRys2VCQEzPQwBz5MEl2zY2NJIVZ8r2i-3p4,46651
|
|
5
5
|
dagster_dbt/compat.py,sha256=iiAez9pLskfbwLdj6LxIrFvoU-efBpy2BETXQyAsQvY,3638
|
|
6
|
-
dagster_dbt/dagster_dbt_translator.py,sha256
|
|
6
|
+
dagster_dbt/dagster_dbt_translator.py,sha256=skUpl7NxG7ysDfDOs1pSq1c9-ciByN7LrGvO0ZsC4W4,30274
|
|
7
7
|
dagster_dbt/dbt_core_version.py,sha256=w1P62qDdbApXKv0XvUNr2p7FlwiW68csugsNaqmjNjM,38
|
|
8
8
|
dagster_dbt/dbt_manifest.py,sha256=q10Qq1whh-dLfWtFbTYXYbqBVCf0oU8T1yRPyy9ASw0,1307
|
|
9
9
|
dagster_dbt/dbt_manifest_asset_selection.py,sha256=KSEHcVdtfFZaEqSQDUsAx8H8BQe62jZxAOE_CLtwVlI,5072
|
|
10
10
|
dagster_dbt/dbt_project.py,sha256=HtUuNYbKMAPWqTOPgPeJK4u6b1nKyqHrRNB8sYwOWVw,12203
|
|
11
|
+
dagster_dbt/dbt_project_manager.py,sha256=vZAa0JmP8zAslyGGeaa-kP6YuEm_-2jgGyyWlwIq8x8,5937
|
|
11
12
|
dagster_dbt/dbt_version.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
12
13
|
dagster_dbt/errors.py,sha256=a8xag0tjh8OQVkiL10_uwY4hkAgRCH6HtTGaZxIbXZI,1073
|
|
13
|
-
dagster_dbt/freshness_builder.py,sha256=
|
|
14
|
+
dagster_dbt/freshness_builder.py,sha256=DHAC3AGWAwIA7psDf72R0t5p8NjxDytGU9g5xnca1hc,6395
|
|
14
15
|
dagster_dbt/metadata_set.py,sha256=lqjASYoYeM_Ey6r8UsPUkRMwmuAIfFCFvkNm0xW5xTg,512
|
|
15
16
|
dagster_dbt/py.typed,sha256=la67KBlbjXN-_-DfGNcdOcjYumVpKG_Tkw-8n5dnGB4,8
|
|
16
17
|
dagster_dbt/utils.py,sha256=gT6xO7buRolkhc2fa5ySUPfD1eXo3e6RJWZAKkM6yFo,8513
|
|
17
|
-
dagster_dbt/version.py,sha256=
|
|
18
|
+
dagster_dbt/version.py,sha256=ZRQKbgDaGz_yuLk-cUKuk6ZBKCSRKZC8nQd041NRNXk,23
|
|
18
19
|
dagster_dbt/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
19
20
|
dagster_dbt/cli/app.py,sha256=9oBx85uzAkMtBdV39vNm5rxuAGPVYzzDUs6Ek-KL0XY,13516
|
|
20
21
|
dagster_dbt/cloud/__init__.py,sha256=8WKaLuPl_pUG9Cv78GW782vrWQfqK8QtAWegkTxA9r4,441
|
|
21
|
-
dagster_dbt/cloud/asset_defs.py,sha256=
|
|
22
|
+
dagster_dbt/cloud/asset_defs.py,sha256=brrMeblPY-UUhAZ16WKOZ1vP7s9z1WZdHdK0SmSyq8o,29311
|
|
22
23
|
dagster_dbt/cloud/cli.py,sha256=VnKzBjn-BPpjn4nPZm5xSrboAKpRhlCa-4IxsN1ROCo,4525
|
|
23
24
|
dagster_dbt/cloud/ops.py,sha256=rsU4qPCRUUzeHRInZph7YEz_iynwu_KidO5vMAYHX5E,4615
|
|
24
25
|
dagster_dbt/cloud/resources.py,sha256=AWW3VTBS6zrSKeDjMJovbnu54sKena93FoZle6ZSSq8,31747
|
|
@@ -34,13 +35,13 @@ dagster_dbt/cloud_v2/sensor_builder.py,sha256=8mAm-1ZFvoiVVdxSkSQbMxg18aQUY5kazu
|
|
|
34
35
|
dagster_dbt/cloud_v2/types.py,sha256=dI-NIguj582LwTTMXdY5r0U4-INDn9anNy-ciGuEc1s,4136
|
|
35
36
|
dagster_dbt/components/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
36
37
|
dagster_dbt/components/dbt_project/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
37
|
-
dagster_dbt/components/dbt_project/component.py,sha256=
|
|
38
|
-
dagster_dbt/components/dbt_project/scaffolder.py,sha256=
|
|
38
|
+
dagster_dbt/components/dbt_project/component.py,sha256=cDgMNJachQQQzRSALIQLc8APylu03xbtCtIYKqzMiK4,18797
|
|
39
|
+
dagster_dbt/components/dbt_project/scaffolder.py,sha256=tbxvFfcvF5KBdHJmjCIxiujtHeo0iLMwnuVM6Ym_iQk,2914
|
|
39
40
|
dagster_dbt/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
40
41
|
dagster_dbt/core/dbt_cli_event.py,sha256=Yqr-UuzFF7jIcI4KRAmiAhGTX9Ay0tIMN2NaJegsrJQ,25203
|
|
41
42
|
dagster_dbt/core/dbt_cli_invocation.py,sha256=7ZrwAgivAtZGwrHLxfpmNan004IuEEdBSaKZeKp69Mg,17534
|
|
42
43
|
dagster_dbt/core/dbt_event_iterator.py,sha256=s0gU6wIgOpmbo50309FHp9zTqlLkIWcgLkETmidxPRo,16953
|
|
43
|
-
dagster_dbt/core/resource.py,sha256=
|
|
44
|
+
dagster_dbt/core/resource.py,sha256=1qIEneW2aZYeArF5w_QV4-Xmf7Kp9kpGaQaxUlL3ZZs,30851
|
|
44
45
|
dagster_dbt/core/utils.py,sha256=ciXjLhFTNVTyDtVWP4Kjee0LWZkIuVJOltvlv0COzDo,577
|
|
45
46
|
dagster_dbt/include/__init__.py,sha256=8ujr-ROlJ5x64POs-bH-0zfjZ2QHx-FgKUZAvAFbSs8,89
|
|
46
47
|
dagster_dbt/include/pyproject.toml.jinja,sha256=h5Od0HP4c7gQLRNIDyDJLzTMpp5sc4qtrzOVM-R75IA,660
|
|
@@ -50,9 +51,9 @@ dagster_dbt/include/scaffold/assets.py.jinja,sha256=JImqnDUP5ewy8RVti4IuXL70yJno
|
|
|
50
51
|
dagster_dbt/include/scaffold/definitions.py.jinja,sha256=Hou7emwkEeh5YXTdqjYFrAc2SK-Q6MgTNsQOKA_Vy3s,364
|
|
51
52
|
dagster_dbt/include/scaffold/project.py.jinja,sha256=YNtkT5Hq4VbGw-b7QcxdelhXsesIKORwVuBFGFdfeUs,432
|
|
52
53
|
dagster_dbt/include/scaffold/schedules.py.jinja,sha256=Xua_VtPjYFc498A5uaBGQ36GwV1gqciO4P3D8Yt9M-Y,413
|
|
53
|
-
dagster_dbt-0.
|
|
54
|
-
dagster_dbt-0.
|
|
55
|
-
dagster_dbt-0.
|
|
56
|
-
dagster_dbt-0.
|
|
57
|
-
dagster_dbt-0.
|
|
58
|
-
dagster_dbt-0.
|
|
54
|
+
dagster_dbt-0.28.1.dist-info/licenses/LICENSE,sha256=4lsMW-RCvfVD4_F57wrmpe3vX1xwUk_OAKKmV_XT7Z0,11348
|
|
55
|
+
dagster_dbt-0.28.1.dist-info/METADATA,sha256=yKZk-fFNzAbHDBKreGzeSbeCnGXTFnGBPTxCNh3hBWE,1621
|
|
56
|
+
dagster_dbt-0.28.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
57
|
+
dagster_dbt-0.28.1.dist-info/entry_points.txt,sha256=pbv0tVoTB7cByG-noE8rC6atvthh64qBaTo7PkQ9HbM,163
|
|
58
|
+
dagster_dbt-0.28.1.dist-info/top_level.txt,sha256=hoOwFvw9OpJUN1azE6UVHcxMKqhUwR_BTN0Ay-iKUDA,12
|
|
59
|
+
dagster_dbt-0.28.1.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|