dagster-sling 0.25.9__py3-none-any.whl → 0.25.11__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of dagster-sling might be problematic. Click here for more details.
- dagster_sling/asset_decorator.py +4 -15
- dagster_sling/dagster_sling_translator.py +285 -18
- dagster_sling/resources.py +2 -2
- dagster_sling/version.py +1 -1
- {dagster_sling-0.25.9.dist-info → dagster_sling-0.25.11.dist-info}/METADATA +2 -2
- dagster_sling-0.25.11.dist-info/RECORD +14 -0
- dagster_sling-0.25.9.dist-info/RECORD +0 -14
- {dagster_sling-0.25.9.dist-info → dagster_sling-0.25.11.dist-info}/LICENSE +0 -0
- {dagster_sling-0.25.9.dist-info → dagster_sling-0.25.11.dist-info}/WHEEL +0 -0
- {dagster_sling-0.25.9.dist-info → dagster_sling-0.25.11.dist-info}/top_level.txt +0 -0
dagster_sling/asset_decorator.py
CHANGED
|
@@ -4,7 +4,6 @@ from typing import Any, Callable, Optional
|
|
|
4
4
|
|
|
5
5
|
from dagster import (
|
|
6
6
|
AssetsDefinition,
|
|
7
|
-
AssetSpec,
|
|
8
7
|
BackfillPolicy,
|
|
9
8
|
PartitionsDefinition,
|
|
10
9
|
_check as check,
|
|
@@ -124,23 +123,13 @@ def sling_assets(
|
|
|
124
123
|
op_tags=op_tags,
|
|
125
124
|
backfill_policy=backfill_policy,
|
|
126
125
|
specs=[
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
description=dagster_sling_translator.get_description(stream),
|
|
126
|
+
dagster_sling_translator.get_asset_spec(stream)
|
|
127
|
+
.replace_attributes(code_version=code_version)
|
|
128
|
+
.merge_attributes(
|
|
131
129
|
metadata={
|
|
132
|
-
**dagster_sling_translator.get_metadata(stream),
|
|
133
130
|
METADATA_KEY_TRANSLATOR: dagster_sling_translator,
|
|
134
131
|
METADATA_KEY_REPLICATION_CONFIG: replication_config,
|
|
135
|
-
}
|
|
136
|
-
tags=dagster_sling_translator.get_tags(stream),
|
|
137
|
-
kinds=dagster_sling_translator.get_kinds(stream),
|
|
138
|
-
group_name=dagster_sling_translator.get_group_name(stream),
|
|
139
|
-
freshness_policy=dagster_sling_translator.get_freshness_policy(stream),
|
|
140
|
-
auto_materialize_policy=dagster_sling_translator.get_auto_materialize_policy(
|
|
141
|
-
stream
|
|
142
|
-
),
|
|
143
|
-
code_version=code_version,
|
|
132
|
+
}
|
|
144
133
|
)
|
|
145
134
|
for stream in streams
|
|
146
135
|
],
|
|
@@ -1,16 +1,77 @@
|
|
|
1
1
|
import re
|
|
2
2
|
from collections.abc import Iterable, Mapping
|
|
3
3
|
from dataclasses import dataclass
|
|
4
|
-
from typing import Any, Optional
|
|
4
|
+
from typing import Any, Callable, Optional
|
|
5
5
|
|
|
6
|
-
from dagster import AssetKey, AutoMaterializePolicy, FreshnessPolicy, MetadataValue
|
|
7
|
-
from dagster._annotations import public
|
|
6
|
+
from dagster import AssetKey, AssetSpec, AutoMaterializePolicy, FreshnessPolicy, MetadataValue
|
|
7
|
+
from dagster._annotations import public, superseded
|
|
8
|
+
from dagster._utils.warnings import supersession_warning
|
|
8
9
|
|
|
9
10
|
|
|
10
11
|
@dataclass
|
|
11
12
|
class DagsterSlingTranslator:
|
|
12
13
|
target_prefix: str = "target"
|
|
13
14
|
|
|
15
|
+
@public
|
|
16
|
+
def get_asset_spec(self, stream_definition: Mapping[str, Any]) -> AssetSpec:
|
|
17
|
+
"""A function that takes a stream definition from a Sling replication config and returns a
|
|
18
|
+
Dagster AssetSpec.
|
|
19
|
+
|
|
20
|
+
The stream definition is a dictionary key/value pair where the key is the stream name and
|
|
21
|
+
the value is a dictionary representing the Sling Replication Stream Config.
|
|
22
|
+
"""
|
|
23
|
+
return AssetSpec(
|
|
24
|
+
key=self._resolve_back_compat_method(
|
|
25
|
+
"get_asset_key", self._default_asset_key_fn, stream_definition
|
|
26
|
+
),
|
|
27
|
+
deps=self._resolve_back_compat_method(
|
|
28
|
+
"get_deps_asset_key", self._default_deps_fn, stream_definition
|
|
29
|
+
),
|
|
30
|
+
description=self._resolve_back_compat_method(
|
|
31
|
+
"get_description", self._default_description_fn, stream_definition
|
|
32
|
+
),
|
|
33
|
+
metadata=self._resolve_back_compat_method(
|
|
34
|
+
"get_metadata", self._default_metadata_fn, stream_definition
|
|
35
|
+
),
|
|
36
|
+
tags=self._resolve_back_compat_method(
|
|
37
|
+
"get_tags", self._default_tags_fn, stream_definition
|
|
38
|
+
),
|
|
39
|
+
kinds=self._resolve_back_compat_method(
|
|
40
|
+
"get_kinds", self._default_kinds_fn, stream_definition
|
|
41
|
+
),
|
|
42
|
+
group_name=self._resolve_back_compat_method(
|
|
43
|
+
"get_group_name", self._default_group_name_fn, stream_definition
|
|
44
|
+
),
|
|
45
|
+
freshness_policy=self._resolve_back_compat_method(
|
|
46
|
+
"get_freshness_policy", self._default_freshness_policy_fn, stream_definition
|
|
47
|
+
),
|
|
48
|
+
auto_materialize_policy=self._resolve_back_compat_method(
|
|
49
|
+
"get_auto_materialize_policy",
|
|
50
|
+
self._default_auto_materialize_policy_fn,
|
|
51
|
+
stream_definition,
|
|
52
|
+
),
|
|
53
|
+
)
|
|
54
|
+
|
|
55
|
+
def _resolve_back_compat_method(
|
|
56
|
+
self,
|
|
57
|
+
method_name: str,
|
|
58
|
+
default_fn: Callable[[Mapping[str, Any]], Any],
|
|
59
|
+
stream_definition: Mapping[str, Any],
|
|
60
|
+
):
|
|
61
|
+
method = getattr(type(self), method_name)
|
|
62
|
+
base_method = getattr(DagsterSlingTranslator, method_name)
|
|
63
|
+
if method is not base_method: # user defined this
|
|
64
|
+
supersession_warning(
|
|
65
|
+
subject=method_name,
|
|
66
|
+
additional_warn_text=(
|
|
67
|
+
f"Instead of overriding DagsterSlingTranslator.{method_name}(), "
|
|
68
|
+
f"override DagsterSlingTranslator.get_asset_spec()."
|
|
69
|
+
),
|
|
70
|
+
)
|
|
71
|
+
return method(self, stream_definition)
|
|
72
|
+
else:
|
|
73
|
+
return default_fn(stream_definition)
|
|
74
|
+
|
|
14
75
|
@public
|
|
15
76
|
def sanitize_stream_name(self, stream_name: str) -> str:
|
|
16
77
|
"""A function that takes a stream name from a Sling replication config and returns a
|
|
@@ -33,6 +94,9 @@ class DagsterSlingTranslator:
|
|
|
33
94
|
"""
|
|
34
95
|
return re.sub(r"[^a-zA-Z0-9_.]", "_", stream_name.replace('"', "").lower())
|
|
35
96
|
|
|
97
|
+
@superseded(
|
|
98
|
+
additional_warn_text="Use `DagsterSlingTranslator.get_asset_spec(...).key` instead.",
|
|
99
|
+
)
|
|
36
100
|
@public
|
|
37
101
|
def get_asset_key(self, stream_definition: Mapping[str, Any]) -> AssetKey:
|
|
38
102
|
"""A function that takes a stream definition from a Sling replication config and returns a
|
|
@@ -50,7 +114,7 @@ class DagsterSlingTranslator:
|
|
|
50
114
|
'object': 'public.all_users'}
|
|
51
115
|
}
|
|
52
116
|
|
|
53
|
-
By default, this returns the class's target_prefix
|
|
117
|
+
By default, this returns the class's target_prefix parameter concatenated with the stream name.
|
|
54
118
|
A stream named "public.accounts" will create an AssetKey named "target_public_accounts".
|
|
55
119
|
|
|
56
120
|
Override this function to customize how to map a Sling stream to a Dagster AssetKey.
|
|
@@ -77,9 +141,58 @@ class DagsterSlingTranslator:
|
|
|
77
141
|
.. code-block:: python
|
|
78
142
|
|
|
79
143
|
class CustomSlingTranslator(DagsterSlingTranslator):
|
|
80
|
-
def
|
|
144
|
+
def get_asset_spec(self, stream_definition: Mapping[str, Any]) -> AssetKey:
|
|
145
|
+
default_spec = super().get_asset_spec(stream_definition)
|
|
146
|
+
map = {"stream1": "asset1", "stream2": "asset2"}
|
|
147
|
+
return default_spec.replace_attributes(key=AssetKey(map[stream_definition["name"]]))
|
|
148
|
+
"""
|
|
149
|
+
return self._default_asset_key_fn(stream_definition)
|
|
150
|
+
|
|
151
|
+
def _default_asset_key_fn(self, stream_definition: Mapping[str, Any]) -> AssetKey:
|
|
152
|
+
"""A function that takes a stream definition from a Sling replication config and returns a
|
|
153
|
+
Dagster AssetKey.
|
|
154
|
+
|
|
155
|
+
The stream definition is a dictionary key/value pair where the key is the stream name and
|
|
156
|
+
the value is a dictionary representing the Sling Replication Stream Config.
|
|
157
|
+
|
|
158
|
+
For example:
|
|
159
|
+
|
|
160
|
+
.. code-block:: python
|
|
161
|
+
|
|
162
|
+
stream_definition = {"public.users":
|
|
163
|
+
{'sql': 'select all_user_id, name from public."all_Users"',
|
|
164
|
+
'object': 'public.all_users'}
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
This returns the class's target_prefix parameter concatenated with the stream name.
|
|
168
|
+
A stream named "public.accounts" will create an AssetKey named "target_public_accounts".
|
|
169
|
+
|
|
170
|
+
Alternatively, you can provide metadata in your Sling replication config to specify the
|
|
171
|
+
Dagster AssetKey for a stream as follows:
|
|
172
|
+
|
|
173
|
+
.. code-block:: yaml
|
|
174
|
+
|
|
175
|
+
public.users:
|
|
176
|
+
meta:
|
|
177
|
+
dagster:
|
|
178
|
+
asset_key: "mydb_users"
|
|
179
|
+
|
|
180
|
+
Args:
|
|
181
|
+
stream_definition (Mapping[str, Any]): A dictionary representing the stream definition
|
|
182
|
+
|
|
183
|
+
Returns:
|
|
184
|
+
AssetKey: The Dagster AssetKey for the replication stream.
|
|
185
|
+
|
|
186
|
+
Examples:
|
|
187
|
+
Using a custom mapping for streams:
|
|
188
|
+
|
|
189
|
+
.. code-block:: python
|
|
190
|
+
|
|
191
|
+
class CustomSlingTranslator(DagsterSlingTranslator):
|
|
192
|
+
def get_asset_spec(self, stream_definition: Mapping[str, Any]) -> AssetKey:
|
|
193
|
+
default_spec = super().get_asset_spec(stream_definition)
|
|
81
194
|
map = {"stream1": "asset1", "stream2": "asset2"}
|
|
82
|
-
return AssetKey(map[
|
|
195
|
+
return default_spec.replace_attributes(key=AssetKey(map[stream_definition["name"]]))
|
|
83
196
|
"""
|
|
84
197
|
config = stream_definition.get("config", {}) or {}
|
|
85
198
|
object_key = config.get("object")
|
|
@@ -99,15 +212,20 @@ class DagsterSlingTranslator:
|
|
|
99
212
|
sanitized_components = self.sanitize_stream_name(stream_name).split(".")
|
|
100
213
|
return AssetKey([self.target_prefix] + sanitized_components)
|
|
101
214
|
|
|
215
|
+
@superseded(
|
|
216
|
+
additional_warn_text=(
|
|
217
|
+
"Iterate over `DagsterSlingTranslator.get_asset_spec(...).deps` to access `AssetDep.asset_key` instead."
|
|
218
|
+
),
|
|
219
|
+
)
|
|
102
220
|
@public
|
|
103
221
|
def get_deps_asset_key(self, stream_definition: Mapping[str, Any]) -> Iterable[AssetKey]:
|
|
104
|
-
"""A function that takes a stream
|
|
105
|
-
Dagster AssetKey for
|
|
222
|
+
"""A function that takes a stream definition from a Sling replication config and returns a
|
|
223
|
+
Dagster AssetKey for each dependency of the replication stream.
|
|
106
224
|
|
|
107
225
|
By default, this returns the stream name. For example, a stream named "public.accounts"
|
|
108
226
|
will create an AssetKey named "target_public_accounts" and a dependency named "public_accounts".
|
|
109
227
|
|
|
110
|
-
Override this function to customize how to map a Sling stream to a Dagster
|
|
228
|
+
Override this function to customize how to map a Sling stream to a Dagster dependency.
|
|
111
229
|
Alternatively, you can provide metadata in your Sling replication config to specify the
|
|
112
230
|
Dagster AssetKey for a stream as follows:
|
|
113
231
|
|
|
@@ -119,21 +237,35 @@ class DagsterSlingTranslator:
|
|
|
119
237
|
deps: "sourcedb_users"
|
|
120
238
|
|
|
121
239
|
Args:
|
|
122
|
-
|
|
240
|
+
stream_definition (Mapping[str, Any]): A dictionary representing the stream definition
|
|
123
241
|
|
|
124
242
|
Returns:
|
|
125
|
-
AssetKey:
|
|
243
|
+
Iterable[AssetKey]: A list of Dagster AssetKey for each dependency of the replication stream.
|
|
244
|
+
"""
|
|
245
|
+
return self._default_deps_fn(stream_definition)
|
|
126
246
|
|
|
127
|
-
|
|
128
|
-
|
|
247
|
+
def _default_deps_fn(self, stream_definition: Mapping[str, Any]) -> Iterable[AssetKey]:
|
|
248
|
+
"""A function that takes a stream definition from a Sling replication config and returns a
|
|
249
|
+
Dagster AssetKey for each dependency of the replication stream.
|
|
129
250
|
|
|
130
|
-
|
|
251
|
+
This returns the stream name. For example, a stream named "public.accounts"
|
|
252
|
+
will create an AssetKey named "target_public_accounts" and a dependency named "public_accounts".
|
|
131
253
|
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
254
|
+
Alternatively, you can provide metadata in your Sling replication config to specify the
|
|
255
|
+
Dagster AssetKey for a stream as follows:
|
|
256
|
+
|
|
257
|
+
.. code-block:: yaml
|
|
136
258
|
|
|
259
|
+
public.users:
|
|
260
|
+
meta:
|
|
261
|
+
dagster:
|
|
262
|
+
deps: "sourcedb_users"
|
|
263
|
+
|
|
264
|
+
Args:
|
|
265
|
+
stream_definition (Mapping[str, Any]): A dictionary representing the stream definition
|
|
266
|
+
|
|
267
|
+
Returns:
|
|
268
|
+
Iterable[AssetKey]: A list of Dagster AssetKey for each dependency of the replication stream.
|
|
137
269
|
"""
|
|
138
270
|
config = stream_definition.get("config", {}) or {}
|
|
139
271
|
meta = config.get("meta", {})
|
|
@@ -156,6 +288,9 @@ class DagsterSlingTranslator:
|
|
|
156
288
|
components = self.sanitize_stream_name(stream_name).split(".")
|
|
157
289
|
return [AssetKey(components)]
|
|
158
290
|
|
|
291
|
+
@superseded(
|
|
292
|
+
additional_warn_text="Use `DagsterSlingTranslator.get_asset_spec(...).description` instead.",
|
|
293
|
+
)
|
|
159
294
|
@public
|
|
160
295
|
def get_description(self, stream_definition: Mapping[str, Any]) -> Optional[str]:
|
|
161
296
|
"""Retrieves the description for a given stream definition.
|
|
@@ -164,6 +299,22 @@ class DagsterSlingTranslator:
|
|
|
164
299
|
for an "sql" key in the configuration and returns its value if found. If not, it looks
|
|
165
300
|
for a description in the metadata under the "dagster" key.
|
|
166
301
|
|
|
302
|
+
Parameters:
|
|
303
|
+
stream_definition (Mapping[str, Any]): A dictionary representing the stream definition,
|
|
304
|
+
which includes configuration details.
|
|
305
|
+
|
|
306
|
+
Returns:
|
|
307
|
+
Optional[str]: The description of the stream if found, otherwise None.
|
|
308
|
+
"""
|
|
309
|
+
return self._default_description_fn(stream_definition)
|
|
310
|
+
|
|
311
|
+
def _default_description_fn(self, stream_definition: Mapping[str, Any]) -> Optional[str]:
|
|
312
|
+
"""Retrieves the description for a given stream definition.
|
|
313
|
+
|
|
314
|
+
This method checks the provided stream definition for a description. It first looks
|
|
315
|
+
for an "sql" key in the configuration and returns its value if found. If not, it looks
|
|
316
|
+
for a description in the metadata under the "dagster" key.
|
|
317
|
+
|
|
167
318
|
Parameters:
|
|
168
319
|
stream_definition (Mapping[str, Any]): A dictionary representing the stream definition,
|
|
169
320
|
which includes configuration details.
|
|
@@ -178,6 +329,9 @@ class DagsterSlingTranslator:
|
|
|
178
329
|
description = meta.get("dagster", {}).get("description")
|
|
179
330
|
return description
|
|
180
331
|
|
|
332
|
+
@superseded(
|
|
333
|
+
additional_warn_text="Use `DagsterSlingTranslator.get_asset_spec(...).metadata` instead.",
|
|
334
|
+
)
|
|
181
335
|
@public
|
|
182
336
|
def get_metadata(self, stream_definition: Mapping[str, Any]) -> Mapping[str, Any]:
|
|
183
337
|
"""Retrieves the metadata for a given stream definition.
|
|
@@ -185,6 +339,21 @@ class DagsterSlingTranslator:
|
|
|
185
339
|
This method extracts the configuration from the provided stream definition and returns
|
|
186
340
|
it as a JSON metadata value.
|
|
187
341
|
|
|
342
|
+
Parameters:
|
|
343
|
+
stream_definition (Mapping[str, Any]): A dictionary representing the stream definition,
|
|
344
|
+
which includes configuration details.
|
|
345
|
+
|
|
346
|
+
Returns:
|
|
347
|
+
Mapping[str, Any]: A dictionary containing the stream configuration as JSON metadata.
|
|
348
|
+
"""
|
|
349
|
+
return self._default_metadata_fn(stream_definition)
|
|
350
|
+
|
|
351
|
+
def _default_metadata_fn(self, stream_definition: Mapping[str, Any]) -> Mapping[str, Any]:
|
|
352
|
+
"""Retrieves the metadata for a given stream definition.
|
|
353
|
+
|
|
354
|
+
This method extracts the configuration from the provided stream definition and returns
|
|
355
|
+
it as a JSON metadata value.
|
|
356
|
+
|
|
188
357
|
Parameters:
|
|
189
358
|
stream_definition (Mapping[str, Any]): A dictionary representing the stream definition,
|
|
190
359
|
which includes configuration details.
|
|
@@ -194,6 +363,9 @@ class DagsterSlingTranslator:
|
|
|
194
363
|
"""
|
|
195
364
|
return {"stream_config": MetadataValue.json(stream_definition.get("config", {}))}
|
|
196
365
|
|
|
366
|
+
@superseded(
|
|
367
|
+
additional_warn_text="Use `DagsterSlingTranslator.get_asset_spec(...).tags` instead.",
|
|
368
|
+
)
|
|
197
369
|
@public
|
|
198
370
|
def get_tags(self, stream_definition: Mapping[str, Any]) -> Mapping[str, Any]:
|
|
199
371
|
"""Retrieves the tags for a given stream definition.
|
|
@@ -201,6 +373,21 @@ class DagsterSlingTranslator:
|
|
|
201
373
|
This method returns an empty dictionary, indicating that no tags are associated with
|
|
202
374
|
the stream definition by default. This method can be overridden to provide custom tags.
|
|
203
375
|
|
|
376
|
+
Parameters:
|
|
377
|
+
stream_definition (Mapping[str, Any]): A dictionary representing the stream definition,
|
|
378
|
+
which includes configuration details.
|
|
379
|
+
|
|
380
|
+
Returns:
|
|
381
|
+
Mapping[str, Any]: An empty dictionary.
|
|
382
|
+
"""
|
|
383
|
+
return self._default_tags_fn(stream_definition)
|
|
384
|
+
|
|
385
|
+
def _default_tags_fn(self, stream_definition: Mapping[str, Any]) -> Mapping[str, Any]:
|
|
386
|
+
"""Retrieves the tags for a given stream definition.
|
|
387
|
+
|
|
388
|
+
This method returns an empty dictionary, indicating that no tags are associated with
|
|
389
|
+
the stream definition by default. This method can be overridden to provide custom tags.
|
|
390
|
+
|
|
204
391
|
Parameters:
|
|
205
392
|
stream_definition (Mapping[str, Any]): A dictionary representing the stream definition,
|
|
206
393
|
which includes configuration details.
|
|
@@ -210,12 +397,29 @@ class DagsterSlingTranslator:
|
|
|
210
397
|
"""
|
|
211
398
|
return {}
|
|
212
399
|
|
|
400
|
+
@superseded(
|
|
401
|
+
additional_warn_text="Use `DagsterSlingTranslator.get_asset_spec(...).kinds` instead.",
|
|
402
|
+
)
|
|
213
403
|
@public
|
|
214
404
|
def get_kinds(self, stream_definition: Mapping[str, Any]) -> set[str]:
|
|
215
405
|
"""Retrieves the kinds for a given stream definition.
|
|
216
406
|
|
|
217
407
|
This method returns "sling" by default. This method can be overridden to provide custom kinds.
|
|
218
408
|
|
|
409
|
+
Parameters:
|
|
410
|
+
stream_definition (Mapping[str, Any]): A dictionary representing the stream definition,
|
|
411
|
+
which includes configuration details.
|
|
412
|
+
|
|
413
|
+
Returns:
|
|
414
|
+
Set[str]: A set containing kinds for the stream's assets.
|
|
415
|
+
"""
|
|
416
|
+
return self._default_kinds_fn(stream_definition)
|
|
417
|
+
|
|
418
|
+
def _default_kinds_fn(self, stream_definition: Mapping[str, Any]) -> set[str]:
|
|
419
|
+
"""Retrieves the kinds for a given stream definition.
|
|
420
|
+
|
|
421
|
+
This method returns "sling" by default. This method can be overridden to provide custom kinds.
|
|
422
|
+
|
|
219
423
|
Parameters:
|
|
220
424
|
stream_definition (Mapping[str, Any]): A dictionary representing the stream definition,
|
|
221
425
|
which includes configuration details.
|
|
@@ -225,6 +429,9 @@ class DagsterSlingTranslator:
|
|
|
225
429
|
"""
|
|
226
430
|
return {"sling"}
|
|
227
431
|
|
|
432
|
+
@superseded(
|
|
433
|
+
additional_warn_text="Use `DagsterSlingTranslator.get_asset_spec(...).group_name` instead.",
|
|
434
|
+
)
|
|
228
435
|
@public
|
|
229
436
|
def get_group_name(self, stream_definition: Mapping[str, Any]) -> Optional[str]:
|
|
230
437
|
"""Retrieves the group name for a given stream definition.
|
|
@@ -232,6 +439,21 @@ class DagsterSlingTranslator:
|
|
|
232
439
|
This method checks the provided stream definition for a group name in the metadata
|
|
233
440
|
under the "dagster" key.
|
|
234
441
|
|
|
442
|
+
Parameters:
|
|
443
|
+
stream_definition (Mapping[str, Any]): A dictionary representing the stream definition,
|
|
444
|
+
which includes configuration details.
|
|
445
|
+
|
|
446
|
+
Returns:
|
|
447
|
+
Optional[str]: The group name if found, otherwise None.
|
|
448
|
+
"""
|
|
449
|
+
return self._default_group_name_fn(stream_definition)
|
|
450
|
+
|
|
451
|
+
def _default_group_name_fn(self, stream_definition: Mapping[str, Any]) -> Optional[str]:
|
|
452
|
+
"""Retrieves the group name for a given stream definition.
|
|
453
|
+
|
|
454
|
+
This method checks the provided stream definition for a group name in the metadata
|
|
455
|
+
under the "dagster" key.
|
|
456
|
+
|
|
235
457
|
Parameters:
|
|
236
458
|
stream_definition (Mapping[str, Any]): A dictionary representing the stream definition,
|
|
237
459
|
which includes configuration details.
|
|
@@ -243,6 +465,9 @@ class DagsterSlingTranslator:
|
|
|
243
465
|
meta = config.get("meta", {})
|
|
244
466
|
return meta.get("dagster", {}).get("group")
|
|
245
467
|
|
|
468
|
+
@superseded(
|
|
469
|
+
additional_warn_text="Use `DagsterSlingTranslator.get_asset_spec(...).freshness_policy` instead.",
|
|
470
|
+
)
|
|
246
471
|
@public
|
|
247
472
|
def get_freshness_policy(
|
|
248
473
|
self, stream_definition: Mapping[str, Any]
|
|
@@ -254,6 +479,26 @@ class DagsterSlingTranslator:
|
|
|
254
479
|
returns a FreshnessPolicy object based on the provided parameters. Otherwise,
|
|
255
480
|
it returns None.
|
|
256
481
|
|
|
482
|
+
Parameters:
|
|
483
|
+
stream_definition (Mapping[str, Any]): A dictionary representing the stream definition,
|
|
484
|
+
which includes configuration details.
|
|
485
|
+
|
|
486
|
+
Returns:
|
|
487
|
+
Optional[FreshnessPolicy]: A FreshnessPolicy object if the configuration is found,
|
|
488
|
+
otherwise None.
|
|
489
|
+
"""
|
|
490
|
+
return self._default_freshness_policy_fn(stream_definition)
|
|
491
|
+
|
|
492
|
+
def _default_freshness_policy_fn(
|
|
493
|
+
self, stream_definition: Mapping[str, Any]
|
|
494
|
+
) -> Optional[FreshnessPolicy]:
|
|
495
|
+
"""Retrieves the freshness policy for a given stream definition.
|
|
496
|
+
|
|
497
|
+
This method checks the provided stream definition for a specific configuration
|
|
498
|
+
indicating a freshness policy. If the configuration is found, it constructs and
|
|
499
|
+
returns a FreshnessPolicy object based on the provided parameters. Otherwise,
|
|
500
|
+
it returns None.
|
|
501
|
+
|
|
257
502
|
Parameters:
|
|
258
503
|
stream_definition (Mapping[str, Any]): A dictionary representing the stream definition,
|
|
259
504
|
which includes configuration details.
|
|
@@ -272,6 +517,9 @@ class DagsterSlingTranslator:
|
|
|
272
517
|
cron_schedule_timezone=freshness_policy_config.get("cron_schedule_timezone"),
|
|
273
518
|
)
|
|
274
519
|
|
|
520
|
+
@superseded(
|
|
521
|
+
additional_warn_text="Use `DagsterSlingTranslator.get_asset_spec(...).auto_materialize_policy` instead.",
|
|
522
|
+
)
|
|
275
523
|
@public
|
|
276
524
|
def get_auto_materialize_policy(
|
|
277
525
|
self, stream_definition: Mapping[str, Any]
|
|
@@ -282,6 +530,25 @@ class DagsterSlingTranslator:
|
|
|
282
530
|
indicating an auto-materialize policy. If the configuration is found, it returns
|
|
283
531
|
an eager auto-materialize policy. Otherwise, it returns None.
|
|
284
532
|
|
|
533
|
+
Parameters:
|
|
534
|
+
stream_definition (Mapping[str, Any]): A dictionary representing the stream definition,
|
|
535
|
+
which includes configuration details.
|
|
536
|
+
|
|
537
|
+
Returns:
|
|
538
|
+
Optional[AutoMaterializePolicy]: An eager auto-materialize policy if the configuration
|
|
539
|
+
is found, otherwise None.
|
|
540
|
+
"""
|
|
541
|
+
return self._default_auto_materialize_policy_fn(stream_definition)
|
|
542
|
+
|
|
543
|
+
def _default_auto_materialize_policy_fn(
|
|
544
|
+
self, stream_definition: Mapping[str, Any]
|
|
545
|
+
) -> Optional[AutoMaterializePolicy]:
|
|
546
|
+
"""Defines the auto-materialize policy for a given stream definition.
|
|
547
|
+
|
|
548
|
+
This method checks the provided stream definition for a specific configuration
|
|
549
|
+
indicating an auto-materialize policy. If the configuration is found, it returns
|
|
550
|
+
an eager auto-materialize policy. Otherwise, it returns None.
|
|
551
|
+
|
|
285
552
|
Parameters:
|
|
286
553
|
stream_definition (Mapping[str, Any]): A dictionary representing the stream definition,
|
|
287
554
|
which includes configuration details.
|
dagster_sling/resources.py
CHANGED
|
@@ -198,7 +198,7 @@ class SlingResource(ConfigurableResource):
|
|
|
198
198
|
streams = streams_with_default_dagster_meta(raw_streams, replication_config)
|
|
199
199
|
selected_asset_keys = context.selected_asset_keys
|
|
200
200
|
for stream in streams:
|
|
201
|
-
asset_key = dagster_sling_translator.
|
|
201
|
+
asset_key = dagster_sling_translator.get_asset_spec(stream).key
|
|
202
202
|
if asset_key in selected_asset_keys:
|
|
203
203
|
context_streams.update({stream["name"]: stream["config"]})
|
|
204
204
|
|
|
@@ -401,7 +401,7 @@ class SlingResource(ConfigurableResource):
|
|
|
401
401
|
# TODO: In the future, it'd be nice to yield these materializations as they come in
|
|
402
402
|
# rather than waiting until the end of the replication
|
|
403
403
|
for stream in stream_definitions:
|
|
404
|
-
asset_key = dagster_sling_translator.
|
|
404
|
+
asset_key = dagster_sling_translator.get_asset_spec(stream).key
|
|
405
405
|
|
|
406
406
|
object_key = (stream.get("config") or {}).get("object")
|
|
407
407
|
destination_stream_name = object_key or stream["name"]
|
dagster_sling/version.py
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
__version__ = "0.25.
|
|
1
|
+
__version__ = "0.25.11"
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: dagster-sling
|
|
3
|
-
Version: 0.25.
|
|
3
|
+
Version: 0.25.11
|
|
4
4
|
Summary: Package for performing ETL/ELT tasks with Sling in Dagster.
|
|
5
5
|
Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-sling
|
|
6
6
|
Author: Dagster Labs
|
|
@@ -14,7 +14,7 @@ Classifier: License :: OSI Approved :: Apache Software License
|
|
|
14
14
|
Classifier: Operating System :: OS Independent
|
|
15
15
|
Requires-Python: >=3.9,<3.13
|
|
16
16
|
License-File: LICENSE
|
|
17
|
-
Requires-Dist: dagster ==1.9.
|
|
17
|
+
Requires-Dist: dagster ==1.9.11
|
|
18
18
|
Requires-Dist: sling >=1.1.5
|
|
19
19
|
Provides-Extra: test
|
|
20
20
|
Requires-Dist: duckdb ; extra == 'test'
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
dagster_sling/__init__.py,sha256=H50_KkqkapbNIPGktYROGoUIF5XP8_S_hkxyYxDUHwE,613
|
|
2
|
+
dagster_sling/asset_decorator.py,sha256=TRSCBF5HXDqt8O7laM-IK28b2LS4vF4-JIOTh4kbPYc,5437
|
|
3
|
+
dagster_sling/asset_defs.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
4
|
+
dagster_sling/dagster_sling_translator.py,sha256=66brFssMpkEccvW0AT_juakKglJLJZmYAs1QjOR5k-U,23921
|
|
5
|
+
dagster_sling/py.typed,sha256=la67KBlbjXN-_-DfGNcdOcjYumVpKG_Tkw-8n5dnGB4,8
|
|
6
|
+
dagster_sling/resources.py,sha256=AiFEmFXfimm_76ERyjxZmdosyZ5MPYTigCqOELLsLeg,17613
|
|
7
|
+
dagster_sling/sling_event_iterator.py,sha256=6m7KWUEhkT_yaXMADWTlwWvpch1EUJ1FX5coa48jBP4,8528
|
|
8
|
+
dagster_sling/sling_replication.py,sha256=TFaJsi0C4yUWtYgCG2N4JnAkTdmYXE1ewoaGuqOCNTk,1111
|
|
9
|
+
dagster_sling/version.py,sha256=5Sp-TI-zZwVYq65iJ5awExRCPmlZkpFks4f30p0eWvY,24
|
|
10
|
+
dagster_sling-0.25.11.dist-info/LICENSE,sha256=TMatHW4_G9ldRdodEAp-l2Xa2WvsdeOh60E3v1R2jis,11349
|
|
11
|
+
dagster_sling-0.25.11.dist-info/METADATA,sha256=UYkYG2n3QTOasI7k5s3HHIeZIqSJ8-E3KHr0W75eljA,792
|
|
12
|
+
dagster_sling-0.25.11.dist-info/WHEEL,sha256=yQN5g4mg4AybRjkgi-9yy4iQEFibGQmlz78Pik5Or-A,92
|
|
13
|
+
dagster_sling-0.25.11.dist-info/top_level.txt,sha256=eoJKEGsD6fqIEmF6xaF8tj5Kq9a7riWyRHbZn6oHTk8,14
|
|
14
|
+
dagster_sling-0.25.11.dist-info/RECORD,,
|
|
@@ -1,14 +0,0 @@
|
|
|
1
|
-
dagster_sling/__init__.py,sha256=H50_KkqkapbNIPGktYROGoUIF5XP8_S_hkxyYxDUHwE,613
|
|
2
|
-
dagster_sling/asset_decorator.py,sha256=Qr8B1Ei1o20fXNhMFe4lYWlrLjqcNAwt8yBgFNuCvKs,6092
|
|
3
|
-
dagster_sling/asset_defs.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
4
|
-
dagster_sling/dagster_sling_translator.py,sha256=-0fNPzk_qdaB7ep8Ru2E9XyQPk4lHIlSmDtY4Ba9b4E,12173
|
|
5
|
-
dagster_sling/py.typed,sha256=la67KBlbjXN-_-DfGNcdOcjYumVpKG_Tkw-8n5dnGB4,8
|
|
6
|
-
dagster_sling/resources.py,sha256=clyqe5oSVCJugC_qefk9yRxL5vLyUUwDvhdphOguQuw,17603
|
|
7
|
-
dagster_sling/sling_event_iterator.py,sha256=6m7KWUEhkT_yaXMADWTlwWvpch1EUJ1FX5coa48jBP4,8528
|
|
8
|
-
dagster_sling/sling_replication.py,sha256=TFaJsi0C4yUWtYgCG2N4JnAkTdmYXE1ewoaGuqOCNTk,1111
|
|
9
|
-
dagster_sling/version.py,sha256=NsKiCCQq5j7wW1paL-Bw27h63w_P0r0bIHvsX9TsjGY,23
|
|
10
|
-
dagster_sling-0.25.9.dist-info/LICENSE,sha256=TMatHW4_G9ldRdodEAp-l2Xa2WvsdeOh60E3v1R2jis,11349
|
|
11
|
-
dagster_sling-0.25.9.dist-info/METADATA,sha256=BiqYsJmy1HOIRft3EVGRo8oWfLfWnL5Bs7DINaDaOCk,790
|
|
12
|
-
dagster_sling-0.25.9.dist-info/WHEEL,sha256=yQN5g4mg4AybRjkgi-9yy4iQEFibGQmlz78Pik5Or-A,92
|
|
13
|
-
dagster_sling-0.25.9.dist-info/top_level.txt,sha256=eoJKEGsD6fqIEmF6xaF8tj5Kq9a7riWyRHbZn6oHTk8,14
|
|
14
|
-
dagster_sling-0.25.9.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|