dagster-sling 0.27.3__py3-none-any.whl → 0.27.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of dagster-sling might be problematic. Click here for more details.
- dagster_sling/components/sling_replication_collection/component.py +13 -34
- dagster_sling/version.py +1 -1
- {dagster_sling-0.27.3.dist-info → dagster_sling-0.27.4.dist-info}/METADATA +2 -2
- {dagster_sling-0.27.3.dist-info → dagster_sling-0.27.4.dist-info}/RECORD +8 -8
- {dagster_sling-0.27.3.dist-info → dagster_sling-0.27.4.dist-info}/WHEEL +0 -0
- {dagster_sling-0.27.3.dist-info → dagster_sling-0.27.4.dist-info}/entry_points.txt +0 -0
- {dagster_sling-0.27.3.dist-info → dagster_sling-0.27.4.dist-info}/licenses/LICENSE +0 -0
- {dagster_sling-0.27.3.dist-info → dagster_sling-0.27.4.dist-info}/top_level.txt +0 -0
|
@@ -3,7 +3,7 @@ from collections.abc import Iterator, Mapping, Sequence
|
|
|
3
3
|
from dataclasses import dataclass, field
|
|
4
4
|
from functools import cached_property
|
|
5
5
|
from pathlib import Path
|
|
6
|
-
from typing import Annotated, Any,
|
|
6
|
+
from typing import Annotated, Any, Literal, Optional, Union
|
|
7
7
|
|
|
8
8
|
from dagster import Resolvable, Resolver
|
|
9
9
|
from dagster._core.definitions.assets.definition.asset_spec import AssetSpec
|
|
@@ -18,9 +18,9 @@ from dagster._core.definitions.result import MaterializeResult
|
|
|
18
18
|
from dagster.components.component.component import Component
|
|
19
19
|
from dagster.components.core.context import ComponentLoadContext
|
|
20
20
|
from dagster.components.resolved.context import ResolutionContext
|
|
21
|
-
from dagster.components.resolved.core_models import
|
|
21
|
+
from dagster.components.resolved.core_models import AssetPostProcessor, OpSpec
|
|
22
22
|
from dagster.components.scaffold.scaffold import scaffold_with
|
|
23
|
-
from dagster.components.utils import
|
|
23
|
+
from dagster.components.utils.translation import TranslationFn, TranslationFnResolver
|
|
24
24
|
from dagster_shared.utils.warnings import deprecation_warning
|
|
25
25
|
from pydantic import BaseModel, ConfigDict, Field
|
|
26
26
|
from typing_extensions import TypeAlias
|
|
@@ -35,36 +35,8 @@ from dagster_sling.resources import AssetExecutionContext, SlingConnectionResour
|
|
|
35
35
|
SlingMetadataAddons: TypeAlias = Literal["column_metadata", "row_count"]
|
|
36
36
|
|
|
37
37
|
|
|
38
|
-
def resolve_translation(context: ResolutionContext, model):
|
|
39
|
-
info = TranslatorResolvingInfo(
|
|
40
|
-
"stream_definition",
|
|
41
|
-
asset_attributes=model,
|
|
42
|
-
resolution_context=context,
|
|
43
|
-
model_key="translation",
|
|
44
|
-
)
|
|
45
|
-
return lambda base_asset_spec, stream_definition: info.get_asset_spec(
|
|
46
|
-
base_asset_spec,
|
|
47
|
-
{
|
|
48
|
-
"stream_definition": stream_definition,
|
|
49
|
-
"spec": base_asset_spec,
|
|
50
|
-
},
|
|
51
|
-
)
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
TranslationFn: TypeAlias = Callable[[AssetSpec, Mapping[str, Any]], AssetSpec]
|
|
55
|
-
|
|
56
|
-
ResolvedTranslationFn: TypeAlias = Annotated[
|
|
57
|
-
TranslationFn,
|
|
58
|
-
Resolver(
|
|
59
|
-
resolve_translation,
|
|
60
|
-
model_field_type=Union[str, AssetAttributesModel],
|
|
61
|
-
inject_before_resolve=False,
|
|
62
|
-
),
|
|
63
|
-
]
|
|
64
|
-
|
|
65
|
-
|
|
66
38
|
class ProxyDagsterSlingTranslator(DagsterSlingTranslator):
|
|
67
|
-
def __init__(self, fn: TranslationFn):
|
|
39
|
+
def __init__(self, fn: TranslationFn[Mapping[str, Any]]):
|
|
68
40
|
self._fn = fn
|
|
69
41
|
|
|
70
42
|
def get_asset_spec(self, stream_definition: Mapping[str, Any]) -> AssetSpec:
|
|
@@ -76,7 +48,14 @@ class ProxyDagsterSlingTranslator(DagsterSlingTranslator):
|
|
|
76
48
|
class SlingReplicationSpecModel(Resolvable):
|
|
77
49
|
path: str
|
|
78
50
|
op: Optional[OpSpec] = None
|
|
79
|
-
translation: Optional[
|
|
51
|
+
translation: Optional[
|
|
52
|
+
Annotated[
|
|
53
|
+
TranslationFn[Mapping[str, Any]],
|
|
54
|
+
TranslationFnResolver(
|
|
55
|
+
template_vars_for_translation_fn=lambda data: {"stream_definition": data}
|
|
56
|
+
),
|
|
57
|
+
]
|
|
58
|
+
] = None
|
|
80
59
|
include_metadata: list[SlingMetadataAddons] = field(default_factory=list)
|
|
81
60
|
|
|
82
61
|
@cached_property
|
|
@@ -152,7 +131,7 @@ class SlingReplicationCollectionComponent(Component, Resolvable):
|
|
|
152
131
|
[Sling](https://slingdata.io/) is a Powerful Data Integration tool enabling seamless ELT
|
|
153
132
|
operations as well as quality checks across files, databases, and storage systems.
|
|
154
133
|
|
|
155
|
-
dg scaffold dagster_sling.SlingReplicationCollectionComponent {
|
|
134
|
+
dg scaffold dagster_sling.SlingReplicationCollectionComponent {defs_path} to get started.
|
|
156
135
|
|
|
157
136
|
This will create a defs.yaml as well as a `replication.yaml` which is a Sling-specific configuration
|
|
158
137
|
file. See Sling's [documentation](https://docs.slingdata.io/concepts/replication#overview) on `replication.yaml`.
|
dagster_sling/version.py
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
__version__ = "0.27.
|
|
1
|
+
__version__ = "0.27.4"
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: dagster-sling
|
|
3
|
-
Version: 0.27.
|
|
3
|
+
Version: 0.27.4
|
|
4
4
|
Summary: Package for performing ETL/ELT tasks with Sling in Dagster.
|
|
5
5
|
Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-sling
|
|
6
6
|
Author: Dagster Labs
|
|
@@ -15,7 +15,7 @@ Classifier: License :: OSI Approved :: Apache Software License
|
|
|
15
15
|
Classifier: Operating System :: OS Independent
|
|
16
16
|
Requires-Python: >=3.9,<3.14
|
|
17
17
|
License-File: LICENSE
|
|
18
|
-
Requires-Dist: dagster==1.11.
|
|
18
|
+
Requires-Dist: dagster==1.11.4
|
|
19
19
|
Requires-Dist: sling>=1.1.5
|
|
20
20
|
Requires-Dist: sling-mac-arm64; platform_system == "Darwin" and platform_machine == "arm64"
|
|
21
21
|
Provides-Extra: test
|
|
@@ -6,14 +6,14 @@ dagster_sling/py.typed,sha256=la67KBlbjXN-_-DfGNcdOcjYumVpKG_Tkw-8n5dnGB4,8
|
|
|
6
6
|
dagster_sling/resources.py,sha256=JKfrNgvmWmrxKsecR5SK_oxe6z7EqpdfS-6FpDiwyzk,25975
|
|
7
7
|
dagster_sling/sling_event_iterator.py,sha256=zLlcYAO-ZbTw-K0lbaIMDuzQPmhNSRrIMcNRvvWkkNY,8460
|
|
8
8
|
dagster_sling/sling_replication.py,sha256=24Fwuokmc2l_8HBStoTMvZSj77Qpt8ZcKUSAJNq34_M,1113
|
|
9
|
-
dagster_sling/version.py,sha256=
|
|
9
|
+
dagster_sling/version.py,sha256=FRehirBY8kLByuBXp81U_RUAg8WYLFropNPtg2RpV2w,23
|
|
10
10
|
dagster_sling/components/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
11
11
|
dagster_sling/components/sling_replication_collection/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
12
|
-
dagster_sling/components/sling_replication_collection/component.py,sha256=
|
|
12
|
+
dagster_sling/components/sling_replication_collection/component.py,sha256=8lsM2bSNci8n-gQwD8MIvs6YMpBLHtVlrxBIRqU77wQ,8716
|
|
13
13
|
dagster_sling/components/sling_replication_collection/scaffolder.py,sha256=b0L4DtJifCg4FGEQ9E3O9N93WyiGSBlJikSbJOLsq1c,608
|
|
14
|
-
dagster_sling-0.27.
|
|
15
|
-
dagster_sling-0.27.
|
|
16
|
-
dagster_sling-0.27.
|
|
17
|
-
dagster_sling-0.27.
|
|
18
|
-
dagster_sling-0.27.
|
|
19
|
-
dagster_sling-0.27.
|
|
14
|
+
dagster_sling-0.27.4.dist-info/licenses/LICENSE,sha256=4lsMW-RCvfVD4_F57wrmpe3vX1xwUk_OAKKmV_XT7Z0,11348
|
|
15
|
+
dagster_sling-0.27.4.dist-info/METADATA,sha256=RfKxLXcUY4LSUmW1Qg96l2CVfo_1syT8C9TQe8nk3Bk,1182
|
|
16
|
+
dagster_sling-0.27.4.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
17
|
+
dagster_sling-0.27.4.dist-info/entry_points.txt,sha256=X75y2oDwNMmfILdNMk3CNOsskBN-RQnDoFV9D3PgVvc,64
|
|
18
|
+
dagster_sling-0.27.4.dist-info/top_level.txt,sha256=eoJKEGsD6fqIEmF6xaF8tj5Kq9a7riWyRHbZn6oHTk8,14
|
|
19
|
+
dagster_sling-0.27.4.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|