dagster-sling 0.26.7rc0__py3-none-any.whl → 0.26.9__py3-none-any.whl

dagster_sling/__init__.py

@@ -1,6 +1,9 @@
 from dagster_shared.libraries import DagsterLibraryRegistry
 
 from dagster_sling.asset_decorator import sling_assets
+from dagster_sling.components.sling_replication_collection.component import (
+    SlingReplicationCollectionComponent,
+)
 from dagster_sling.dagster_sling_translator import DagsterSlingTranslator
 from dagster_sling.resources import SlingConnectionResource, SlingMode, SlingResource
 from dagster_sling.sling_replication import SlingReplicationParam
@@ -10,6 +13,7 @@ __all__ = [
     "DagsterSlingTranslator",
     "SlingConnectionResource",
     "SlingMode",
+    "SlingReplicationCollectionComponent",
     "SlingReplicationParam",
     "SlingResource",
     "sling_assets",

dagster_sling/components/sling_replication_collection/component.py

@@ -0,0 +1,169 @@
+from collections.abc import Iterator, Mapping, Sequence
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Annotated, Any, Literal, Optional, Union
+
+from dagster._core.definitions.asset_spec import AssetSpec
+from dagster._core.definitions.assets import AssetsDefinition
+from dagster._core.definitions.declarative_automation.automation_condition import (
+    AutomationCondition,
+)
+from dagster._core.definitions.definitions_class import Definitions
+from dagster._core.definitions.events import AssetMaterialization
+from dagster._core.definitions.result import MaterializeResult
+from dagster.components import Resolvable, Resolver
+from dagster.components.component.component import Component
+from dagster.components.core.context import ComponentLoadContext
+from dagster.components.resolved.context import ResolutionContext
+from dagster.components.resolved.core_models import AssetAttributesModel, AssetPostProcessor, OpSpec
+from dagster.components.scaffold.scaffold import scaffold_with
+from dagster.components.utils import TranslatorResolvingInfo
+from typing_extensions import TypeAlias
+
+from dagster_sling.asset_decorator import sling_assets
+from dagster_sling.components.sling_replication_collection.scaffolder import (
+    SlingReplicationComponentScaffolder,
+)
+from dagster_sling.dagster_sling_translator import DagsterSlingTranslator
+from dagster_sling.resources import AssetExecutionContext, SlingResource
+
+SlingMetadataAddons: TypeAlias = Literal["column_metadata", "row_count"]
+
+
+class DagsterSlingTranslatorAutomationCondition(DagsterSlingTranslator):
+    """Subclassed to pull in any overridden impl for get_automation_condition
+    into produced asset specs.
+    """
+
+    def get_automation_condition(
+        self, stream_definition: Mapping[str, Any]
+    ) -> Optional[AutomationCondition]: ...
+
+    def get_asset_spec(self, stream_definition: Mapping[str, Any]) -> AssetSpec:
+        asset_spec = super().get_asset_spec(stream_definition)
+        return asset_spec.replace_attributes(
+            automation_condition=self.get_automation_condition(stream_definition)
+        )
+
+
+class ComponentsDagsterSlingTranslator(DagsterSlingTranslator):
+    def __init__(self, *, resolving_info: TranslatorResolvingInfo):
+        super().__init__()
+        self.resolving_info = resolving_info
+
+    def get_asset_spec(self, obj: Any) -> AssetSpec:  # pyright: ignore[reportIncompatibleMethodOverride]
+        base_spec = super().get_asset_spec(obj)
+        return self.resolving_info.get_asset_spec(
+            base_spec,
+            {self.resolving_info.obj_name: obj, "spec": base_spec},
+        )
+
+
+def resolve_translator(
+    context: ResolutionContext,
+    asset_attributes,
+) -> DagsterSlingTranslator:
+    # TODO: Consider supporting owners and code_version in the future
+    if asset_attributes and isinstance(asset_attributes, AssetAttributesModel):
+        set_vals = asset_attributes.model_dump(exclude_unset=True)
+        if "owners" in set_vals:
+            raise ValueError("owners are not supported for sling_replication_collection component")
+        if "code_version" in set_vals:
+            raise ValueError(
+                "code_version is not supported for sling_replication_collection component"
+            )
+
+    return ComponentsDagsterSlingTranslator(
+        resolving_info=TranslatorResolvingInfo(
+            "stream_definition",
+            asset_attributes or AssetAttributesModel(),
+            context,
+        )
+    )
+
+
+@dataclass
+class SlingReplicationSpecModel(Resolvable):
+    path: str
+    op: Optional[OpSpec] = None
+    translator: Annotated[
+        Optional[DagsterSlingTranslator],
+        Resolver(
+            resolve_translator,
+            model_field_name="asset_attributes",
+            model_field_type=Union[str, AssetAttributesModel],
+        ),
+    ] = None
+    include_metadata: list[SlingMetadataAddons] = field(default_factory=list)
+
+
+def resolve_resource(
+    context: ResolutionContext,
+    sling,
+) -> SlingResource:
+    return SlingResource(**context.resolve_value(sling.model_dump())) if sling else SlingResource()
+
+
+@scaffold_with(SlingReplicationComponentScaffolder)
+@dataclass
+class SlingReplicationCollectionComponent(Component, Resolvable):
+    """Expose one or more Sling replications to Dagster as assets.
+
+    [Sling](https://slingdata.io/) is a Powerful Data Integration tool enabling seamless ELT
+    operations as well as quality checks across files, databases, and storage systems.
+
+    dg scaffold component dagster_sling.SlingReplicationCollectionComponent {component_name} to get started.
+
+    This will create a component.yaml as well as a `replication.yaml` which is a Sling-specific configuration
+    file. See Sling's [documentation](https://docs.slingdata.io/concepts/replication#overview) on `replication.yaml`.
+    """
+
+    resource: Annotated[
+        SlingResource,
+        Resolver(
+            resolve_resource,
+            model_field_name="sling",
+        ),
+    ] = field(default_factory=SlingResource)
+    replications: Sequence[SlingReplicationSpecModel] = field(default_factory=list)
+    asset_post_processors: Optional[Sequence[AssetPostProcessor]] = None
+
+    def build_asset(
+        self, context: ComponentLoadContext, replication_spec_model: SlingReplicationSpecModel
+    ) -> AssetsDefinition:
+        op_spec = replication_spec_model.op or OpSpec()
+
+        @sling_assets(
+            name=op_spec.name or Path(replication_spec_model.path).stem,
+            op_tags=op_spec.tags,
+            replication_config=context.path / replication_spec_model.path,
+            dagster_sling_translator=replication_spec_model.translator,
+            backfill_policy=op_spec.backfill_policy,
+        )
+        def _asset(context: AssetExecutionContext):
+            yield from self.execute(
+                context=context, sling=self.resource, replication_spec_model=replication_spec_model
+            )
+
+        return _asset
+
+    def execute(
+        self,
+        context: AssetExecutionContext,
+        sling: SlingResource,
+        replication_spec_model: SlingReplicationSpecModel,
+    ) -> Iterator[Union[AssetMaterialization, MaterializeResult]]:
+        iterator = sling.replicate(context=context)
+        if "column_metadata" in replication_spec_model.include_metadata:
+            iterator = iterator.fetch_column_metadata()
+        if "row_count" in replication_spec_model.include_metadata:
+            iterator = iterator.fetch_row_count()
+        yield from iterator
+
+    def build_defs(self, context: ComponentLoadContext) -> Definitions:
+        defs = Definitions(
+            assets=[self.build_asset(context, replication) for replication in self.replications],
+        )
+        for post_processor in self.asset_post_processors or []:
+            defs = post_processor(defs)
+        return defs

dagster_sling/components/sling_replication_collection/scaffolder.py

@@ -0,0 +1,13 @@
+from typing import Any
+
+import yaml
+from dagster.components.component.component_scaffolder import Scaffolder, ScaffoldRequest
+from dagster.components.component_scaffolding import scaffold_component
+
+
+class SlingReplicationComponentScaffolder(Scaffolder):
+    def scaffold(self, request: ScaffoldRequest, params: Any) -> None:
+        scaffold_component(request, {"replications": [{"path": "replication.yaml"}]})
+        replication_path = request.target_path / "replication.yaml"
+        with open(replication_path, "w") as f:
+            yaml.dump({"source": {}, "target": {}, "streams": {}}, f)

dagster_sling/version.py

@@ -1 +1 @@
-__version__ = "0.26.7rc0"
+__version__ = "0.26.9"

{dagster_sling-0.26.7rc0.dist-info → dagster_sling-0.26.9.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dagster-sling
-Version: 0.26.7rc0
+Version: 0.26.9
 Summary: Package for performing ETL/ELT tasks with Sling in Dagster.
 Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-sling
 Author: Dagster Labs
@@ -14,7 +14,7 @@ Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Operating System :: OS Independent
 Requires-Python: >=3.9,<3.13
 License-File: LICENSE
-Requires-Dist: dagster ==1.10.7rc0
+Requires-Dist: dagster ==1.10.9
 Requires-Dist: sling >=1.1.5
 Requires-Dist: sling-mac-arm64 ; platform_system == "Darwin" and platform_machine == "arm64"
 Provides-Extra: test

dagster_sling-0.26.9.dist-info/RECORD

@@ -0,0 +1,19 @@
+dagster_sling/__init__.py,sha256=9oGBCLSQ5_1kla5WVkHMlhktcHTHFw4L-F9E8ZfhnZs,778
+dagster_sling/asset_decorator.py,sha256=GUr2_azLV4lzu68IkObhq4NjOxr4LSd7vJW0Ax-2Yrw,5606
+dagster_sling/asset_defs.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+dagster_sling/dagster_sling_translator.py,sha256=66brFssMpkEccvW0AT_juakKglJLJZmYAs1QjOR5k-U,23921
+dagster_sling/py.typed,sha256=la67KBlbjXN-_-DfGNcdOcjYumVpKG_Tkw-8n5dnGB4,8
+dagster_sling/resources.py,sha256=2Jaqcj91ZHUOxhp0KD0DmEJNUVckFyY1-46VkR6Bo3o,25894
+dagster_sling/sling_event_iterator.py,sha256=rVJwngxZATM_dbBcZ9W0EM9N29i8vF9h4l8Y6YNJ6h8,8456
+dagster_sling/sling_replication.py,sha256=TFaJsi0C4yUWtYgCG2N4JnAkTdmYXE1ewoaGuqOCNTk,1111
+dagster_sling/version.py,sha256=ZG17eZIeBB7Zg7MM33Pu3Eg0xdJHjWdE5Wl10Wuq_wo,23
+dagster_sling/components/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+dagster_sling/components/sling_replication_collection/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+dagster_sling/components/sling_replication_collection/component.py,sha256=B3q8mO3SzPwZFMkmyTAgKEi7hWDknY7Bpn0JARSIRiA,6843
+dagster_sling/components/sling_replication_collection/scaffolder.py,sha256=ah4qRzQMlfWYB78AZh_NxcwQuMKzQoCGlIBVR1-DFf8,597
+dagster_sling-0.26.9.dist-info/LICENSE,sha256=TMatHW4_G9ldRdodEAp-l2Xa2WvsdeOh60E3v1R2jis,11349
+dagster_sling-0.26.9.dist-info/METADATA,sha256=6dB3jQZ5zHORiltM3lBzU0TNUhwbvwC0acmMslZzZg8,884
+dagster_sling-0.26.9.dist-info/WHEEL,sha256=yQN5g4mg4AybRjkgi-9yy4iQEFibGQmlz78Pik5Or-A,92
+dagster_sling-0.26.9.dist-info/entry_points.txt,sha256=2ZRJiiClXwxc_iUankInHASgDSLz9cWAoEyAyM-kI_Q,51
+dagster_sling-0.26.9.dist-info/top_level.txt,sha256=eoJKEGsD6fqIEmF6xaF8tj5Kq9a7riWyRHbZn6oHTk8,14
+dagster_sling-0.26.9.dist-info/RECORD,,

dagster_sling-0.26.9.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[dagster_dg.library]
+dagster_sling = dagster_sling

dagster_sling-0.26.7rc0.dist-info/RECORD

@@ -1,14 +0,0 @@
-dagster_sling/__init__.py,sha256=0yJ4OLlOzDt9eAdWuvELQ-Mn9NJ8ubG6lkfzD2rqh50,614
-dagster_sling/asset_decorator.py,sha256=GUr2_azLV4lzu68IkObhq4NjOxr4LSd7vJW0Ax-2Yrw,5606
-dagster_sling/asset_defs.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-dagster_sling/dagster_sling_translator.py,sha256=66brFssMpkEccvW0AT_juakKglJLJZmYAs1QjOR5k-U,23921
-dagster_sling/py.typed,sha256=la67KBlbjXN-_-DfGNcdOcjYumVpKG_Tkw-8n5dnGB4,8
-dagster_sling/resources.py,sha256=2Jaqcj91ZHUOxhp0KD0DmEJNUVckFyY1-46VkR6Bo3o,25894
-dagster_sling/sling_event_iterator.py,sha256=rVJwngxZATM_dbBcZ9W0EM9N29i8vF9h4l8Y6YNJ6h8,8456
-dagster_sling/sling_replication.py,sha256=TFaJsi0C4yUWtYgCG2N4JnAkTdmYXE1ewoaGuqOCNTk,1111
-dagster_sling/version.py,sha256=9hqj21XjzMNl9XZPpk0y8Cd92O1F17GVFLdV80PLJ3k,26
-dagster_sling-0.26.7rc0.dist-info/LICENSE,sha256=TMatHW4_G9ldRdodEAp-l2Xa2WvsdeOh60E3v1R2jis,11349
-dagster_sling-0.26.7rc0.dist-info/METADATA,sha256=jqefWii3l1hC1kBkLQl6x0G8inRXXkeNH82raRKq0j8,890
-dagster_sling-0.26.7rc0.dist-info/WHEEL,sha256=yQN5g4mg4AybRjkgi-9yy4iQEFibGQmlz78Pik5Or-A,92
-dagster_sling-0.26.7rc0.dist-info/top_level.txt,sha256=eoJKEGsD6fqIEmF6xaF8tj5Kq9a7riWyRHbZn6oHTk8,14
-dagster_sling-0.26.7rc0.dist-info/RECORD,,