dagster-sling 0.26.7rc0__tar.gz → 0.26.8__tar.gz

{dagster-sling-0.26.7rc0/dagster_sling.egg-info → dagster-sling-0.26.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dagster-sling
-Version: 0.26.7rc0
+Version: 0.26.8
 Summary: Package for performing ETL/ELT tasks with Sling in Dagster.
 Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-sling
 Author: Dagster Labs

{dagster-sling-0.26.7rc0 → dagster-sling-0.26.8}/dagster_sling/__init__.py

@@ -1,6 +1,9 @@
 from dagster_shared.libraries import DagsterLibraryRegistry
 
 from dagster_sling.asset_decorator import sling_assets
+from dagster_sling.components.sling_replication_collection.component import (
+    SlingReplicationCollectionComponent,
+)
 from dagster_sling.dagster_sling_translator import DagsterSlingTranslator
 from dagster_sling.resources import SlingConnectionResource, SlingMode, SlingResource
 from dagster_sling.sling_replication import SlingReplicationParam
@@ -10,6 +13,7 @@ __all__ = [
     "DagsterSlingTranslator",
     "SlingConnectionResource",
     "SlingMode",
+    "SlingReplicationCollectionComponent",
     "SlingReplicationParam",
     "SlingResource",
     "sling_assets",

dagster-sling-0.26.8/dagster_sling/components/sling_replication_collection/component.py

@@ -0,0 +1,169 @@
+from collections.abc import Iterator, Mapping, Sequence
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Annotated, Any, Literal, Optional, Union
+
+from dagster._core.definitions.asset_spec import AssetSpec
+from dagster._core.definitions.assets import AssetsDefinition
+from dagster._core.definitions.declarative_automation.automation_condition import (
+    AutomationCondition,
+)
+from dagster._core.definitions.definitions_class import Definitions
+from dagster._core.definitions.events import AssetMaterialization
+from dagster._core.definitions.result import MaterializeResult
+from dagster.components import Resolvable, Resolver
+from dagster.components.component.component import Component
+from dagster.components.core.context import ComponentLoadContext
+from dagster.components.resolved.context import ResolutionContext
+from dagster.components.resolved.core_models import AssetAttributesModel, AssetPostProcessor, OpSpec
+from dagster.components.scaffold.scaffold import scaffold_with
+from dagster.components.utils import TranslatorResolvingInfo
+from typing_extensions import TypeAlias
+
+from dagster_sling.asset_decorator import sling_assets
+from dagster_sling.components.sling_replication_collection.scaffolder import (
+    SlingReplicationComponentScaffolder,
+)
+from dagster_sling.dagster_sling_translator import DagsterSlingTranslator
+from dagster_sling.resources import AssetExecutionContext, SlingResource
+
+SlingMetadataAddons: TypeAlias = Literal["column_metadata", "row_count"]
+
+
+class DagsterSlingTranslatorAutomationCondition(DagsterSlingTranslator):
+    """Subclassed to pull in any overridden impl for get_automation_condition
+    into produced asset specs.
+    """
+
+    def get_automation_condition(
+        self, stream_definition: Mapping[str, Any]
+    ) -> Optional[AutomationCondition]: ...
+
+    def get_asset_spec(self, stream_definition: Mapping[str, Any]) -> AssetSpec:
+        asset_spec = super().get_asset_spec(stream_definition)
+        return asset_spec.replace_attributes(
+            automation_condition=self.get_automation_condition(stream_definition)
+        )
+
+
+class ComponentsDagsterSlingTranslator(DagsterSlingTranslator):
+    def __init__(self, *, resolving_info: TranslatorResolvingInfo):
+        super().__init__()
+        self.resolving_info = resolving_info
+
+    def get_asset_spec(self, obj: Any) -> AssetSpec:  # pyright: ignore[reportIncompatibleMethodOverride]
+        base_spec = super().get_asset_spec(obj)
+        return self.resolving_info.get_asset_spec(
+            base_spec,
+            {self.resolving_info.obj_name: obj, "spec": base_spec},
+        )
+
+
+def resolve_translator(
+    context: ResolutionContext,
+    asset_attributes,
+) -> DagsterSlingTranslator:
+    # TODO: Consider supporting owners and code_version in the future
+    if asset_attributes and isinstance(asset_attributes, AssetAttributesModel):
+        set_vals = asset_attributes.model_dump(exclude_unset=True)
+        if "owners" in set_vals:
+            raise ValueError("owners are not supported for sling_replication_collection component")
+        if "code_version" in set_vals:
+            raise ValueError(
+                "code_version is not supported for sling_replication_collection component"
+            )
+
+    return ComponentsDagsterSlingTranslator(
+        resolving_info=TranslatorResolvingInfo(
+            "stream_definition",
+            asset_attributes or AssetAttributesModel(),
+            context,
+        )
+    )
+
+
+@dataclass
+class SlingReplicationSpecModel(Resolvable):
+    path: str
+    op: Optional[OpSpec] = None
+    translator: Annotated[
+        Optional[DagsterSlingTranslator],
+        Resolver(
+            resolve_translator,
+            model_field_name="asset_attributes",
+            model_field_type=Union[str, AssetAttributesModel],
+        ),
+    ] = None
+    include_metadata: list[SlingMetadataAddons] = field(default_factory=list)
+
+
+def resolve_resource(
+    context: ResolutionContext,
+    sling,
+) -> SlingResource:
+    return SlingResource(**context.resolve_value(sling.model_dump())) if sling else SlingResource()
+
+
+@scaffold_with(SlingReplicationComponentScaffolder)
+@dataclass
+class SlingReplicationCollectionComponent(Component, Resolvable):
+    """Expose one or more Sling replications to Dagster as assets.
+
+    [Sling](https://slingdata.io/) is a Powerful Data Integration tool enabling seamless ELT
+    operations as well as quality checks across files, databases, and storage systems.
+
+    dg scaffold component dagster_sling.SlingReplicationCollectionComponent {component_name} to get started.
+
+    This will create a component.yaml as well as a `replication.yaml` which is a Sling-specific configuration
+    file. See Sling's [documentation](https://docs.slingdata.io/concepts/replication#overview) on `replication.yaml`.
+    """
+
+    resource: Annotated[
+        SlingResource,
+        Resolver(
+            resolve_resource,
+            model_field_name="sling",
+        ),
+    ] = field(default_factory=SlingResource)
+    replications: Sequence[SlingReplicationSpecModel] = field(default_factory=list)
+    asset_post_processors: Optional[Sequence[AssetPostProcessor]] = None
+
+    def build_asset(
+        self, context: ComponentLoadContext, replication_spec_model: SlingReplicationSpecModel
+    ) -> AssetsDefinition:
+        op_spec = replication_spec_model.op or OpSpec()
+
+        @sling_assets(
+            name=op_spec.name or Path(replication_spec_model.path).stem,
+            op_tags=op_spec.tags,
+            replication_config=context.path / replication_spec_model.path,
+            dagster_sling_translator=replication_spec_model.translator,
+            backfill_policy=op_spec.backfill_policy,
+        )
+        def _asset(context: AssetExecutionContext):
+            yield from self.execute(
+                context=context, sling=self.resource, replication_spec_model=replication_spec_model
+            )
+
+        return _asset
+
+    def execute(
+        self,
+        context: AssetExecutionContext,
+        sling: SlingResource,
+        replication_spec_model: SlingReplicationSpecModel,
+    ) -> Iterator[Union[AssetMaterialization, MaterializeResult]]:
+        iterator = sling.replicate(context=context)
+        if "column_metadata" in replication_spec_model.include_metadata:
+            iterator = iterator.fetch_column_metadata()
+        if "row_count" in replication_spec_model.include_metadata:
+            iterator = iterator.fetch_row_count()
+        yield from iterator
+
+    def build_defs(self, context: ComponentLoadContext) -> Definitions:
+        defs = Definitions(
+            assets=[self.build_asset(context, replication) for replication in self.replications],
+        )
+        for post_processor in self.asset_post_processors or []:
+            defs = post_processor(defs)
+        return defs

dagster-sling-0.26.8/dagster_sling/components/sling_replication_collection/scaffolder.py

@@ -0,0 +1,13 @@
+from typing import Any
+
+import yaml
+from dagster.components.component.component_scaffolder import Scaffolder, ScaffoldRequest
+from dagster.components.component_scaffolding import scaffold_component
+
+
+class SlingReplicationComponentScaffolder(Scaffolder):
+    def scaffold(self, request: ScaffoldRequest, params: Any) -> None:
+        scaffold_component(request, {"replications": [{"path": "replication.yaml"}]})
+        replication_path = request.target_path / "replication.yaml"
+        with open(replication_path, "w") as f:
+            yaml.dump({"source": {}, "target": {}, "streams": {}}, f)

dagster-sling-0.26.8/dagster_sling/version.py

@@ -0,0 +1 @@
+__version__ = "0.26.8"

{dagster-sling-0.26.7rc0 → dagster-sling-0.26.8/dagster_sling.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dagster-sling
-Version: 0.26.7rc0
+Version: 0.26.8
 Summary: Package for performing ETL/ELT tasks with Sling in Dagster.
 Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-sling
 Author: Dagster Labs

{dagster-sling-0.26.7rc0 → dagster-sling-0.26.8}/dagster_sling.egg-info/SOURCES.txt

@@ -16,6 +16,11 @@ dagster_sling/version.py
 dagster_sling.egg-info/PKG-INFO
 dagster_sling.egg-info/SOURCES.txt
 dagster_sling.egg-info/dependency_links.txt
+dagster_sling.egg-info/entry_points.txt
 dagster_sling.egg-info/not-zip-safe
 dagster_sling.egg-info/requires.txt
-dagster_sling.egg-info/top_level.txt
+dagster_sling.egg-info/top_level.txt
+dagster_sling/components/__init__.py
+dagster_sling/components/sling_replication_collection/__init__.py
+dagster_sling/components/sling_replication_collection/component.py
+dagster_sling/components/sling_replication_collection/scaffolder.py

dagster-sling-0.26.8/dagster_sling.egg-info/entry_points.txt

@@ -0,0 +1,2 @@
+[dagster_dg.library]
+dagster_sling = dagster_sling

{dagster-sling-0.26.7rc0 → dagster-sling-0.26.8}/dagster_sling.egg-info/requires.txt

@@ -1,4 +1,4 @@
-dagster==1.10.7rc0
+dagster==1.10.8
 sling>=1.1.5
 
 [:platform_system == "Darwin" and platform_machine == "arm64"]

{dagster-sling-0.26.7rc0 → dagster-sling-0.26.8}/setup.py

@@ -34,7 +34,7 @@ setup(
     include_package_data=True,
     python_requires=">=3.9,<3.13",
     install_requires=[
-        "dagster==1.10.7rc0",
+        "dagster==1.10.8",
         "sling>=1.1.5",
         # Required due to a bug in uv that can cause sling-linux-amd64 to be installed instead.
         # See: https://github.com/astral-sh/uv/issues/10945
@@ -46,4 +46,9 @@ setup(
             "duckdb",
         ]
     },
+    entry_points={
+        "dagster_dg.library": [
+            "dagster_sling = dagster_sling",
+        ],
+    },
 )

dagster-sling-0.26.7rc0/dagster_sling/version.py

@@ -1 +0,0 @@
-__version__ = "0.26.7rc0"