dagster-sling 0.26.20__tar.gz → 0.27.0__tar.gz

{dagster_sling-0.26.20/dagster_sling.egg-info → dagster_sling-0.27.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dagster-sling
-Version: 0.26.20
+Version: 0.27.0
 Summary: Package for performing ETL/ELT tasks with Sling in Dagster.
 Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-sling
 Author: Dagster Labs
@@ -14,7 +14,7 @@ Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Operating System :: OS Independent
 Requires-Python: >=3.9,<3.13
 License-File: LICENSE
-Requires-Dist: dagster==1.10.20
+Requires-Dist: dagster==1.11.0
 Requires-Dist: sling>=1.1.5
 Requires-Dist: sling-mac-arm64; platform_system == "Darwin" and platform_machine == "arm64"
 Provides-Extra: test

{dagster_sling-0.26.20 → dagster_sling-0.27.0}/dagster_sling/asset_decorator.py

@@ -104,6 +104,22 @@ def sling_assets(
             def my_assets(context, sling: SlingResource):
                 yield from sling.replicate(context=context)
     """
+    return multi_asset(
+        name=name,
+        partitions_def=partitions_def,
+        can_subset=True,
+        op_tags=op_tags,
+        backfill_policy=backfill_policy,
+        specs=get_sling_asset_specs(replication_config, dagster_sling_translator, partitions_def),
+        pool=pool,
+    )
+
+
+def get_sling_asset_specs(
+    replication_config: SlingReplicationParam,
+    dagster_sling_translator: Optional[DagsterSlingTranslator] = None,
+    partitions_def: Optional[PartitionsDefinition] = None,
+) -> list[AssetSpec]:
     replication_config = validate_replication(replication_config)
 
     raw_streams = get_streams_from_replication(replication_config)
@@ -124,22 +140,20 @@ def sling_assets(
             return asset_spec.replace_attributes(code_version=code_version)
         return asset_spec
 
-    return multi_asset(
-        name=name,
-        partitions_def=partitions_def,
-        can_subset=True,
-        op_tags=op_tags,
-        backfill_policy=backfill_policy,
-        specs=[
-            update_code_version_if_unset_by_translator(
-                dagster_sling_translator.get_asset_spec(stream).merge_attributes(
-                    metadata={
-                        METADATA_KEY_TRANSLATOR: dagster_sling_translator,
-                        METADATA_KEY_REPLICATION_CONFIG: replication_config,
-                    }
-                )
+    base_specs = [
+        update_code_version_if_unset_by_translator(
+            dagster_sling_translator.get_asset_spec(stream).merge_attributes(
+                metadata={
+                    METADATA_KEY_TRANSLATOR: dagster_sling_translator,
+                    METADATA_KEY_REPLICATION_CONFIG: replication_config,
+                }
             )
-            for stream in streams
-        ],
-        pool=pool,
-    )
+        )
+        for stream in streams
+    ]
+    return [
+        spec.replace_attributes(
+            partitions_def=partitions_def or spec.partitions_def, skippable=True
+        )
+        for spec in base_specs
+    ]

{dagster_sling-0.26.20 → dagster_sling-0.27.0}/dagster_sling/components/sling_replication_collection/component.py

@@ -1,3 +1,4 @@
+import textwrap
 from collections.abc import Iterator, Mapping, Sequence
 from dataclasses import dataclass, field
 from functools import cached_property
@@ -20,6 +21,8 @@ from dagster.components.resolved.context import ResolutionContext
 from dagster.components.resolved.core_models import AssetAttributesModel, AssetPostProcessor, OpSpec
 from dagster.components.scaffold.scaffold import scaffold_with
 from dagster.components.utils import TranslatorResolvingInfo
+from dagster_shared.utils.warnings import deprecation_warning
+from pydantic import BaseModel, ConfigDict, Field
 from typing_extensions import TypeAlias
 
 from dagster_sling.asset_decorator import sling_assets
@@ -27,7 +30,7 @@ from dagster_sling.components.sling_replication_collection.scaffolder import (
     SlingReplicationComponentScaffolder,
 )
 from dagster_sling.dagster_sling_translator import DagsterSlingTranslator
-from dagster_sling.resources import AssetExecutionContext, SlingResource
+from dagster_sling.resources import AssetExecutionContext, SlingConnectionResource, SlingResource
 
 SlingMetadataAddons: TypeAlias = Literal["column_metadata", "row_count"]
 
@@ -86,8 +89,57 @@ class SlingReplicationSpecModel(Resolvable):
 def resolve_resource(
     context: ResolutionContext,
     sling,
-) -> SlingResource:
-    return SlingResource(**context.resolve_value(sling.model_dump())) if sling else SlingResource()
+) -> Optional[SlingResource]:
+    if sling:
+        deprecation_warning(
+            "The `sling` field is deprecated, use `connections` instead. This field will be removed in a future release.",
+            "1.11.1",
+        )
+    return SlingResource(**context.resolve_value(sling.model_dump())) if sling else None
+
+
+def replicate(
+    context: AssetExecutionContext,
+    connections: list[SlingConnectionResource],
+) -> Iterator[Union[AssetMaterialization, MaterializeResult]]:
+    sling = SlingResource(connections=connections)
+    yield from sling.replicate(context=context)
+
+
+class SlingConnectionResourcePropertiesModel(Resolvable, BaseModel):
+    """Properties of a Sling connection resource."""
+
+    # each connection type supports a variety of different properties
+    model_config = ConfigDict(extra="allow")
+
+    type: str = Field(
+        description="Type of the source connection, must match the Sling connection types. Use 'file' for local storage."
+    )
+    connection_string: Optional[str] = Field(
+        description="The optional connection string for the source database, if not using keyword arguments.",
+        default=None,
+    )
+
+
+def resolve_connections(
+    context: ResolutionContext,
+    connections: Mapping[str, SlingConnectionResourcePropertiesModel],
+) -> list[SlingConnectionResource]:
+    return [
+        SlingConnectionResource(
+            name=name,
+            **context.resolve_value(connection.model_dump()),
+        )
+        for name, connection in connections.items()
+    ]
+
+
+ResolvedSlingConnections: TypeAlias = Annotated[
+    list[SlingConnectionResource],
+    Resolver(
+        resolve_connections, model_field_type=Mapping[str, SlingConnectionResourcePropertiesModel]
+    ),
+]
 
 
 @scaffold_with(SlingReplicationComponentScaffolder)
@@ -104,15 +156,18 @@ class SlingReplicationCollectionComponent(Component, Resolvable):
     file. See Sling's [documentation](https://docs.slingdata.io/concepts/replication#overview) on `replication.yaml`.
     """
 
-    resource: Annotated[
-        SlingResource,
-        Resolver(
-            resolve_resource,
-            model_field_name="sling",
-        ),
-    ] = field(default_factory=SlingResource)
+    connections: ResolvedSlingConnections = field(default_factory=list)
     replications: Sequence[SlingReplicationSpecModel] = field(default_factory=list)
+    # TODO: deprecate and then delete -- schrockn 2025-06-10
     asset_post_processors: Optional[Sequence[AssetPostProcessor]] = None
+    resource: Annotated[
+        Optional[SlingResource],
+        Resolver(resolve_resource, model_field_name="sling"),
+    ] = None
+
+    @cached_property
+    def sling_resource(self) -> SlingResource:
+        return self.resource or SlingResource(connections=self.connections)
 
     def build_asset(
         self, context: ComponentLoadContext, replication_spec_model: SlingReplicationSpecModel
@@ -140,7 +195,9 @@ class SlingReplicationCollectionComponent(Component, Resolvable):
         )
         def _asset(context: AssetExecutionContext):
             yield from self.execute(
-                context=context, sling=self.resource, replication_spec_model=replication_spec_model
+                context=context,
+                sling=self.sling_resource,
+                replication_spec_model=replication_spec_model,
             )
 
         return _asset
@@ -159,9 +216,25 @@ class SlingReplicationCollectionComponent(Component, Resolvable):
         yield from iterator
 
     def build_defs(self, context: ComponentLoadContext) -> Definitions:
-        defs = Definitions(
+        if self.asset_post_processors:
+            raise Exception(
+                "The asset_post_processors field is deprecated, place your post-processors in the assets"
+                " field in the top-level post_processing field instead, as in this example:\n"
+                + textwrap.dedent(
+                    """
+                    type: dagster_sling.SlingReplicationCollectionComponent
+
+                    attributes: ~
+
+                    post_processing:
+                      assets:
+                        - target: "*"
+                          attributes:
+                            group_name: "my_group"
+                    """
+                )
+            )
+
+        return Definitions(
             assets=[self.build_asset(context, replication) for replication in self.replications],
         )
-        for post_processor in self.asset_post_processors or []:
-            defs = post_processor(defs)
-        return defs

{dagster_sling-0.26.20 → dagster_sling-0.27.0}/dagster_sling/dagster_sling_translator.py

@@ -2,7 +2,7 @@ from collections.abc import Iterable, Mapping
 from dataclasses import dataclass
 from typing import Any, Callable, Optional
 
-from dagster import AssetKey, AssetSpec, AutoMaterializePolicy, FreshnessPolicy, MetadataValue
+from dagster import AssetKey, AssetSpec, AutoMaterializePolicy, LegacyFreshnessPolicy, MetadataValue
 from dagster._annotations import public, superseded
 from dagster._utils.names import clean_name_lower_with_dots
 from dagster._utils.warnings import supersession_warning
@@ -42,7 +42,7 @@ class DagsterSlingTranslator:
             group_name=self._resolve_back_compat_method(
                 "get_group_name", self._default_group_name_fn, stream_definition
             ),
-            freshness_policy=self._resolve_back_compat_method(
+            legacy_freshness_policy=self._resolve_back_compat_method(
                 "get_freshness_policy", self._default_freshness_policy_fn, stream_definition
             ),
             auto_materialize_policy=self._resolve_back_compat_method(
@@ -471,7 +471,7 @@ class DagsterSlingTranslator:
     @public
     def get_freshness_policy(
         self, stream_definition: Mapping[str, Any]
-    ) -> Optional[FreshnessPolicy]:
+    ) -> Optional[LegacyFreshnessPolicy]:
         """Retrieves the freshness policy for a given stream definition.
 
         This method checks the provided stream definition for a specific configuration
@@ -491,7 +491,7 @@ class DagsterSlingTranslator:
 
     def _default_freshness_policy_fn(
         self, stream_definition: Mapping[str, Any]
-    ) -> Optional[FreshnessPolicy]:
+    ) -> Optional[LegacyFreshnessPolicy]:
         """Retrieves the freshness policy for a given stream definition.
 
         This method checks the provided stream definition for a specific configuration
@@ -511,7 +511,7 @@ class DagsterSlingTranslator:
         meta = config.get("meta", {})
         freshness_policy_config = meta.get("dagster", {}).get("freshness_policy")
         if freshness_policy_config:
-            return FreshnessPolicy(
+            return LegacyFreshnessPolicy(
                 maximum_lag_minutes=float(freshness_policy_config["maximum_lag_minutes"]),
                 cron_schedule=freshness_policy_config.get("cron_schedule"),
                 cron_schedule_timezone=freshness_policy_config.get("cron_schedule_timezone"),

{dagster_sling-0.26.20 → dagster_sling-0.27.0}/dagster_sling/resources.py

@@ -635,6 +635,8 @@ def _process_env_vars(config: dict[str, Any]) -> dict[str, Any]:
     for key, value in config.items():
         if isinstance(value, dict) and len(value) == 1 and next(iter(value.keys())) == "env":
             out[key] = EnvVar(next(iter(value.values()))).get_value()
+        elif isinstance(value, EnvVar):
+            out[key] = value.get_value()
         else:
             out[key] = value
     return out

dagster_sling-0.27.0/dagster_sling/version.py

@@ -0,0 +1 @@
+__version__ = "0.27.0"

{dagster_sling-0.26.20 → dagster_sling-0.27.0/dagster_sling.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dagster-sling
-Version: 0.26.20
+Version: 0.27.0
 Summary: Package for performing ETL/ELT tasks with Sling in Dagster.
 Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-sling
 Author: Dagster Labs
@@ -14,7 +14,7 @@ Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Operating System :: OS Independent
 Requires-Python: >=3.9,<3.13
 License-File: LICENSE
-Requires-Dist: dagster==1.10.20
+Requires-Dist: dagster==1.11.0
 Requires-Dist: sling>=1.1.5
 Requires-Dist: sling-mac-arm64; platform_system == "Darwin" and platform_machine == "arm64"
 Provides-Extra: test

{dagster_sling-0.26.20 → dagster_sling-0.27.0}/dagster_sling.egg-info/requires.txt

@@ -1,4 +1,4 @@
-dagster==1.10.20
+dagster==1.11.0
 sling>=1.1.5
 
 [:platform_system == "Darwin" and platform_machine == "arm64"]

{dagster_sling-0.26.20 → dagster_sling-0.27.0}/setup.py

@@ -34,7 +34,7 @@ setup(
     include_package_data=True,
     python_requires=">=3.9,<3.13",
     install_requires=[
-        "dagster==1.10.20",
+        "dagster==1.11.0",
         "sling>=1.1.5",
         # Required due to a bug in uv that can cause sling-linux-amd64 to be installed instead.
         # See: https://github.com/astral-sh/uv/issues/10945

dagster_sling-0.26.20/dagster_sling/version.py

@@ -1 +0,0 @@
-__version__ = "0.26.20"