dagster-sling 0.26.21__py3-none-any.whl → 0.27.1__py3-none-any.whl

dagster_sling/asset_decorator.py

@@ -104,6 +104,22 @@ def sling_assets(
             def my_assets(context, sling: SlingResource):
                 yield from sling.replicate(context=context)
     """
+    return multi_asset(
+        name=name,
+        partitions_def=partitions_def,
+        can_subset=True,
+        op_tags=op_tags,
+        backfill_policy=backfill_policy,
+        specs=get_sling_asset_specs(replication_config, dagster_sling_translator, partitions_def),
+        pool=pool,
+    )
+
+
+def get_sling_asset_specs(
+    replication_config: SlingReplicationParam,
+    dagster_sling_translator: Optional[DagsterSlingTranslator] = None,
+    partitions_def: Optional[PartitionsDefinition] = None,
+) -> list[AssetSpec]:
     replication_config = validate_replication(replication_config)
 
     raw_streams = get_streams_from_replication(replication_config)
@@ -124,22 +140,20 @@ def sling_assets(
             return asset_spec.replace_attributes(code_version=code_version)
         return asset_spec
 
-    return multi_asset(
-        name=name,
-        partitions_def=partitions_def,
-        can_subset=True,
-        op_tags=op_tags,
-        backfill_policy=backfill_policy,
-        specs=[
-            update_code_version_if_unset_by_translator(
-                dagster_sling_translator.get_asset_spec(stream).merge_attributes(
-                    metadata={
-                        METADATA_KEY_TRANSLATOR: dagster_sling_translator,
-                        METADATA_KEY_REPLICATION_CONFIG: replication_config,
-                    }
-                )
+    base_specs = [
+        update_code_version_if_unset_by_translator(
+            dagster_sling_translator.get_asset_spec(stream).merge_attributes(
+                metadata={
+                    METADATA_KEY_TRANSLATOR: dagster_sling_translator,
+                    METADATA_KEY_REPLICATION_CONFIG: replication_config,
+                }
             )
-            for stream in streams
-        ],
-        pool=pool,
-    )
+        )
+        for stream in streams
+    ]
+    return [
+        spec.replace_attributes(
+            partitions_def=partitions_def or spec.partitions_def, skippable=True
+        )
+        for spec in base_specs
+    ]

dagster_sling/components/sling_replication_collection/component.py

@@ -21,6 +21,8 @@ from dagster.components.resolved.context import ResolutionContext
 from dagster.components.resolved.core_models import AssetAttributesModel, AssetPostProcessor, OpSpec
 from dagster.components.scaffold.scaffold import scaffold_with
 from dagster.components.utils import TranslatorResolvingInfo
+from dagster_shared.utils.warnings import deprecation_warning
+from pydantic import BaseModel, ConfigDict, Field
 from typing_extensions import TypeAlias
 
 from dagster_sling.asset_decorator import sling_assets
@@ -28,7 +30,7 @@ from dagster_sling.components.sling_replication_collection.scaffolder import (
     SlingReplicationComponentScaffolder,
 )
 from dagster_sling.dagster_sling_translator import DagsterSlingTranslator
-from dagster_sling.resources import AssetExecutionContext, SlingResource
+from dagster_sling.resources import AssetExecutionContext, SlingConnectionResource, SlingResource
 
 SlingMetadataAddons: TypeAlias = Literal["column_metadata", "row_count"]
 
@@ -87,8 +89,59 @@ class SlingReplicationSpecModel(Resolvable):
 def resolve_resource(
     context: ResolutionContext,
     sling,
-) -> SlingResource:
-    return SlingResource(**context.resolve_value(sling.model_dump())) if sling else SlingResource()
+) -> Optional[SlingResource]:
+    if sling:
+        deprecation_warning(
+            "The `sling` field is deprecated, use `connections` instead. This field will be removed in a future release.",
+            "1.11.1",
+        )
+    return SlingResource(**context.resolve_value(sling.model_dump())) if sling else None
+
+
+def replicate(
+    context: AssetExecutionContext,
+    connections: list[SlingConnectionResource],
+) -> Iterator[Union[AssetMaterialization, MaterializeResult]]:
+    sling = SlingResource(connections=connections)
+    yield from sling.replicate(context=context)
+
+
+class SlingConnectionResourcePropertiesModel(Resolvable, BaseModel):
+    """Properties of a Sling connection resource."""
+
+    # each connection type supports a variety of different properties
+    model_config = ConfigDict(extra="allow")
+
+    type: str = Field(
+        description="Type of the source connection, must match the Sling connection types. Use 'file' for local storage."
+    )
+    connection_string: Optional[str] = Field(
+        description="The optional connection string for the source database, if not using keyword arguments.",
+        default=None,
+    )
+
+
+def resolve_connections(
+    context: ResolutionContext,
+    connections: Mapping[str, SlingConnectionResourcePropertiesModel],
+) -> list[SlingConnectionResource]:
+    return [
+        SlingConnectionResource(
+            name=name,
+            **context.resolve_value(
+                connection if isinstance(connection, dict) else connection.model_dump()
+            ),
+        )
+        for name, connection in connections.items()
+    ]
+
+
+ResolvedSlingConnections: TypeAlias = Annotated[
+    list[SlingConnectionResource],
+    Resolver(
+        resolve_connections, model_field_type=Mapping[str, SlingConnectionResourcePropertiesModel]
+    ),
+]
 
 
 @scaffold_with(SlingReplicationComponentScaffolder)
@@ -105,16 +158,18 @@ class SlingReplicationCollectionComponent(Component, Resolvable):
     file. See Sling's [documentation](https://docs.slingdata.io/concepts/replication#overview) on `replication.yaml`.
     """
 
-    resource: Annotated[
-        SlingResource,
-        Resolver(
-            resolve_resource,
-            model_field_name="sling",
-        ),
-    ] = field(default_factory=SlingResource)
+    connections: ResolvedSlingConnections = field(default_factory=list)
     replications: Sequence[SlingReplicationSpecModel] = field(default_factory=list)
     # TODO: deprecate and then delete -- schrockn 2025-06-10
     asset_post_processors: Optional[Sequence[AssetPostProcessor]] = None
+    resource: Annotated[
+        Optional[SlingResource],
+        Resolver(resolve_resource, model_field_name="sling"),
+    ] = None
+
+    @cached_property
+    def sling_resource(self) -> SlingResource:
+        return self.resource or SlingResource(connections=self.connections)
 
     def build_asset(
         self, context: ComponentLoadContext, replication_spec_model: SlingReplicationSpecModel
@@ -142,7 +197,9 @@ class SlingReplicationCollectionComponent(Component, Resolvable):
         )
         def _asset(context: AssetExecutionContext):
             yield from self.execute(
-                context=context, sling=self.resource, replication_spec_model=replication_spec_model
+                context=context,
+                sling=self.sling_resource,
+                replication_spec_model=replication_spec_model,
             )
 
         return _asset

dagster_sling/dagster_sling_translator.py

@@ -2,7 +2,7 @@ from collections.abc import Iterable, Mapping
 from dataclasses import dataclass
 from typing import Any, Callable, Optional
 
-from dagster import AssetKey, AssetSpec, AutoMaterializePolicy, FreshnessPolicy, MetadataValue
+from dagster import AssetKey, AssetSpec, AutoMaterializePolicy, LegacyFreshnessPolicy, MetadataValue
 from dagster._annotations import public, superseded
 from dagster._utils.names import clean_name_lower_with_dots
 from dagster._utils.warnings import supersession_warning
@@ -42,7 +42,7 @@ class DagsterSlingTranslator:
             group_name=self._resolve_back_compat_method(
                 "get_group_name", self._default_group_name_fn, stream_definition
             ),
-            freshness_policy=self._resolve_back_compat_method(
+            legacy_freshness_policy=self._resolve_back_compat_method(
                 "get_freshness_policy", self._default_freshness_policy_fn, stream_definition
             ),
             auto_materialize_policy=self._resolve_back_compat_method(
@@ -471,7 +471,7 @@ class DagsterSlingTranslator:
     @public
     def get_freshness_policy(
         self, stream_definition: Mapping[str, Any]
-    ) -> Optional[FreshnessPolicy]:
+    ) -> Optional[LegacyFreshnessPolicy]:
         """Retrieves the freshness policy for a given stream definition.
 
         This method checks the provided stream definition for a specific configuration
@@ -491,7 +491,7 @@ class DagsterSlingTranslator:
 
     def _default_freshness_policy_fn(
         self, stream_definition: Mapping[str, Any]
-    ) -> Optional[FreshnessPolicy]:
+    ) -> Optional[LegacyFreshnessPolicy]:
         """Retrieves the freshness policy for a given stream definition.
 
         This method checks the provided stream definition for a specific configuration
@@ -511,7 +511,7 @@ class DagsterSlingTranslator:
         meta = config.get("meta", {})
         freshness_policy_config = meta.get("dagster", {}).get("freshness_policy")
         if freshness_policy_config:
-            return FreshnessPolicy(
+            return LegacyFreshnessPolicy(
                 maximum_lag_minutes=float(freshness_policy_config["maximum_lag_minutes"]),
                 cron_schedule=freshness_policy_config.get("cron_schedule"),
                 cron_schedule_timezone=freshness_policy_config.get("cron_schedule_timezone"),

dagster_sling/version.py

@@ -1 +1 @@
-__version__ = "0.26.21"
+__version__ = "0.27.1"

{dagster_sling-0.26.21.dist-info → dagster_sling-0.27.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dagster-sling
-Version: 0.26.21
+Version: 0.27.1
 Summary: Package for performing ETL/ELT tasks with Sling in Dagster.
 Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-sling
 Author: Dagster Labs
@@ -14,7 +14,7 @@ Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Operating System :: OS Independent
 Requires-Python: >=3.9,<3.13
 License-File: LICENSE
-Requires-Dist: dagster==1.10.21
+Requires-Dist: dagster==1.11.1
 Requires-Dist: sling>=1.1.5
 Requires-Dist: sling-mac-arm64; platform_system == "Darwin" and platform_machine == "arm64"
 Provides-Extra: test

{dagster_sling-0.26.21.dist-info → dagster_sling-0.27.1.dist-info}/RECORD

@@ -1,19 +1,19 @@
 dagster_sling/__init__.py,sha256=bMx84ssIRAWK3TrxK1CMrDiOAgq5ynJvcJX-EKXKjvM,842
-dagster_sling/asset_decorator.py,sha256=2rciTJShIc5faBYel0aj_JAX5dtDeTA31GPGag1NoVs,5922
+dagster_sling/asset_decorator.py,sha256=W86dh-20oHJ4JC9X4nEdrEDH0LAaeTBublUb1qh_zDU,6385
 dagster_sling/asset_defs.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-dagster_sling/dagster_sling_translator.py,sha256=6lH8hcZt0IHuxXaT2XwpX7Pf8ppFdZ9en6qkLo4ZFzs,23964
+dagster_sling/dagster_sling_translator.py,sha256=4HRgqTp5PMxO-73jEvB1oMbguYGfhSlNYVogCtHMWGA,23995
 dagster_sling/py.typed,sha256=la67KBlbjXN-_-DfGNcdOcjYumVpKG_Tkw-8n5dnGB4,8
 dagster_sling/resources.py,sha256=JKfrNgvmWmrxKsecR5SK_oxe6z7EqpdfS-6FpDiwyzk,25975
 dagster_sling/sling_event_iterator.py,sha256=zLlcYAO-ZbTw-K0lbaIMDuzQPmhNSRrIMcNRvvWkkNY,8460
 dagster_sling/sling_replication.py,sha256=24Fwuokmc2l_8HBStoTMvZSj77Qpt8ZcKUSAJNq34_M,1113
-dagster_sling/version.py,sha256=gtsPKN_satzDwTv3o1otQl1HwWzwBXeUyb8XS8-QBNs,24
+dagster_sling/version.py,sha256=lnka9HWRxSmlQAffgSpaSitns-Djhy2OArtj9IVwxrY,23
 dagster_sling/components/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dagster_sling/components/sling_replication_collection/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-dagster_sling/components/sling_replication_collection/component.py,sha256=qbXUcPv4FWbChY15c7uE-32oNZEbKEblSlavYdrn4To,7184
+dagster_sling/components/sling_replication_collection/component.py,sha256=loPqICKK1-WEi_YmRgRarFxqE53-bLPqCFoKUakMjlI,9208
 dagster_sling/components/sling_replication_collection/scaffolder.py,sha256=b0L4DtJifCg4FGEQ9E3O9N93WyiGSBlJikSbJOLsq1c,608
-dagster_sling-0.26.21.dist-info/licenses/LICENSE,sha256=4lsMW-RCvfVD4_F57wrmpe3vX1xwUk_OAKKmV_XT7Z0,11348
-dagster_sling-0.26.21.dist-info/METADATA,sha256=CGY8C8q5Uti7GU50aumnEzARrUMreSOIjQfCOQZL-YY,1133
-dagster_sling-0.26.21.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-dagster_sling-0.26.21.dist-info/entry_points.txt,sha256=X75y2oDwNMmfILdNMk3CNOsskBN-RQnDoFV9D3PgVvc,64
-dagster_sling-0.26.21.dist-info/top_level.txt,sha256=eoJKEGsD6fqIEmF6xaF8tj5Kq9a7riWyRHbZn6oHTk8,14
-dagster_sling-0.26.21.dist-info/RECORD,,
+dagster_sling-0.27.1.dist-info/licenses/LICENSE,sha256=4lsMW-RCvfVD4_F57wrmpe3vX1xwUk_OAKKmV_XT7Z0,11348
+dagster_sling-0.27.1.dist-info/METADATA,sha256=Yc1Z49XVL-Tlgnc8KNrKApgtwGHrbhy561zldPZ0Ysc,1131
+dagster_sling-0.27.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+dagster_sling-0.27.1.dist-info/entry_points.txt,sha256=X75y2oDwNMmfILdNMk3CNOsskBN-RQnDoFV9D3PgVvc,64
+dagster_sling-0.27.1.dist-info/top_level.txt,sha256=eoJKEGsD6fqIEmF6xaF8tj5Kq9a7riWyRHbZn6oHTk8,14
+dagster_sling-0.27.1.dist-info/RECORD,,