dagster-sling 0.25.9__tar.gz → 0.25.11__tar.gz

{dagster-sling-0.25.9/dagster_sling.egg-info → dagster-sling-0.25.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dagster-sling
-Version: 0.25.9
+Version: 0.25.11
 Summary: Package for performing ETL/ELT tasks with Sling in Dagster.
 Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-sling
 Author: Dagster Labs

{dagster-sling-0.25.9 → dagster-sling-0.25.11}/dagster_sling/asset_decorator.py

@@ -4,7 +4,6 @@ from typing import Any, Callable, Optional
 
 from dagster import (
     AssetsDefinition,
-    AssetSpec,
     BackfillPolicy,
     PartitionsDefinition,
     _check as check,
@@ -124,23 +123,13 @@ def sling_assets(
         op_tags=op_tags,
         backfill_policy=backfill_policy,
         specs=[
-            AssetSpec(
-                key=dagster_sling_translator.get_asset_key(stream),
-                deps=dagster_sling_translator.get_deps_asset_key(stream),
-                description=dagster_sling_translator.get_description(stream),
+            dagster_sling_translator.get_asset_spec(stream)
+            .replace_attributes(code_version=code_version)
+            .merge_attributes(
                 metadata={
-                    **dagster_sling_translator.get_metadata(stream),
                     METADATA_KEY_TRANSLATOR: dagster_sling_translator,
                     METADATA_KEY_REPLICATION_CONFIG: replication_config,
-                },
-                tags=dagster_sling_translator.get_tags(stream),
-                kinds=dagster_sling_translator.get_kinds(stream),
-                group_name=dagster_sling_translator.get_group_name(stream),
-                freshness_policy=dagster_sling_translator.get_freshness_policy(stream),
-                auto_materialize_policy=dagster_sling_translator.get_auto_materialize_policy(
-                    stream
-                ),
-                code_version=code_version,
+                }
             )
             for stream in streams
         ],

dagster-sling-0.25.11/dagster_sling/dagster_sling_translator.py

@@ -0,0 +1,564 @@
+import re
+from collections.abc import Iterable, Mapping
+from dataclasses import dataclass
+from typing import Any, Callable, Optional
+
+from dagster import AssetKey, AssetSpec, AutoMaterializePolicy, FreshnessPolicy, MetadataValue
+from dagster._annotations import public, superseded
+from dagster._utils.warnings import supersession_warning
+
+
+@dataclass
+class DagsterSlingTranslator:
+    target_prefix: str = "target"
+
+    @public
+    def get_asset_spec(self, stream_definition: Mapping[str, Any]) -> AssetSpec:
+        """A function that takes a stream definition from a Sling replication config and returns a
+        Dagster AssetSpec.
+
+        The stream definition is a dictionary key/value pair where the key is the stream name and
+        the value is a dictionary representing the Sling Replication Stream Config.
+        """
+        return AssetSpec(
+            key=self._resolve_back_compat_method(
+                "get_asset_key", self._default_asset_key_fn, stream_definition
+            ),
+            deps=self._resolve_back_compat_method(
+                "get_deps_asset_key", self._default_deps_fn, stream_definition
+            ),
+            description=self._resolve_back_compat_method(
+                "get_description", self._default_description_fn, stream_definition
+            ),
+            metadata=self._resolve_back_compat_method(
+                "get_metadata", self._default_metadata_fn, stream_definition
+            ),
+            tags=self._resolve_back_compat_method(
+                "get_tags", self._default_tags_fn, stream_definition
+            ),
+            kinds=self._resolve_back_compat_method(
+                "get_kinds", self._default_kinds_fn, stream_definition
+            ),
+            group_name=self._resolve_back_compat_method(
+                "get_group_name", self._default_group_name_fn, stream_definition
+            ),
+            freshness_policy=self._resolve_back_compat_method(
+                "get_freshness_policy", self._default_freshness_policy_fn, stream_definition
+            ),
+            auto_materialize_policy=self._resolve_back_compat_method(
+                "get_auto_materialize_policy",
+                self._default_auto_materialize_policy_fn,
+                stream_definition,
+            ),
+        )
+
+    def _resolve_back_compat_method(
+        self,
+        method_name: str,
+        default_fn: Callable[[Mapping[str, Any]], Any],
+        stream_definition: Mapping[str, Any],
+    ):
+        method = getattr(type(self), method_name)
+        base_method = getattr(DagsterSlingTranslator, method_name)
+        if method is not base_method:  # user defined this
+            supersession_warning(
+                subject=method_name,
+                additional_warn_text=(
+                    f"Instead of overriding DagsterSlingTranslator.{method_name}(), "
+                    f"override DagsterSlingTranslator.get_asset_spec()."
+                ),
+            )
+            return method(self, stream_definition)
+        else:
+            return default_fn(stream_definition)
+
+    @public
+    def sanitize_stream_name(self, stream_name: str) -> str:
+        """A function that takes a stream name from a Sling replication config and returns a
+        sanitized name for the stream.
+
+        By default, this removes any non-alphanumeric characters from the stream name and replaces
+        them with underscores, while removing any double quotes.
+
+        Args:
+            stream_name (str): The name of the stream.
+
+        Examples:
+            Using a custom stream name sanitizer:
+
+            .. code-block:: python
+
+                class CustomSlingTranslator(DagsterSlingTranslator):
+                    def sanitize_stream_name(self, stream_name: str) -> str:
+                        return stream_name.replace(".", "")
+        """
+        return re.sub(r"[^a-zA-Z0-9_.]", "_", stream_name.replace('"', "").lower())
+
+    @superseded(
+        additional_warn_text="Use `DagsterSlingTranslator.get_asset_spec(...).key` instead.",
+    )
+    @public
+    def get_asset_key(self, stream_definition: Mapping[str, Any]) -> AssetKey:
+        """A function that takes a stream definition from a Sling replication config and returns a
+        Dagster AssetKey.
+
+        The stream definition is a dictionary key/value pair where the key is the stream name and
+        the value is a dictionary representing the Sling Replication Stream Config.
+
+        For example:
+
+        .. code-block:: python
+
+            stream_definition = {"public.users":
+                {'sql': 'select all_user_id, name from public."all_Users"',
+                'object': 'public.all_users'}
+            }
+
+        By default, this returns the class's target_prefix parameter concatenated with the stream name.
+        A stream named "public.accounts" will create an AssetKey named "target_public_accounts".
+
+        Override this function to customize how to map a Sling stream to a Dagster AssetKey.
+
+        Alternatively, you can provide metadata in your Sling replication config to specify the
+        Dagster AssetKey for a stream as follows:
+
+        .. code-block:: yaml
+
+            public.users:
+               meta:
+                 dagster:
+                   asset_key: "mydb_users"
+
+        Args:
+            stream_definition (Mapping[str, Any]): A dictionary representing the stream definition
+
+        Returns:
+            AssetKey: The Dagster AssetKey for the replication stream.
+
+        Examples:
+            Using a custom mapping for streams:
+
+            .. code-block:: python
+
+                class CustomSlingTranslator(DagsterSlingTranslator):
+                    def get_asset_spec(self, stream_definition: Mapping[str, Any]) -> AssetKey:
+                        default_spec = super().get_asset_spec(stream_definition)
+                        map = {"stream1": "asset1", "stream2": "asset2"}
+                        return default_spec.replace_attributes(key=AssetKey(map[stream_definition["name"]]))
+        """
+        return self._default_asset_key_fn(stream_definition)
+
+    def _default_asset_key_fn(self, stream_definition: Mapping[str, Any]) -> AssetKey:
+        """A function that takes a stream definition from a Sling replication config and returns a
+        Dagster AssetKey.
+
+        The stream definition is a dictionary key/value pair where the key is the stream name and
+        the value is a dictionary representing the Sling Replication Stream Config.
+
+        For example:
+
+        .. code-block:: python
+
+            stream_definition = {"public.users":
+                {'sql': 'select all_user_id, name from public."all_Users"',
+                'object': 'public.all_users'}
+            }
+
+        This returns the class's target_prefix parameter concatenated with the stream name.
+        A stream named "public.accounts" will create an AssetKey named "target_public_accounts".
+
+        Alternatively, you can provide metadata in your Sling replication config to specify the
+        Dagster AssetKey for a stream as follows:
+
+        .. code-block:: yaml
+
+            public.users:
+               meta:
+                 dagster:
+                   asset_key: "mydb_users"
+
+        Args:
+            stream_definition (Mapping[str, Any]): A dictionary representing the stream definition
+
+        Returns:
+            AssetKey: The Dagster AssetKey for the replication stream.
+
+        Examples:
+            Using a custom mapping for streams:
+
+            .. code-block:: python
+
+                class CustomSlingTranslator(DagsterSlingTranslator):
+                    def get_asset_spec(self, stream_definition: Mapping[str, Any]) -> AssetKey:
+                        default_spec = super().get_asset_spec(stream_definition)
+                        map = {"stream1": "asset1", "stream2": "asset2"}
+                        return default_spec.replace_attributes(key=AssetKey(map[stream_definition["name"]]))
+        """
+        config = stream_definition.get("config", {}) or {}
+        object_key = config.get("object")
+        meta = config.get("meta", {})
+        asset_key = meta.get("dagster", {}).get("asset_key")
+
+        if asset_key:
+            if self.sanitize_stream_name(asset_key) != asset_key:
+                raise ValueError(
+                    f"Asset key {asset_key} for stream {stream_definition['name']} is not "
+                    "sanitized. Please use only alphanumeric characters and underscores."
+                )
+            return AssetKey(asset_key.split("."))
+
+        # You can override the Sling Replication default object with an object key
+        stream_name = object_key or stream_definition["name"]
+        sanitized_components = self.sanitize_stream_name(stream_name).split(".")
+        return AssetKey([self.target_prefix] + sanitized_components)
+
+    @superseded(
+        additional_warn_text=(
+            "Iterate over `DagsterSlingTranslator.get_asset_spec(...).deps` to access `AssetDep.asset_key` instead."
+        ),
+    )
+    @public
+    def get_deps_asset_key(self, stream_definition: Mapping[str, Any]) -> Iterable[AssetKey]:
+        """A function that takes a stream definition from a Sling replication config and returns a
+        Dagster AssetKey for each dependency of the replication stream.
+
+        By default, this returns the stream name. For example, a stream named "public.accounts"
+        will create an AssetKey named "target_public_accounts" and a dependency named "public_accounts".
+
+        Override this function to customize how to map a Sling stream to a Dagster dependency.
+        Alternatively, you can provide metadata in your Sling replication config to specify the
+        Dagster AssetKey for a stream as follows:
+
+        .. code-block:: yaml
+
+            public.users:
+                meta:
+                    dagster:
+                        deps: "sourcedb_users"
+
+        Args:
+            stream_definition (Mapping[str, Any]): A dictionary representing the stream definition
+
+        Returns:
+            Iterable[AssetKey]: A list of Dagster AssetKey for each dependency of the replication stream.
+        """
+        return self._default_deps_fn(stream_definition)
+
+    def _default_deps_fn(self, stream_definition: Mapping[str, Any]) -> Iterable[AssetKey]:
+        """A function that takes a stream definition from a Sling replication config and returns a
+        Dagster AssetKey for each dependency of the replication stream.
+
+        This returns the stream name. For example, a stream named "public.accounts"
+        will create an AssetKey named "target_public_accounts" and a dependency named "public_accounts".
+
+        Alternatively, you can provide metadata in your Sling replication config to specify the
+        Dagster AssetKey for a stream as follows:
+
+        .. code-block:: yaml
+
+            public.users:
+                meta:
+                    dagster:
+                        deps: "sourcedb_users"
+
+        Args:
+            stream_definition (Mapping[str, Any]): A dictionary representing the stream definition
+
+        Returns:
+            Iterable[AssetKey]: A list of Dagster AssetKey for each dependency of the replication stream.
+        """
+        config = stream_definition.get("config", {}) or {}
+        meta = config.get("meta", {})
+        deps = meta.get("dagster", {}).get("deps")
+        deps_out = []
+        if deps and isinstance(deps, str):
+            deps = [deps]
+        if deps:
+            assert isinstance(deps, list)
+            for asset_key in deps:
+                if self.sanitize_stream_name(asset_key) != asset_key:
+                    raise ValueError(
+                        f"Deps Asset key {asset_key} for stream {stream_definition['name']} is not "
+                        "sanitized. Please use only alphanumeric characters and underscores."
+                    )
+                deps_out.append(AssetKey(asset_key.split(".")))
+            return deps_out
+
+        stream_name = stream_definition["name"]
+        components = self.sanitize_stream_name(stream_name).split(".")
+        return [AssetKey(components)]
+
+    @superseded(
+        additional_warn_text="Use `DagsterSlingTranslator.get_asset_spec(...).description` instead.",
+    )
+    @public
+    def get_description(self, stream_definition: Mapping[str, Any]) -> Optional[str]:
+        """Retrieves the description for a given stream definition.
+
+        This method checks the provided stream definition for a description. It first looks
+        for an "sql" key in the configuration and returns its value if found. If not, it looks
+        for a description in the metadata under the "dagster" key.
+
+        Parameters:
+            stream_definition (Mapping[str, Any]): A dictionary representing the stream definition,
+            which includes configuration details.
+
+        Returns:
+            Optional[str]: The description of the stream if found, otherwise None.
+        """
+        return self._default_description_fn(stream_definition)
+
+    def _default_description_fn(self, stream_definition: Mapping[str, Any]) -> Optional[str]:
+        """Retrieves the description for a given stream definition.
+
+        This method checks the provided stream definition for a description. It first looks
+        for an "sql" key in the configuration and returns its value if found. If not, it looks
+        for a description in the metadata under the "dagster" key.
+
+        Parameters:
+            stream_definition (Mapping[str, Any]): A dictionary representing the stream definition,
+            which includes configuration details.
+
+        Returns:
+            Optional[str]: The description of the stream if found, otherwise None.
+        """
+        config = stream_definition.get("config", {}) or {}
+        if "sql" in config:
+            return config["sql"]
+        meta = config.get("meta", {})
+        description = meta.get("dagster", {}).get("description")
+        return description
+
+    @superseded(
+        additional_warn_text="Use `DagsterSlingTranslator.get_asset_spec(...).metadata` instead.",
+    )
+    @public
+    def get_metadata(self, stream_definition: Mapping[str, Any]) -> Mapping[str, Any]:
+        """Retrieves the metadata for a given stream definition.
+
+        This method extracts the configuration from the provided stream definition and returns
+        it as a JSON metadata value.
+
+        Parameters:
+            stream_definition (Mapping[str, Any]): A dictionary representing the stream definition,
+            which includes configuration details.
+
+        Returns:
+            Mapping[str, Any]: A dictionary containing the stream configuration as JSON metadata.
+        """
+        return self._default_metadata_fn(stream_definition)
+
+    def _default_metadata_fn(self, stream_definition: Mapping[str, Any]) -> Mapping[str, Any]:
+        """Retrieves the metadata for a given stream definition.
+
+        This method extracts the configuration from the provided stream definition and returns
+        it as a JSON metadata value.
+
+        Parameters:
+            stream_definition (Mapping[str, Any]): A dictionary representing the stream definition,
+            which includes configuration details.
+
+        Returns:
+            Mapping[str, Any]: A dictionary containing the stream configuration as JSON metadata.
+        """
+        return {"stream_config": MetadataValue.json(stream_definition.get("config", {}))}
+
+    @superseded(
+        additional_warn_text="Use `DagsterSlingTranslator.get_asset_spec(...).tags` instead.",
+    )
+    @public
+    def get_tags(self, stream_definition: Mapping[str, Any]) -> Mapping[str, Any]:
+        """Retrieves the tags for a given stream definition.
+
+        This method returns an empty dictionary, indicating that no tags are associated with
+        the stream definition by default. This method can be overridden to provide custom tags.
+
+        Parameters:
+            stream_definition (Mapping[str, Any]): A dictionary representing the stream definition,
+            which includes configuration details.
+
+        Returns:
+            Mapping[str, Any]: An empty dictionary.
+        """
+        return self._default_tags_fn(stream_definition)
+
+    def _default_tags_fn(self, stream_definition: Mapping[str, Any]) -> Mapping[str, Any]:
+        """Retrieves the tags for a given stream definition.
+
+        This method returns an empty dictionary, indicating that no tags are associated with
+        the stream definition by default. This method can be overridden to provide custom tags.
+
+        Parameters:
+            stream_definition (Mapping[str, Any]): A dictionary representing the stream definition,
+            which includes configuration details.
+
+        Returns:
+            Mapping[str, Any]: An empty dictionary.
+        """
+        return {}
+
+    @superseded(
+        additional_warn_text="Use `DagsterSlingTranslator.get_asset_spec(...).kinds` instead.",
+    )
+    @public
+    def get_kinds(self, stream_definition: Mapping[str, Any]) -> set[str]:
+        """Retrieves the kinds for a given stream definition.
+
+        This method returns "sling" by default. This method can be overridden to provide custom kinds.
+
+        Parameters:
+            stream_definition (Mapping[str, Any]): A dictionary representing the stream definition,
+            which includes configuration details.
+
+        Returns:
+            Set[str]: A set containing kinds for the stream's assets.
+        """
+        return self._default_kinds_fn(stream_definition)
+
+    def _default_kinds_fn(self, stream_definition: Mapping[str, Any]) -> set[str]:
+        """Retrieves the kinds for a given stream definition.
+
+        This method returns "sling" by default. This method can be overridden to provide custom kinds.
+
+        Parameters:
+            stream_definition (Mapping[str, Any]): A dictionary representing the stream definition,
+            which includes configuration details.
+
+        Returns:
+            Set[str]: A set containing kinds for the stream's assets.
+        """
+        return {"sling"}
+
+    @superseded(
+        additional_warn_text="Use `DagsterSlingTranslator.get_asset_spec(...).group_name` instead.",
+    )
+    @public
+    def get_group_name(self, stream_definition: Mapping[str, Any]) -> Optional[str]:
+        """Retrieves the group name for a given stream definition.
+
+        This method checks the provided stream definition for a group name in the metadata
+        under the "dagster" key.
+
+        Parameters:
+            stream_definition (Mapping[str, Any]): A dictionary representing the stream definition,
+            which includes configuration details.
+
+        Returns:
+            Optional[str]: The group name if found, otherwise None.
+        """
+        return self._default_group_name_fn(stream_definition)
+
+    def _default_group_name_fn(self, stream_definition: Mapping[str, Any]) -> Optional[str]:
+        """Retrieves the group name for a given stream definition.
+
+        This method checks the provided stream definition for a group name in the metadata
+        under the "dagster" key.
+
+        Parameters:
+            stream_definition (Mapping[str, Any]): A dictionary representing the stream definition,
+            which includes configuration details.
+
+        Returns:
+            Optional[str]: The group name if found, otherwise None.
+        """
+        config = stream_definition.get("config", {}) or {}
+        meta = config.get("meta", {})
+        return meta.get("dagster", {}).get("group")
+
+    @superseded(
+        additional_warn_text="Use `DagsterSlingTranslator.get_asset_spec(...).freshness_policy` instead.",
+    )
+    @public
+    def get_freshness_policy(
+        self, stream_definition: Mapping[str, Any]
+    ) -> Optional[FreshnessPolicy]:
+        """Retrieves the freshness policy for a given stream definition.
+
+        This method checks the provided stream definition for a specific configuration
+        indicating a freshness policy. If the configuration is found, it constructs and
+        returns a FreshnessPolicy object based on the provided parameters. Otherwise,
+        it returns None.
+
+        Parameters:
+            stream_definition (Mapping[str, Any]): A dictionary representing the stream definition,
+            which includes configuration details.
+
+        Returns:
+            Optional[FreshnessPolicy]: A FreshnessPolicy object if the configuration is found,
+            otherwise None.
+        """
+        return self._default_freshness_policy_fn(stream_definition)
+
+    def _default_freshness_policy_fn(
+        self, stream_definition: Mapping[str, Any]
+    ) -> Optional[FreshnessPolicy]:
+        """Retrieves the freshness policy for a given stream definition.
+
+        This method checks the provided stream definition for a specific configuration
+        indicating a freshness policy. If the configuration is found, it constructs and
+        returns a FreshnessPolicy object based on the provided parameters. Otherwise,
+        it returns None.
+
+        Parameters:
+            stream_definition (Mapping[str, Any]): A dictionary representing the stream definition,
+            which includes configuration details.
+
+        Returns:
+            Optional[FreshnessPolicy]: A FreshnessPolicy object if the configuration is found,
+            otherwise None.
+        """
+        config = stream_definition.get("config", {}) or {}
+        meta = config.get("meta", {})
+        freshness_policy_config = meta.get("dagster", {}).get("freshness_policy")
+        if freshness_policy_config:
+            return FreshnessPolicy(
+                maximum_lag_minutes=float(freshness_policy_config["maximum_lag_minutes"]),
+                cron_schedule=freshness_policy_config.get("cron_schedule"),
+                cron_schedule_timezone=freshness_policy_config.get("cron_schedule_timezone"),
+            )
+
+    @superseded(
+        additional_warn_text="Use `DagsterSlingTranslator.get_asset_spec(...).auto_materialize_policy` instead.",
+    )
+    @public
+    def get_auto_materialize_policy(
+        self, stream_definition: Mapping[str, Any]
+    ) -> Optional[AutoMaterializePolicy]:
+        """Defines the auto-materialize policy for a given stream definition.
+
+        This method checks the provided stream definition for a specific configuration
+        indicating an auto-materialize policy. If the configuration is found, it returns
+        an eager auto-materialize policy. Otherwise, it returns None.
+
+        Parameters:
+            stream_definition (Mapping[str, Any]): A dictionary representing the stream definition,
+            which includes configuration details.
+
+        Returns:
+            Optional[AutoMaterializePolicy]: An eager auto-materialize policy if the configuration
+            is found, otherwise None.
+        """
+        return self._default_auto_materialize_policy_fn(stream_definition)
+
+    def _default_auto_materialize_policy_fn(
+        self, stream_definition: Mapping[str, Any]
+    ) -> Optional[AutoMaterializePolicy]:
+        """Defines the auto-materialize policy for a given stream definition.
+
+        This method checks the provided stream definition for a specific configuration
+        indicating an auto-materialize policy. If the configuration is found, it returns
+        an eager auto-materialize policy. Otherwise, it returns None.
+
+        Parameters:
+            stream_definition (Mapping[str, Any]): A dictionary representing the stream definition,
+            which includes configuration details.
+
+        Returns:
+            Optional[AutoMaterializePolicy]: An eager auto-materialize policy if the configuration
+            is found, otherwise None.
+        """
+        config = stream_definition.get("config", {}) or {}
+        meta = config.get("meta", {})
+        auto_materialize_policy_config = "auto_materialize_policy" in meta.get("dagster", {})
+        if auto_materialize_policy_config:
+            return AutoMaterializePolicy.eager()

{dagster-sling-0.25.9 → dagster-sling-0.25.11}/dagster_sling/resources.py

@@ -198,7 +198,7 @@ class SlingResource(ConfigurableResource):
             streams = streams_with_default_dagster_meta(raw_streams, replication_config)
             selected_asset_keys = context.selected_asset_keys
             for stream in streams:
-                asset_key = dagster_sling_translator.get_asset_key(stream)
+                asset_key = dagster_sling_translator.get_asset_spec(stream).key
                 if asset_key in selected_asset_keys:
                     context_streams.update({stream["name"]: stream["config"]})
 
@@ -401,7 +401,7 @@ class SlingResource(ConfigurableResource):
             # TODO: In the future, it'd be nice to yield these materializations as they come in
             # rather than waiting until the end of the replication
             for stream in stream_definitions:
-                asset_key = dagster_sling_translator.get_asset_key(stream)
+                asset_key = dagster_sling_translator.get_asset_spec(stream).key
 
                 object_key = (stream.get("config") or {}).get("object")
                 destination_stream_name = object_key or stream["name"]

dagster-sling-0.25.11/dagster_sling/version.py

@@ -0,0 +1 @@
+__version__ = "0.25.11"

{dagster-sling-0.25.9 → dagster-sling-0.25.11/dagster_sling.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dagster-sling
-Version: 0.25.9
+Version: 0.25.11
 Summary: Package for performing ETL/ELT tasks with Sling in Dagster.
 Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-sling
 Author: Dagster Labs

{dagster-sling-0.25.9 → dagster-sling-0.25.11}/dagster_sling.egg-info/requires.txt

@@ -1,4 +1,4 @@
-dagster==1.9.9
+dagster==1.9.11
 sling>=1.1.5
 
 [test]

{dagster-sling-0.25.9 → dagster-sling-0.25.11}/setup.py

@@ -33,7 +33,7 @@ setup(
     packages=find_packages(exclude=["dagster_sling_tests*"]),
     include_package_data=True,
     python_requires=">=3.9,<3.13",
-    install_requires=["dagster==1.9.9", "sling>=1.1.5"],
+    install_requires=["dagster==1.9.11", "sling>=1.1.5"],
     zip_safe=False,
     extras_require={
         "test": [

dagster-sling-0.25.9/dagster_sling/dagster_sling_translator.py

@@ -1,297 +0,0 @@
-import re
-from collections.abc import Iterable, Mapping
-from dataclasses import dataclass
-from typing import Any, Optional
-
-from dagster import AssetKey, AutoMaterializePolicy, FreshnessPolicy, MetadataValue
-from dagster._annotations import public
-
-
-@dataclass
-class DagsterSlingTranslator:
-    target_prefix: str = "target"
-
-    @public
-    def sanitize_stream_name(self, stream_name: str) -> str:
-        """A function that takes a stream name from a Sling replication config and returns a
-        sanitized name for the stream.
-
-        By default, this removes any non-alphanumeric characters from the stream name and replaces
-        them with underscores, while removing any double quotes.
-
-        Args:
-            stream_name (str): The name of the stream.
-
-        Examples:
-            Using a custom stream name sanitizer:
-
-            .. code-block:: python
-
-                class CustomSlingTranslator(DagsterSlingTranslator):
-                    def sanitize_stream_name(self, stream_name: str) -> str:
-                        return stream_name.replace(".", "")
-        """
-        return re.sub(r"[^a-zA-Z0-9_.]", "_", stream_name.replace('"', "").lower())
-
-    @public
-    def get_asset_key(self, stream_definition: Mapping[str, Any]) -> AssetKey:
-        """A function that takes a stream definition from a Sling replication config and returns a
-        Dagster AssetKey.
-
-        The stream definition is a dictionary key/value pair where the key is the stream name and
-        the value is a dictionary representing the Sling Replication Stream Config.
-
-        For example:
-
-        .. code-block:: python
-
-            stream_definition = {"public.users":
-                {'sql': 'select all_user_id, name from public."all_Users"',
-                'object': 'public.all_users'}
-            }
-
-        By default, this returns the class's target_prefix paramater concatenated with the stream name.
-        A stream named "public.accounts" will create an AssetKey named "target_public_accounts".
-
-        Override this function to customize how to map a Sling stream to a Dagster AssetKey.
-
-        Alternatively, you can provide metadata in your Sling replication config to specify the
-        Dagster AssetKey for a stream as follows:
-
-        .. code-block:: yaml
-
-            public.users:
-               meta:
-                 dagster:
-                   asset_key: "mydb_users"
-
-        Args:
-            stream_definition (Mapping[str, Any]): A dictionary representing the stream definition
-
-        Returns:
-            AssetKey: The Dagster AssetKey for the replication stream.
-
-        Examples:
-            Using a custom mapping for streams:
-
-            .. code-block:: python
-
-                class CustomSlingTranslator(DagsterSlingTranslator):
-                    def get_asset_key_for_target(self, stream_definition) -> AssetKey:
-                        map = {"stream1": "asset1", "stream2": "asset2"}
-                        return AssetKey(map[stream_name])
-        """
-        config = stream_definition.get("config", {}) or {}
-        object_key = config.get("object")
-        meta = config.get("meta", {})
-        asset_key = meta.get("dagster", {}).get("asset_key")
-
-        if asset_key:
-            if self.sanitize_stream_name(asset_key) != asset_key:
-                raise ValueError(
-                    f"Asset key {asset_key} for stream {stream_definition['name']} is not "
-                    "sanitized. Please use only alphanumeric characters and underscores."
-                )
-            return AssetKey(asset_key.split("."))
-
-        # You can override the Sling Replication default object with an object key
-        stream_name = object_key or stream_definition["name"]
-        sanitized_components = self.sanitize_stream_name(stream_name).split(".")
-        return AssetKey([self.target_prefix] + sanitized_components)
-
-    @public
-    def get_deps_asset_key(self, stream_definition: Mapping[str, Any]) -> Iterable[AssetKey]:
-        """A function that takes a stream name from a Sling replication config and returns a
-        Dagster AssetKey for the dependencies of the replication stream.
-
-        By default, this returns the stream name. For example, a stream named "public.accounts"
-        will create an AssetKey named "target_public_accounts" and a dependency named "public_accounts".
-
-        Override this function to customize how to map a Sling stream to a Dagster depenency.
-        Alternatively, you can provide metadata in your Sling replication config to specify the
-        Dagster AssetKey for a stream as follows:
-
-        .. code-block:: yaml
-
-            public.users:
-                meta:
-                    dagster:
-                        deps: "sourcedb_users"
-
-        Args:
-            stream_name (str): The name of the stream.
-
-        Returns:
-            AssetKey: The Dagster AssetKey dependency for the replication stream.
-
-        Examples:
-            Using a custom mapping for streams:
-
-            .. code-block:: python
-
-                class CustomSlingTranslator(DagsterSlingTranslator):
-                    def get_deps_asset_key(self, stream_name: str) -> AssetKey:
-                        map = {"stream1": "asset1", "stream2": "asset2"}
-                        return AssetKey(map[stream_name])
-
-        """
-        config = stream_definition.get("config", {}) or {}
-        meta = config.get("meta", {})
-        deps = meta.get("dagster", {}).get("deps")
-        deps_out = []
-        if deps and isinstance(deps, str):
-            deps = [deps]
-        if deps:
-            assert isinstance(deps, list)
-            for asset_key in deps:
-                if self.sanitize_stream_name(asset_key) != asset_key:
-                    raise ValueError(
-                        f"Deps Asset key {asset_key} for stream {stream_definition['name']} is not "
-                        "sanitized. Please use only alphanumeric characters and underscores."
-                    )
-                deps_out.append(AssetKey(asset_key.split(".")))
-            return deps_out
-
-        stream_name = stream_definition["name"]
-        components = self.sanitize_stream_name(stream_name).split(".")
-        return [AssetKey(components)]
-
-    @public
-    def get_description(self, stream_definition: Mapping[str, Any]) -> Optional[str]:
-        """Retrieves the description for a given stream definition.
-
-        This method checks the provided stream definition for a description. It first looks
-        for an "sql" key in the configuration and returns its value if found. If not, it looks
-        for a description in the metadata under the "dagster" key.
-
-        Parameters:
-            stream_definition (Mapping[str, Any]): A dictionary representing the stream definition,
-            which includes configuration details.
-
-        Returns:
-            Optional[str]: The description of the stream if found, otherwise None.
-        """
-        config = stream_definition.get("config", {}) or {}
-        if "sql" in config:
-            return config["sql"]
-        meta = config.get("meta", {})
-        description = meta.get("dagster", {}).get("description")
-        return description
-
-    @public
-    def get_metadata(self, stream_definition: Mapping[str, Any]) -> Mapping[str, Any]:
-        """Retrieves the metadata for a given stream definition.
-
-        This method extracts the configuration from the provided stream definition and returns
-        it as a JSON metadata value.
-
-        Parameters:
-            stream_definition (Mapping[str, Any]): A dictionary representing the stream definition,
-            which includes configuration details.
-
-        Returns:
-            Mapping[str, Any]: A dictionary containing the stream configuration as JSON metadata.
-        """
-        return {"stream_config": MetadataValue.json(stream_definition.get("config", {}))}
-
-    @public
-    def get_tags(self, stream_definition: Mapping[str, Any]) -> Mapping[str, Any]:
-        """Retrieves the tags for a given stream definition.
-
-        This method returns an empty dictionary, indicating that no tags are associated with
-        the stream definition by default. This method can be overridden to provide custom tags.
-
-        Parameters:
-            stream_definition (Mapping[str, Any]): A dictionary representing the stream definition,
-            which includes configuration details.
-
-        Returns:
-            Mapping[str, Any]: An empty dictionary.
-        """
-        return {}
-
-    @public
-    def get_kinds(self, stream_definition: Mapping[str, Any]) -> set[str]:
-        """Retrieves the kinds for a given stream definition.
-
-        This method returns "sling" by default. This method can be overridden to provide custom kinds.
-
-        Parameters:
-            stream_definition (Mapping[str, Any]): A dictionary representing the stream definition,
-            which includes configuration details.
-
-        Returns:
-            Set[str]: A set containing kinds for the stream's assets.
-        """
-        return {"sling"}
-
-    @public
-    def get_group_name(self, stream_definition: Mapping[str, Any]) -> Optional[str]:
-        """Retrieves the group name for a given stream definition.
-
-        This method checks the provided stream definition for a group name in the metadata
-        under the "dagster" key.
-
-        Parameters:
-            stream_definition (Mapping[str, Any]): A dictionary representing the stream definition,
-            which includes configuration details.
-
-        Returns:
-            Optional[str]: The group name if found, otherwise None.
-        """
-        config = stream_definition.get("config", {}) or {}
-        meta = config.get("meta", {})
-        return meta.get("dagster", {}).get("group")
-
-    @public
-    def get_freshness_policy(
-        self, stream_definition: Mapping[str, Any]
-    ) -> Optional[FreshnessPolicy]:
-        """Retrieves the freshness policy for a given stream definition.
-
-        This method checks the provided stream definition for a specific configuration
-        indicating a freshness policy. If the configuration is found, it constructs and
-        returns a FreshnessPolicy object based on the provided parameters. Otherwise,
-        it returns None.
-
-        Parameters:
-            stream_definition (Mapping[str, Any]): A dictionary representing the stream definition,
-            which includes configuration details.
-
-        Returns:
-            Optional[FreshnessPolicy]: A FreshnessPolicy object if the configuration is found,
-            otherwise None.
-        """
-        config = stream_definition.get("config", {}) or {}
-        meta = config.get("meta", {})
-        freshness_policy_config = meta.get("dagster", {}).get("freshness_policy")
-        if freshness_policy_config:
-            return FreshnessPolicy(
-                maximum_lag_minutes=float(freshness_policy_config["maximum_lag_minutes"]),
-                cron_schedule=freshness_policy_config.get("cron_schedule"),
-                cron_schedule_timezone=freshness_policy_config.get("cron_schedule_timezone"),
-            )
-
-    @public
-    def get_auto_materialize_policy(
-        self, stream_definition: Mapping[str, Any]
-    ) -> Optional[AutoMaterializePolicy]:
-        """Defines the auto-materialize policy for a given stream definition.
-
-        This method checks the provided stream definition for a specific configuration
-        indicating an auto-materialize policy. If the configuration is found, it returns
-        an eager auto-materialize policy. Otherwise, it returns None.
-
-        Parameters:
-            stream_definition (Mapping[str, Any]): A dictionary representing the stream definition,
-            which includes configuration details.
-
-        Returns:
-            Optional[AutoMaterializePolicy]: An eager auto-materialize policy if the configuration
-            is found, otherwise None.
-        """
-        config = stream_definition.get("config", {}) or {}
-        meta = config.get("meta", {})
-        auto_materialize_policy_config = "auto_materialize_policy" in meta.get("dagster", {})
-        if auto_materialize_policy_config:
-            return AutoMaterializePolicy.eager()

dagster-sling-0.25.9/dagster_sling/version.py

@@ -1 +0,0 @@
-__version__ = "0.25.9"