dagster-fivetran 0.27.13__tar.gz → 0.27.14__tar.gz

{dagster_fivetran-0.27.13/dagster_fivetran.egg-info → dagster_fivetran-0.27.14}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dagster-fivetran
-Version: 0.27.13
+Version: 0.27.14
 Summary: Package for integrating Fivetran with Dagster.
 Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-fivetran
 Author: Dagster Labs
@@ -15,9 +15,9 @@ Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Operating System :: OS Independent
 Requires-Python: >=3.9,<3.14
 License-File: LICENSE
-Requires-Dist: dagster==1.11.13
+Requires-Dist: dagster==1.11.14
 Provides-Extra: managed
-Requires-Dist: dagster-managed-elements==0.27.13; extra == "managed"
+Requires-Dist: dagster-managed-elements==0.27.14; extra == "managed"
 Provides-Extra: test
 Requires-Dist: pytest-order; extra == "test"
 Dynamic: author

{dagster_fivetran-0.27.13 → dagster_fivetran-0.27.14}/dagster_fivetran/components/workspace_component/component.py

@@ -1,16 +1,21 @@
-from collections.abc import Sequence
+from collections.abc import Iterable, Sequence
 from functools import cached_property
 from typing import Annotated, Callable, Optional, Union
 
 import dagster as dg
 import pydantic
 from dagster._annotations import public
-from dagster._core.definitions.job_definition import default_job_io_manager
+from dagster._utils.names import clean_name
 from dagster.components.resolved.base import resolve_fields
-from dagster.components.utils.translation import TranslationFn, TranslationFnResolver
+from dagster.components.utils.translation import (
+    ComponentTranslator,
+    TranslationFn,
+    TranslationFnResolver,
+    create_component_translator_cls,
+)
 from dagster_shared import check
 
-from dagster_fivetran.asset_defs import build_fivetran_assets_definitions
+from dagster_fivetran.asset_decorator import fivetran_assets
 from dagster_fivetran.components.workspace_component.scaffolder import (
     FivetranAccountComponentScaffolder,
 )
@@ -19,20 +24,10 @@ from dagster_fivetran.translator import (
     DagsterFivetranTranslator,
     FivetranConnector,
     FivetranConnectorTableProps,
+    FivetranMetadataSet,
 )
 
 
-class ProxyDagsterFivetranTranslator(DagsterFivetranTranslator):
-    def __init__(self, fn: TranslationFn[FivetranConnectorTableProps]):
-        self.fn = fn
-
-    def get_asset_spec(self, props: FivetranConnectorTableProps) -> dg.AssetSpec:
-        base_asset_spec = super().get_asset_spec(props)
-        spec = self.fn(base_asset_spec, props)
-
-        return spec
-
-
 class FivetranWorkspaceModel(pydantic.BaseModel):
     account_id: str = pydantic.Field(..., description="The Fivetran account ID.")
     api_key: str = pydantic.Field(
@@ -102,7 +97,7 @@ class FivetranAccountComponent(dg.Component, dg.Model, dg.Resolvable):
             TranslationFnResolver(template_vars_for_translation_fn=lambda data: {"props": data}),
         ]
     ] = pydantic.Field(
-        None,
+        default=None,
         description="Function used to translate Fivetran connector table properties into Dagster asset specs.",
     )
 
@@ -112,23 +107,63 @@ class FivetranAccountComponent(dg.Component, dg.Model, dg.Resolvable):
 
     @cached_property
     def translator(self) -> DagsterFivetranTranslator:
-        if self.translation:
-            return ProxyDagsterFivetranTranslator(self.translation)
+        return FivetranComponentTranslator(self)
+
+    @cached_property
+    def _base_translator(self) -> DagsterFivetranTranslator:
         return DagsterFivetranTranslator()
 
+    def get_asset_spec(self, props: FivetranConnectorTableProps) -> dg.AssetSpec:
+        return self._base_translator.get_asset_spec(props)
+
+    def execute(
+        self, context: dg.AssetExecutionContext, fivetran: FivetranWorkspace
+    ) -> Iterable[Union[dg.AssetMaterialization, dg.MaterializeResult]]:
+        yield from fivetran.sync_and_poll(context=context)
+
     def build_defs(self, context: dg.ComponentLoadContext) -> dg.Definitions:
-        fivetran_assets = build_fivetran_assets_definitions(
-            workspace=self.workspace_resource,
+        connector_selector_fn = self.connector_selector or (lambda connector: bool(connector))
+        all_asset_specs = self.workspace_resource.load_asset_specs(
             dagster_fivetran_translator=self.translator,
-            connector_selector_fn=self.connector_selector,
+            connector_selector_fn=connector_selector_fn,
         )
-        assets_with_resource = [
-            fivetran_asset.with_resources(
-                {
-                    "fivetran": self.workspace_resource.get_resource_definition(),
-                    "io_manager": default_job_io_manager,
-                }
+
+        connectors = {
+            (
+                check.not_none(FivetranMetadataSet.extract(spec.metadata).connector_id),
+                check.not_none(FivetranMetadataSet.extract(spec.metadata).connector_name),
             )
-            for fivetran_asset in fivetran_assets
-        ]
-        return dg.Definitions(assets=assets_with_resource)
+            for spec in all_asset_specs
+        }
+
+        assets = []
+        for connector_id, connector_name in connectors:
+
+            @fivetran_assets(
+                connector_id=connector_id,
+                workspace=self.workspace_resource,
+                name=f"fivetran_{clean_name(connector_name)}",
+                dagster_fivetran_translator=self.translator,
+                connector_selector_fn=connector_selector_fn,
+            )
+            def _asset(context: dg.AssetExecutionContext):
+                yield from self.execute(context=context, fivetran=self.workspace_resource)
+
+            assets.append(_asset)
+
+        return dg.Definitions(assets=assets)
+
+
+class FivetranComponentTranslator(
+    create_component_translator_cls(FivetranAccountComponent, DagsterFivetranTranslator),
+    ComponentTranslator[FivetranAccountComponent],
+):
+    def __init__(self, component: "FivetranAccountComponent"):
+        self._component = component
+
+    def get_asset_spec(self, props: FivetranConnectorTableProps) -> dg.AssetSpec:
+        base_asset_spec = super().get_asset_spec(props)
+        if self.component.translation is None:
+            return base_asset_spec
+        else:
+            return self.component.translation(base_asset_spec, props)

dagster_fivetran-0.27.14/dagster_fivetran/version.py

@@ -0,0 +1 @@
+__version__ = "0.27.14"

{dagster_fivetran-0.27.13 → dagster_fivetran-0.27.14/dagster_fivetran.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dagster-fivetran
-Version: 0.27.13
+Version: 0.27.14
 Summary: Package for integrating Fivetran with Dagster.
 Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-fivetran
 Author: Dagster Labs
@@ -15,9 +15,9 @@ Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Operating System :: OS Independent
 Requires-Python: >=3.9,<3.14
 License-File: LICENSE
-Requires-Dist: dagster==1.11.13
+Requires-Dist: dagster==1.11.14
 Provides-Extra: managed
-Requires-Dist: dagster-managed-elements==0.27.13; extra == "managed"
+Requires-Dist: dagster-managed-elements==0.27.14; extra == "managed"
 Provides-Extra: test
 Requires-Dist: pytest-order; extra == "test"
 Dynamic: author

dagster_fivetran-0.27.14/dagster_fivetran.egg-info/requires.txt

@@ -0,0 +1,7 @@
+dagster==1.11.14
+
+[managed]
+dagster-managed-elements==0.27.14
+
+[test]
+pytest-order

{dagster_fivetran-0.27.13 → dagster_fivetran-0.27.14}/setup.py

@@ -34,7 +34,7 @@ setup(
     packages=find_packages(exclude=["dagster_fivetran_tests*"]),
     include_package_data=True,
     python_requires=">=3.9,<3.14",
-    install_requires=["dagster==1.11.13"],
+    install_requires=["dagster==1.11.14"],
     zip_safe=False,
     entry_points={
         "console_scripts": [
@@ -46,7 +46,7 @@ setup(
     },
     extras_require={
         "managed": [
-            "dagster-managed-elements==0.27.13",
+            "dagster-managed-elements==0.27.14",
         ],
         "test": [
             "pytest-order",

dagster_fivetran-0.27.13/dagster_fivetran/version.py

@@ -1 +0,0 @@
-__version__ = "0.27.13"

dagster_fivetran-0.27.13/dagster_fivetran.egg-info/requires.txt

@@ -1,7 +0,0 @@
-dagster==1.11.13
-
-[managed]
-dagster-managed-elements==0.27.13
-
-[test]
-pytest-order