dagster-fivetran 0.28.2__py3-none-any.whl → 0.28.3__py3-none-any.whl

dagster_fivetran/__init__.py

@@ -15,6 +15,7 @@ from dagster_fivetran.ops import (
 )
 from dagster_fivetran.resources import (
     FivetranResource as FivetranResource,
+    FivetranSyncConfig as FivetranSyncConfig,
     FivetranWorkspace as FivetranWorkspace,
     fivetran_resource as fivetran_resource,
     load_fivetran_asset_specs as load_fivetran_asset_specs,

dagster_fivetran/resources.py

@@ -13,6 +13,7 @@ import requests
 from dagster import (
     AssetExecutionContext,
     AssetMaterialization,
+    Config,
     Definitions,
     Failure,
     InitResourceContext,
@@ -928,6 +929,29 @@ class FivetranClient:
         return FivetranOutput(connector_details=final_details, schema_config=schema_config_details)
 
 
+class FivetranSyncConfig(Config):
+    """Configuration for controlling Fivetran sync behavior.
+
+    Attributes:
+        resync: If True, performs a historical resync. If False, performs a normal sync.
+        resync_parameters: Optional parameters to control which tables to resync.
+            If not provided with resync=True, all tables will be resynced.
+            Example: {"schema_name": ["table1", "table2"], "another_schema": ["table3"]}
+    """
+
+    resync: bool = Field(
+        default=False,
+        description="Whether to perform a historical resync instead of a normal sync",
+    )
+    resync_parameters: Optional[dict[str, Any]] = Field(
+        default=None,
+        description=(
+            "Optional parameters to control which tables to resync. "
+            "If not provided with resync=True, all tables will be resynced."
+        ),
+    )
+
+
 class FivetranWorkspace(ConfigurableResource):
     """This class represents a Fivetran workspace and provides utilities
     to interact with Fivetran APIs.
@@ -1185,7 +1209,9 @@ class FivetranWorkspace(ConfigurableResource):
 
     @public
     def sync_and_poll(
-        self, context: AssetExecutionContext
+        self,
+        context: AssetExecutionContext,
+        config: Optional[FivetranSyncConfig] = None,
     ) -> FivetranEventIterator[Union[AssetMaterialization, MaterializeResult]]:
         """Executes a sync and poll process to materialize Fivetran assets.
             This method can only be used in the context of an asset execution.
@@ -1193,14 +1219,74 @@ class FivetranWorkspace(ConfigurableResource):
         Args:
             context (AssetExecutionContext): The execution context
                 from within `@fivetran_assets`.
+            config (Optional[FivetranSyncConfig]): Optional configuration to control sync behavior.
+                If config.resync is True, performs a historical resync instead of a normal sync.
+                If config.resync_parameters is provided, only the specified tables will be resynced.
 
         Returns:
             Iterator[Union[AssetMaterialization, MaterializeResult]]: An iterator of MaterializeResult
                 or AssetMaterialization.
+
+        Examples:
+            Normal sync (without config):
+
+            .. code-block:: python
+
+                from dagster import AssetExecutionContext
+                from dagster_fivetran import FivetranWorkspace, fivetran_assets
+
+                @fivetran_assets(connector_id="my_connector", workspace=fivetran_workspace)
+                def my_fivetran_assets(context: AssetExecutionContext, fivetran: FivetranWorkspace):
+                    yield from fivetran.sync_and_poll(context=context)
+
+            Historical resync of specific tables (config passed at runtime):
+
+            .. code-block:: python
+
+                from dagster import AssetExecutionContext
+                from dagster_fivetran import FivetranWorkspace, FivetranSyncConfig, fivetran_assets
+
+                @fivetran_assets(connector_id="my_connector", workspace=fivetran_workspace)
+                def my_fivetran_assets(
+                    context: AssetExecutionContext,
+                    fivetran: FivetranWorkspace,
+                    config: FivetranSyncConfig,
+                ):
+                    # When materializing, pass config with:
+                    # resync=True
+                    # resync_parameters={"schema_name": ["table1", "table2"]}
+                    yield from fivetran.sync_and_poll(context=context, config=config)
+
+            Full historical resync (config passed at runtime):
+
+            .. code-block:: python
+
+                from dagster import AssetExecutionContext
+                from dagster_fivetran import FivetranWorkspace, FivetranSyncConfig, fivetran_assets
+
+                @fivetran_assets(connector_id="my_connector", workspace=fivetran_workspace)
+                def my_fivetran_assets(
+                    context: AssetExecutionContext,
+                    fivetran: FivetranWorkspace,
+                    config: FivetranSyncConfig,
+                ):
+                    # When materializing, pass config with resync=True to resync all tables
+                    yield from fivetran.sync_and_poll(context=context, config=config)
         """
-        return FivetranEventIterator(
-            events=self._sync_and_poll(context=context), fivetran_workspace=self, context=context
-        )
+        if config and config.resync:
+            return FivetranEventIterator(
+                events=self._resync_and_poll(
+                    context=context, resync_parameters=config.resync_parameters
+                ),
+                fivetran_workspace=self,
+                context=context,
+            )
+        else:
+            return FivetranEventIterator(
+                events=self._sync_and_poll(context=context),
+                fivetran_workspace=self,
+                context=context,
+            )
 
     def _sync_and_poll(self, context: AssetExecutionContext):
         assets_def = context.assets_def
@@ -1245,6 +1331,54 @@ class FivetranWorkspace(ConfigurableResource):
         if unmaterialized_asset_keys:
             context.log.warning(f"Assets were not materialized: {unmaterialized_asset_keys}")
 
+    def _resync_and_poll(
+        self,
+        context: AssetExecutionContext,
+        resync_parameters: Optional[Mapping[str, Sequence[str]]] = None,
+    ):
+        assets_def = context.assets_def
+        dagster_fivetran_translator = get_translator_from_fivetran_assets(assets_def)
+        connector_id = next(
+            check.not_none(FivetranMetadataSet.extract(spec.metadata).connector_id)
+            for spec in assets_def.specs
+        )
+
+        client = self.get_client()
+        fivetran_output = client.resync_and_poll(
+            connector_id=connector_id,
+            resync_parameters=resync_parameters,
+        )
+
+        # The FivetranOutput is None if the connector hasn't been synced
+        if not fivetran_output:
+            context.log.warning(
+                f"The connector with ID {connector_id} is currently paused and so it has not been resynced. "
+                f"Make sure that your connector is enabled before resyncing it with Dagster."
+            )
+            return
+
+        materialized_asset_keys = set()
+        for materialization in self._generate_materialization(
+            fivetran_output=fivetran_output, dagster_fivetran_translator=dagster_fivetran_translator
+        ):
+            # Scan through all tables actually created, if it was expected then emit a MaterializeResult.
+            # Otherwise, emit a runtime AssetMaterialization.
+            if materialization.asset_key in context.selected_asset_keys:
+                yield MaterializeResult(
+                    asset_key=materialization.asset_key, metadata=materialization.metadata
+                )
+                materialized_asset_keys.add(materialization.asset_key)
+            else:
+                context.log.warning(
+                    f"An unexpected asset was materialized: {materialization.asset_key}. "
+                    f"Yielding a materialization event."
+                )
+                yield materialization
+
+        unmaterialized_asset_keys = context.selected_asset_keys - materialized_asset_keys
+        if unmaterialized_asset_keys:
+            context.log.warning(f"Assets were not materialized: {unmaterialized_asset_keys}")
+
 
 def load_fivetran_asset_specs(
     workspace: FivetranWorkspace,

dagster_fivetran/version.py

@@ -1 +1 @@
-__version__ = "0.28.2"
+__version__ = "0.28.3"

{dagster_fivetran-0.28.2.dist-info → dagster_fivetran-0.28.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dagster-fivetran
-Version: 0.28.2
+Version: 0.28.3
 Summary: Package for integrating Fivetran with Dagster.
 Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-fivetran
 Author: Dagster Labs
@@ -14,9 +14,9 @@ Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Operating System :: OS Independent
 Requires-Python: >=3.10,<3.14
 License-File: LICENSE
-Requires-Dist: dagster==1.12.2
+Requires-Dist: dagster==1.12.3
 Provides-Extra: managed
-Requires-Dist: dagster-managed-elements==0.28.2; extra == "managed"
+Requires-Dist: dagster-managed-elements==0.28.3; extra == "managed"
 Provides-Extra: test
 Requires-Dist: pytest-order; extra == "test"
 Dynamic: author

{dagster_fivetran-0.28.2.dist-info → dagster_fivetran-0.28.3.dist-info}/RECORD

@@ -1,4 +1,4 @@
-dagster_fivetran/__init__.py,sha256=XgXtaGmeFA-506MDyxIlQn0uqo7iXyPWZ4RZ1qSUvtI,1558
+dagster_fivetran/__init__.py,sha256=9k5iCUhCx7DCYO-aj93oVhS6Ekc5zyaUBhZuofm77sg,1604
 dagster_fivetran/asset_decorator.py,sha256=nH6PlVnvP_xJgkfYzhwB7eHvzEXK7tqr3XqrY4qo2ng,5242
 dagster_fivetran/asset_defs.py,sha256=cxyPXdotaR7rDRN-6Go1rQ9ewehBW4zfBhZxxtcplMs,34531
 dagster_fivetran/cli.py,sha256=nG998z2QDE4s2rHTWsu1Hj4MH2cQW_H2x3ocgodHaJg,3228
@@ -6,11 +6,11 @@ dagster_fivetran/constants.py,sha256=hsdYi7WlDIfvrw4ATVha7K6GQHNnSelVieqNao1hzco
 dagster_fivetran/fivetran_event_iterator.py,sha256=DyZK5HQ7cFD4CWnhvPB7Hj5nUycjcoQ6IFrTX3SvgbQ,5652
 dagster_fivetran/ops.py,sha256=SNxRuShLI3MtqQLRAuIfRaUpKGqUmapCmEaop48_mIM,7694
 dagster_fivetran/py.typed,sha256=la67KBlbjXN-_-DfGNcdOcjYumVpKG_Tkw-8n5dnGB4,8
-dagster_fivetran/resources.py,sha256=1d6HCCBFz3ctfs0JoNsmz0O_49wdxARgeJ6_anI0lnU,55703
+dagster_fivetran/resources.py,sha256=0APIjz2ovJo5gpoVzxs2rSmcs73BbhTYt8DPI3iaiHU,61553
 dagster_fivetran/translator.py,sha256=KyjB6nUuvN27hnIC-Oh4U_rrdud6Vfn9GH5GGFLBU9M,12470
 dagster_fivetran/types.py,sha256=1IXrLu937TMFdwi0tJJFRcT8Td5N8tfJe3rHArvPi68,957
 dagster_fivetran/utils.py,sha256=-xwgtr-UP3Q25ZIQzir4qPkLhYurnkdJY--5cr1ANC8,5918
-dagster_fivetran/version.py,sha256=K-TM2fq9AmH_Dk8Cadam72wILDZ_6qftLHvY9P1Fc3I,23
+dagster_fivetran/version.py,sha256=PFj1PlFYtLhN7k4TLEHMneuQ4Ep8o7DCrl69TL8I0SY,23
 dagster_fivetran/components/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dagster_fivetran/components/workspace_component/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dagster_fivetran/components/workspace_component/component.py,sha256=tq68-TjMyVBoRWBJ0YRXofd153lzABpT2b96L_92Kso,10535
@@ -18,9 +18,9 @@ dagster_fivetran/components/workspace_component/scaffolder.py,sha256=2mv1vqN2KPn
 dagster_fivetran/managed/__init__.py,sha256=ki6yF6pMBQXXu6TwUom1BP0-PYW5RTYgvWmh3hRuT7w,269
 dagster_fivetran/managed/reconciliation.py,sha256=a0qLo52hqB5BCGBpLr5uLumkjgy72m0OmZYIyV5yCx8,14630
 dagster_fivetran/managed/types.py,sha256=PamX40i5JSAIrr97G4lEsHA6VdJLGBm99aVGNzgd-og,3622
-dagster_fivetran-0.28.2.dist-info/licenses/LICENSE,sha256=4lsMW-RCvfVD4_F57wrmpe3vX1xwUk_OAKKmV_XT7Z0,11348
-dagster_fivetran-0.28.2.dist-info/METADATA,sha256=Pq4QDwkV2eTlpwWXFR2stPLkaq4TW7WKlTXLe9Eptz8,1057
-dagster_fivetran-0.28.2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-dagster_fivetran-0.28.2.dist-info/entry_points.txt,sha256=2F7i9srZ1_8pwixGIodaNaWECGjqmBf8yvbYzzoULsc,134
-dagster_fivetran-0.28.2.dist-info/top_level.txt,sha256=8GH5pXbKSZEW4MVEAeOMN0e7cSH0h_4tjxp5WpZtkGA,17
-dagster_fivetran-0.28.2.dist-info/RECORD,,
+dagster_fivetran-0.28.3.dist-info/licenses/LICENSE,sha256=4lsMW-RCvfVD4_F57wrmpe3vX1xwUk_OAKKmV_XT7Z0,11348
+dagster_fivetran-0.28.3.dist-info/METADATA,sha256=xBmLBQ4-yzUMRcjsV5sNBnLZAM351EUmnjaF8OYAedM,1057
+dagster_fivetran-0.28.3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+dagster_fivetran-0.28.3.dist-info/entry_points.txt,sha256=2F7i9srZ1_8pwixGIodaNaWECGjqmBf8yvbYzzoULsc,134
+dagster_fivetran-0.28.3.dist-info/top_level.txt,sha256=8GH5pXbKSZEW4MVEAeOMN0e7cSH0h_4tjxp5WpZtkGA,17
+dagster_fivetran-0.28.3.dist-info/RECORD,,