dagster-fivetran 0.28.1__tar.gz → 0.28.3__tar.gz

{dagster_fivetran-0.28.1/dagster_fivetran.egg-info → dagster_fivetran-0.28.3}/PKG-INFO

@@ -1,23 +1,22 @@
 Metadata-Version: 2.4
 Name: dagster-fivetran
-Version: 0.28.1
+Version: 0.28.3
 Summary: Package for integrating Fivetran with Dagster.
 Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-fivetran
 Author: Dagster Labs
 Author-email: hello@dagsterlabs.com
 License: Apache-2.0
-Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Operating System :: OS Independent
-Requires-Python: >=3.9,<3.14
+Requires-Python: >=3.10,<3.14
 License-File: LICENSE
-Requires-Dist: dagster==1.12.1
+Requires-Dist: dagster==1.12.3
 Provides-Extra: managed
-Requires-Dist: dagster-managed-elements==0.28.1; extra == "managed"
+Requires-Dist: dagster-managed-elements==0.28.3; extra == "managed"
 Provides-Extra: test
 Requires-Dist: pytest-order; extra == "test"
 Dynamic: author

{dagster_fivetran-0.28.1 → dagster_fivetran-0.28.3}/dagster_fivetran/__init__.py

@@ -15,6 +15,7 @@ from dagster_fivetran.ops import (
 )
 from dagster_fivetran.resources import (
     FivetranResource as FivetranResource,
+    FivetranSyncConfig as FivetranSyncConfig,
     FivetranWorkspace as FivetranWorkspace,
     fivetran_resource as fivetran_resource,
     load_fivetran_asset_specs as load_fivetran_asset_specs,

{dagster_fivetran-0.28.1 → dagster_fivetran-0.28.3}/dagster_fivetran/asset_decorator.py

@@ -1,4 +1,5 @@
-from typing import Any, Callable, Optional
+from collections.abc import Callable
+from typing import Any, Optional
 
 from dagster import AssetsDefinition, multi_asset
 from dagster._core.errors import DagsterInvariantViolationError

{dagster_fivetran-0.28.1 → dagster_fivetran-0.28.3}/dagster_fivetran/asset_defs.py

@@ -1,9 +1,9 @@
 import hashlib
 import inspect
-from collections.abc import Mapping, Sequence
+from collections.abc import Callable, Mapping, Sequence
 from concurrent.futures import ThreadPoolExecutor
 from functools import partial
-from typing import Any, Callable, NamedTuple, Optional, Union, cast
+from typing import Any, NamedTuple, Optional, Union, cast
 
 from dagster import (
     AssetExecutionContext,

{dagster_fivetran-0.28.1 → dagster_fivetran-0.28.3}/dagster_fivetran/components/workspace_component/component.py

@@ -1,8 +1,8 @@
 from collections import defaultdict
-from collections.abc import Iterable, Sequence
+from collections.abc import Callable, Iterable, Sequence
 from functools import cached_property
 from pathlib import Path
-from typing import Annotated, Callable, Optional, Union
+from typing import Annotated, Optional, Union
 
 import dagster as dg
 import pydantic

{dagster_fivetran-0.28.1 → dagster_fivetran-0.28.3}/dagster_fivetran/fivetran_event_iterator.py

@@ -1,7 +1,7 @@
 import os
-from collections.abc import Iterator
+from collections.abc import Callable, Iterator
 from concurrent.futures import ThreadPoolExecutor
-from typing import TYPE_CHECKING, Any, Callable, Union
+from typing import TYPE_CHECKING, Any, Union
 
 from dagster import (
     AssetExecutionContext,

{dagster_fivetran-0.28.1 → dagster_fivetran-0.28.3}/dagster_fivetran/resources.py

@@ -2,17 +2,18 @@ import json
 import logging
 import os
 import time
-from collections.abc import Mapping, Sequence
+from collections.abc import Callable, Mapping, Sequence
 from datetime import datetime, timedelta
 from functools import cached_property, partial
 from pathlib import Path
-from typing import Any, Callable, Optional, Union
+from typing import Any, Optional, Union
 from urllib.parse import urljoin
 
 import requests
 from dagster import (
     AssetExecutionContext,
     AssetMaterialization,
+    Config,
     Definitions,
     Failure,
     InitResourceContext,
@@ -928,6 +929,29 @@ class FivetranClient:
         return FivetranOutput(connector_details=final_details, schema_config=schema_config_details)
 
 
+class FivetranSyncConfig(Config):
+    """Configuration for controlling Fivetran sync behavior.
+
+    Attributes:
+        resync: If True, performs a historical resync. If False, performs a normal sync.
+        resync_parameters: Optional parameters to control which tables to resync.
+            If not provided with resync=True, all tables will be resynced.
+            Example: {"schema_name": ["table1", "table2"], "another_schema": ["table3"]}
+    """
+
+    resync: bool = Field(
+        default=False,
+        description="Whether to perform a historical resync instead of a normal sync",
+    )
+    resync_parameters: Optional[dict[str, Any]] = Field(
+        default=None,
+        description=(
+            "Optional parameters to control which tables to resync. "
+            "If not provided with resync=True, all tables will be resynced."
+        ),
+    )
+
+
 class FivetranWorkspace(ConfigurableResource):
     """This class represents a Fivetran workspace and provides utilities
     to interact with Fivetran APIs.
@@ -1185,7 +1209,9 @@ class FivetranWorkspace(ConfigurableResource):
 
     @public
     def sync_and_poll(
-        self, context: AssetExecutionContext
+        self,
+        context: AssetExecutionContext,
+        config: Optional[FivetranSyncConfig] = None,
     ) -> FivetranEventIterator[Union[AssetMaterialization, MaterializeResult]]:
         """Executes a sync and poll process to materialize Fivetran assets.
             This method can only be used in the context of an asset execution.
@@ -1193,14 +1219,74 @@ class FivetranWorkspace(ConfigurableResource):
         Args:
             context (AssetExecutionContext): The execution context
                 from within `@fivetran_assets`.
+            config (Optional[FivetranSyncConfig]): Optional configuration to control sync behavior.
+                If config.resync is True, performs a historical resync instead of a normal sync.
+                If config.resync_parameters is provided, only the specified tables will be resynced.
 
         Returns:
             Iterator[Union[AssetMaterialization, MaterializeResult]]: An iterator of MaterializeResult
                 or AssetMaterialization.
+
+        Examples:
+            Normal sync (without config):
+
+            .. code-block:: python
+
+                from dagster import AssetExecutionContext
+                from dagster_fivetran import FivetranWorkspace, fivetran_assets
+
+                @fivetran_assets(connector_id="my_connector", workspace=fivetran_workspace)
+                def my_fivetran_assets(context: AssetExecutionContext, fivetran: FivetranWorkspace):
+                    yield from fivetran.sync_and_poll(context=context)
+
+            Historical resync of specific tables (config passed at runtime):
+
+            .. code-block:: python
+
+                from dagster import AssetExecutionContext
+                from dagster_fivetran import FivetranWorkspace, FivetranSyncConfig, fivetran_assets
+
+                @fivetran_assets(connector_id="my_connector", workspace=fivetran_workspace)
+                def my_fivetran_assets(
+                    context: AssetExecutionContext,
+                    fivetran: FivetranWorkspace,
+                    config: FivetranSyncConfig,
+                ):
+                    # When materializing, pass config with:
+                    # resync=True
+                    # resync_parameters={"schema_name": ["table1", "table2"]}
+                    yield from fivetran.sync_and_poll(context=context, config=config)
+
+            Full historical resync (config passed at runtime):
+
+            .. code-block:: python
+
+                from dagster import AssetExecutionContext
+                from dagster_fivetran import FivetranWorkspace, FivetranSyncConfig, fivetran_assets
+
+                @fivetran_assets(connector_id="my_connector", workspace=fivetran_workspace)
+                def my_fivetran_assets(
+                    context: AssetExecutionContext,
+                    fivetran: FivetranWorkspace,
+                    config: FivetranSyncConfig,
+                ):
+                    # When materializing, pass config with resync=True to resync all tables
+                    yield from fivetran.sync_and_poll(context=context, config=config)
         """
-        return FivetranEventIterator(
-            events=self._sync_and_poll(context=context), fivetran_workspace=self, context=context
-        )
+        if config and config.resync:
+            return FivetranEventIterator(
+                events=self._resync_and_poll(
+                    context=context, resync_parameters=config.resync_parameters
+                ),
+                fivetran_workspace=self,
+                context=context,
+            )
+        else:
+            return FivetranEventIterator(
+                events=self._sync_and_poll(context=context),
+                fivetran_workspace=self,
+                context=context,
+            )
 
     def _sync_and_poll(self, context: AssetExecutionContext):
         assets_def = context.assets_def
@@ -1245,6 +1331,54 @@ class FivetranWorkspace(ConfigurableResource):
         if unmaterialized_asset_keys:
             context.log.warning(f"Assets were not materialized: {unmaterialized_asset_keys}")
 
+    def _resync_and_poll(
+        self,
+        context: AssetExecutionContext,
+        resync_parameters: Optional[Mapping[str, Sequence[str]]] = None,
+    ):
+        assets_def = context.assets_def
+        dagster_fivetran_translator = get_translator_from_fivetran_assets(assets_def)
+        connector_id = next(
+            check.not_none(FivetranMetadataSet.extract(spec.metadata).connector_id)
+            for spec in assets_def.specs
+        )
+
+        client = self.get_client()
+        fivetran_output = client.resync_and_poll(
+            connector_id=connector_id,
+            resync_parameters=resync_parameters,
+        )
+
+        # The FivetranOutput is None if the connector hasn't been synced
+        if not fivetran_output:
+            context.log.warning(
+                f"The connector with ID {connector_id} is currently paused and so it has not been resynced. "
+                f"Make sure that your connector is enabled before resyncing it with Dagster."
+            )
+            return
+
+        materialized_asset_keys = set()
+        for materialization in self._generate_materialization(
+            fivetran_output=fivetran_output, dagster_fivetran_translator=dagster_fivetran_translator
+        ):
+            # Scan through all tables actually created, if it was expected then emit a MaterializeResult.
+            # Otherwise, emit a runtime AssetMaterialization.
+            if materialization.asset_key in context.selected_asset_keys:
+                yield MaterializeResult(
+                    asset_key=materialization.asset_key, metadata=materialization.metadata
+                )
+                materialized_asset_keys.add(materialization.asset_key)
+            else:
+                context.log.warning(
+                    f"An unexpected asset was materialized: {materialization.asset_key}. "
+                    f"Yielding a materialization event."
+                )
+                yield materialization
+
+        unmaterialized_asset_keys = context.selected_asset_keys - materialized_asset_keys
+        if unmaterialized_asset_keys:
+            context.log.warning(f"Assets were not materialized: {unmaterialized_asset_keys}")
+
 
 def load_fivetran_asset_specs(
     workspace: FivetranWorkspace,

{dagster_fivetran-0.28.1 → dagster_fivetran-0.28.3}/dagster_fivetran/translator.py

@@ -1,7 +1,7 @@
-from collections.abc import Mapping, Sequence
+from collections.abc import Callable, Mapping, Sequence
 from datetime import datetime
 from enum import Enum
-from typing import Any, Callable, NamedTuple, Optional
+from typing import Any, NamedTuple, Optional, TypeAlias
 
 from dagster import Failure
 from dagster._core.definitions.asset_key import AssetKey
@@ -12,7 +12,6 @@ from dagster._utils.cached_method import cached_method
 from dagster._utils.names import clean_name_lower
 from dagster._vendored.dateutil import parser
 from dagster_shared.serdes import whitelist_for_serdes
-from typing_extensions import TypeAlias
 
 from dagster_fivetran.utils import get_fivetran_connector_table_name, metadata_for_table
 

dagster_fivetran-0.28.3/dagster_fivetran/version.py

@@ -0,0 +1 @@
+__version__ = "0.28.3"

{dagster_fivetran-0.28.1 → dagster_fivetran-0.28.3/dagster_fivetran.egg-info}/PKG-INFO

@@ -1,23 +1,22 @@
 Metadata-Version: 2.4
 Name: dagster-fivetran
-Version: 0.28.1
+Version: 0.28.3
 Summary: Package for integrating Fivetran with Dagster.
 Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-fivetran
 Author: Dagster Labs
 Author-email: hello@dagsterlabs.com
 License: Apache-2.0
-Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Operating System :: OS Independent
-Requires-Python: >=3.9,<3.14
+Requires-Python: >=3.10,<3.14
 License-File: LICENSE
-Requires-Dist: dagster==1.12.1
+Requires-Dist: dagster==1.12.3
 Provides-Extra: managed
-Requires-Dist: dagster-managed-elements==0.28.1; extra == "managed"
+Requires-Dist: dagster-managed-elements==0.28.3; extra == "managed"
 Provides-Extra: test
 Requires-Dist: pytest-order; extra == "test"
 Dynamic: author

dagster_fivetran-0.28.3/dagster_fivetran.egg-info/requires.txt

@@ -0,0 +1,7 @@
+dagster==1.12.3
+
+[managed]
+dagster-managed-elements==0.28.3
+
+[test]
+pytest-order

{dagster_fivetran-0.28.1 → dagster_fivetran-0.28.3}/setup.py

@@ -23,7 +23,6 @@ setup(
     description="Package for integrating Fivetran with Dagster.",
     url="https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-fivetran",
     classifiers=[
-        "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.10",
         "Programming Language :: Python :: 3.11",
         "Programming Language :: Python :: 3.12",
@@ -33,8 +32,8 @@ setup(
     ],
     packages=find_packages(exclude=["dagster_fivetran_tests*"]),
     include_package_data=True,
-    python_requires=">=3.9,<3.14",
-    install_requires=["dagster==1.12.1"],
+    python_requires=">=3.10,<3.14",
+    install_requires=["dagster==1.12.3"],
     zip_safe=False,
     entry_points={
         "console_scripts": [
@@ -46,7 +45,7 @@ setup(
     },
     extras_require={
         "managed": [
-            "dagster-managed-elements==0.28.1",
+            "dagster-managed-elements==0.28.3",
         ],
         "test": [
             "pytest-order",

dagster_fivetran-0.28.1/dagster_fivetran/version.py

@@ -1 +0,0 @@
-__version__ = "0.28.1"

dagster_fivetran-0.28.1/dagster_fivetran.egg-info/requires.txt

@@ -1,7 +0,0 @@
-dagster==1.12.1
-
-[managed]
-dagster-managed-elements==0.28.1
-
-[test]
-pytest-order