dagster-airbyte 0.27.16__py3-none-any.whl → 0.28.0__py3-none-any.whl

dagster_airbyte/asset_defs.py

@@ -20,7 +20,7 @@ from dagster import (
     SourceAsset,
     _check as check,
 )
-from dagster._annotations import beta, deprecated_param
+from dagster._annotations import beta, hidden_param, only_allow_hidden_params_in_kwargs
 from dagster._core.definitions import AssetsDefinition, multi_asset
 from dagster._core.definitions.assets.definition.cacheable_assets_definition import (
     AssetsDefinitionCacheableData,
@@ -253,7 +253,14 @@ def _build_airbyte_assets_from_metadata(
     return _assets
 
 
-@deprecated_param(param="legacy_freshness_policy", breaking_version="1.12.0")
+@hidden_param(
+    param="legacy_freshness_policy",
+    breaking_version="1.13.0",
+)
+@hidden_param(
+    param="auto_materialize_policy",
+    breaking_version="1.10.0",
+)
 def build_airbyte_assets(
     connection_id: str,
     destination_tables: Sequence[str],
@@ -265,9 +272,8 @@ def build_airbyte_assets(
     deps: Optional[Iterable[Union[CoercibleToAssetKey, AssetsDefinition, SourceAsset]]] = None,
     upstream_assets: Optional[set[AssetKey]] = None,
     schema_by_table_name: Optional[Mapping[str, TableSchema]] = None,
-    legacy_freshness_policy: Optional[LegacyFreshnessPolicy] = None,
     stream_to_asset_map: Optional[Mapping[str, str]] = None,
-    auto_materialize_policy: Optional[AutoMaterializePolicy] = None,
+    **kwargs,
 ) -> Sequence[AssetsDefinition]:
     """Builds a set of assets representing the tables created by an Airbyte sync operation.
 
@@ -287,11 +293,13 @@ def build_airbyte_assets(
         deps (Optional[Sequence[Union[AssetsDefinition, SourceAsset, str, AssetKey]]]):
             A list of assets to add as sources.
         upstream_assets (Optional[Set[AssetKey]]): Deprecated, use deps instead. A list of assets to add as sources.
-        legacy_freshness_policy (Optional[LegacyFreshnessPolicy]): A legacy freshness policy to apply to the assets
         stream_to_asset_map (Optional[Mapping[str, str]]): A mapping of an Airbyte stream name to a Dagster asset.
             This allows the use of the "prefix" setting in Airbyte with special characters that aren't valid asset names.
-        auto_materialize_policy (Optional[AutoMaterializePolicy]): An auto materialization policy to apply to the assets.
     """
+    only_allow_hidden_params_in_kwargs(build_airbyte_assets, kwargs)
+    legacy_freshness_policy = kwargs.get("legacy_freshness_policy")
+    auto_materialize_policy = kwargs.get("auto_materialize_policy")
+
     if upstream_assets is not None and deps is not None:
         raise DagsterInvalidDefinitionError(
             "Cannot specify both deps and upstream_assets to build_airbyte_assets. Use only deps"

dagster_airbyte/components/workspace_component/component.py

@@ -7,6 +7,7 @@ from typing import Annotated, Callable, Optional, Union
 import dagster as dg
 import pydantic
 from dagster._annotations import superseded
+from dagster._symbol_annotations.public import public
 from dagster._utils.names import clean_name
 from dagster.components.component.state_backed_component import StateBackedComponent
 from dagster.components.resolved.base import resolve_fields
@@ -172,11 +173,31 @@ def resolve_airbyte_workspace_type(context: dg.ResolutionContext, model):
         check.failed(f"Unknown Airbyte workspace type: {type(model)}")
 
 
+@public
 @dg.scaffold_with(AirbyteWorkspaceComponentScaffolder)
 class AirbyteWorkspaceComponent(StateBackedComponent, dg.Model, dg.Resolvable):
     """Loads Airbyte connections from a given Airbyte workspace as Dagster assets.
     Materializing these assets will trigger a sync of the Airbyte connection, enabling
     you to schedule Airbyte syncs using Dagster.
+
+    Example:
+
+        .. code-block:: yaml
+
+            # defs.yaml
+
+            type: dagster_airbyte.AirbyteWorkspaceComponent
+            attributes:
+              workspace:
+                rest_api_base_url: http://localhost:8000/api/public/v1
+                configuration_api_base_url: http://localhost:8000/api/v1
+                workspace_id: your-workspace-id
+                client_id: "{{ env.AIRBYTE_CLIENT_ID }}"
+                client_secret: "{{ env.AIRBYTE_CLIENT_SECRET }}"
+              connection_selector:
+                by_name:
+                  - my_postgres_to_snowflake_connection
+                  - my_mysql_to_bigquery_connection
     """
 
     workspace: Annotated[
@@ -220,12 +241,72 @@ class AirbyteWorkspaceComponent(StateBackedComponent, dg.Model, dg.Resolvable):
     def _base_translator(self) -> DagsterAirbyteTranslator:
         return DagsterAirbyteTranslator()
 
+    @public
     def get_asset_spec(self, props: AirbyteConnectionTableProps) -> dg.AssetSpec:
+        """Generates an AssetSpec for a given Airbyte connection table.
+
+        This method can be overridden in a subclass to customize how Airbyte connection tables
+        are converted to Dagster asset specs. By default, it delegates to the configured
+        DagsterAirbyteTranslator.
+
+        Args:
+            props: The AirbyteConnectionTableProps containing information about the connection
+                and table/stream being synced
+
+        Returns:
+            An AssetSpec that represents the Airbyte connection table as a Dagster asset
+
+        Example:
+            Override this method to add custom metadata to all Airbyte assets:
+
+            .. code-block:: python
+
+                from dagster_airbyte import AirbyteWorkspaceComponent
+                import dagster as dg
+
+                class CustomAirbyteWorkspaceComponent(AirbyteWorkspaceComponent):
+                    def get_asset_spec(self, props):
+                        base_spec = super().get_asset_spec(props)
+                        return base_spec.replace_attributes(
+                            metadata={
+                                **base_spec.metadata,
+                                "data_source": "airbyte",
+                                "connection_id": props.connection_id
+                            }
+                        )
+        """
         return self._base_translator.get_asset_spec(props)
 
+    @public
     def execute(
         self, context: dg.AssetExecutionContext, airbyte: BaseAirbyteWorkspace
     ) -> Iterable[Union[dg.AssetMaterialization, dg.MaterializeResult]]:
+        """Executes an Airbyte sync for the selected connection.
+
+        This method can be overridden in a subclass to customize the sync execution behavior,
+        such as adding custom logging or handling sync results differently.
+
+        Args:
+            context: The asset execution context provided by Dagster
+            airbyte: The BaseAirbyteWorkspace resource used to trigger and monitor syncs
+
+        Yields:
+            AssetMaterialization or MaterializeResult events from the Airbyte sync
+
+        Example:
+            Override this method to add custom logging during sync execution:
+
+            .. code-block:: python
+
+                from dagster_airbyte import AirbyteWorkspaceComponent
+                import dagster as dg
+
+                class CustomAirbyteWorkspaceComponent(AirbyteWorkspaceComponent):
+                    def execute(self, context, airbyte):
+                        context.log.info(f"Starting Airbyte sync for connection")
+                        yield from super().execute(context, airbyte)
+                        context.log.info("Airbyte sync completed successfully")
+        """
         yield from airbyte.sync_and_poll(context=context)
 
     def _load_asset_specs(self, state: AirbyteWorkspaceData) -> Sequence[dg.AssetSpec]:

dagster_airbyte/version.py

@@ -1 +1 @@
-__version__ = "0.27.16"
+__version__ = "0.28.0"

{dagster_airbyte-0.27.16.dist-info → dagster_airbyte-0.28.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dagster-airbyte
-Version: 0.27.16
+Version: 0.28.0
 Summary: Package for integrating Airbyte with Dagster.
 Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-airbyte
 Author: Dagster Labs
@@ -15,13 +15,13 @@ Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Operating System :: OS Independent
 Requires-Python: >=3.9,<3.14
 License-File: LICENSE
-Requires-Dist: dagster==1.11.16
+Requires-Dist: dagster==1.12.0
 Requires-Dist: requests
 Provides-Extra: test
 Requires-Dist: requests-mock; extra == "test"
 Requires-Dist: flaky; extra == "test"
 Provides-Extra: managed
-Requires-Dist: dagster-managed-elements==0.27.16; extra == "managed"
+Requires-Dist: dagster-managed-elements==0.28.0; extra == "managed"
 Dynamic: author
 Dynamic: author-email
 Dynamic: classifier

{dagster_airbyte-0.27.16.dist-info → dagster_airbyte-0.28.0.dist-info}/RECORD

@@ -1,6 +1,6 @@
 dagster_airbyte/__init__.py,sha256=Bc3BuYTIcz5XU6NbKHqIemxgFSydLsqVLyLcx6K2pVA,2024
 dagster_airbyte/asset_decorator.py,sha256=ziG1U5qleIIR8xbYmoKSA00gLazbrd09FzcgmGdZmkY,4640
-dagster_airbyte/asset_defs.py,sha256=0lGOKSAlTxOYFonDsEI4FzuiVAViV8pr2Zgx3R8fAvQ,51694
+dagster_airbyte/asset_defs.py,sha256=uHMW_m54kqoEc0w4CvpDQqL5bmJfmmyzv_KXEeM9x_g,51662
 dagster_airbyte/cli.py,sha256=HErteP1MjfHozKKSrznh0yAreKETbXp5NDHzXGsdvvE,425
 dagster_airbyte/ops.py,sha256=oOEczVYpqVRTc1-0osfpR5FZbPjNJDYihgRjk_qtOQA,4229
 dagster_airbyte/py.typed,sha256=la67KBlbjXN-_-DfGNcdOcjYumVpKG_Tkw-8n5dnGB4,8
@@ -8,10 +8,10 @@ dagster_airbyte/resources.py,sha256=AXN_mtNl_JKauoOY29wyuiv9jjAiCNF9M4baVCQPMYo,
 dagster_airbyte/translator.py,sha256=RY2LhGPACIfyd2zOSH1swdLz4koX6_bvclW-alS3Bic,7547
 dagster_airbyte/types.py,sha256=TYUjI3skjLYeANjesgJ-IAJNu8bAnL1ymsUfz5LsRTE,1565
 dagster_airbyte/utils.py,sha256=wG9119kXi87JgcOjK7iNozr-svZocJBQYoHBMmnXZcE,4092
-dagster_airbyte/version.py,sha256=gNIskoVbgqFeY6KiyOzkhfP7GLwnNVmVeil7tBvkL44,24
+dagster_airbyte/version.py,sha256=MRQGtOXBhcDKeeNOL0LiB-cllo6kfd8_KGJOvaDp0XQ,23
 dagster_airbyte/components/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dagster_airbyte/components/workspace_component/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-dagster_airbyte/components/workspace_component/component.py,sha256=nGOQXivXS-Zc69_K1aq2xATT06Roci3fAJ-5x1xOqgU,11238
+dagster_airbyte/components/workspace_component/component.py,sha256=Ao7dXsDuEUXTpBENJy83cnQfF-fceZGBMKxoxFoC5DE,14463
 dagster_airbyte/components/workspace_component/scaffolder.py,sha256=zk5HDJ6C8zcfUWKslJVyqyBB0LV3JzNAocY2ptT517s,1046
 dagster_airbyte/managed/__init__.py,sha256=6SBtyNOMJ9Cu2UIwFExJHpL_ZVFo3rPMvyIxVOsKvWE,469
 dagster_airbyte/managed/reconciliation.py,sha256=xoVfqPBpNSldkiqOLIPnc7ei8CppHKWtzv8bvxjdqlI,34859
@@ -19,9 +19,9 @@ dagster_airbyte/managed/types.py,sha256=isPfX8L9YwtZAf9Vk4hhxBePLR00AEldsdK2TsM1
 dagster_airbyte/managed/generated/__init__.py,sha256=eYq-yfXEeffuKAVFXY8plD0se1wHjFNVqklpbu9gljw,108
 dagster_airbyte/managed/generated/destinations.py,sha256=x1wmWlXvOJHtfaZva3ErdKuVS--sDvfidSXR5ji9G5w,119692
 dagster_airbyte/managed/generated/sources.py,sha256=y0TPNvcRd8c9mhje-NoXsHeKRPt1nXcpww8mNAtqCps,282685
-dagster_airbyte-0.27.16.dist-info/licenses/LICENSE,sha256=4lsMW-RCvfVD4_F57wrmpe3vX1xwUk_OAKKmV_XT7Z0,11348
-dagster_airbyte-0.27.16.dist-info/METADATA,sha256=Ia6R-HAZnzscGRmv7ROOH2YGTzOGinrTaVIrbfxs_nQ,1169
-dagster_airbyte-0.27.16.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-dagster_airbyte-0.27.16.dist-info/entry_points.txt,sha256=096yvfMP-gNsCgDg9vDQtinis5QGpD-e_kHEhcHaML8,120
-dagster_airbyte-0.27.16.dist-info/top_level.txt,sha256=HLwIRQCzqItn88_KbPP8DNTKKQEBUVKk6NCn4PrCtqY,16
-dagster_airbyte-0.27.16.dist-info/RECORD,,
+dagster_airbyte-0.28.0.dist-info/licenses/LICENSE,sha256=4lsMW-RCvfVD4_F57wrmpe3vX1xwUk_OAKKmV_XT7Z0,11348
+dagster_airbyte-0.28.0.dist-info/METADATA,sha256=za8j6v-hzsbyohRvqCr94JdA4S8PCvY7Ojs_b9vm6W4,1166
+dagster_airbyte-0.28.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+dagster_airbyte-0.28.0.dist-info/entry_points.txt,sha256=096yvfMP-gNsCgDg9vDQtinis5QGpD-e_kHEhcHaML8,120
+dagster_airbyte-0.28.0.dist-info/top_level.txt,sha256=HLwIRQCzqItn88_KbPP8DNTKKQEBUVKk6NCn4PrCtqY,16
+dagster_airbyte-0.28.0.dist-info/RECORD,,