dagster-airbyte 0.26.18rc2__py3-none-any.whl → 0.26.19__py3-none-any.whl

dagster_airbyte/asset_defs.py

@@ -40,7 +40,11 @@ from dagster_airbyte.resources import (
     AirbyteResource,
     BaseAirbyteResource,
 )
-from dagster_airbyte.translator import AirbyteMetadataSet, DagsterAirbyteTranslator
+from dagster_airbyte.translator import (
+    AirbyteConnection,
+    AirbyteMetadataSet,
+    DagsterAirbyteTranslator,
+)
 from dagster_airbyte.types import AirbyteTableMetadata
 from dagster_airbyte.utils import (
     clean_name,
@@ -1035,6 +1039,7 @@ def build_airbyte_assets_definitions(
     *,
     workspace: AirbyteCloudWorkspace,
     dagster_airbyte_translator: Optional[DagsterAirbyteTranslator] = None,
+    connection_selector_fn: Optional[Callable[[AirbyteConnection], bool]] = None,
 ) -> Sequence[AssetsDefinition]:
     """The list of AssetsDefinition for all connections in the Airbyte workspace.
 
@@ -1043,6 +1048,8 @@ def build_airbyte_assets_definitions(
         dagster_airbyte_translator (Optional[DagsterAirbyteTranslator], optional): The translator to use
             to convert Airbyte content into :py:class:`dagster.AssetSpec`.
             Defaults to :py:class:`DagsterAirbyteTranslator`.
+        connection_selector_fn (Optional[Callable[[AirbyteConnection], bool]]): A function that allows for filtering
+            which Airbyte connection assets are created for.
 
     Returns:
         List[AssetsDefinition]: The list of AssetsDefinition for all connections in the Airbyte workspace.
@@ -1062,7 +1069,6 @@ def build_airbyte_assets_definitions(
                 client_secret=dg.EnvVar("AIRBYTE_CLOUD_CLIENT_SECRET"),
             )
 
-
             airbyte_assets = build_airbyte_assets_definitions(workspace=workspace)
 
             defs = dg.Definitions(
@@ -1096,21 +1102,46 @@ def build_airbyte_assets_definitions(
                 client_secret=dg.EnvVar("AIRBYTE_CLOUD_CLIENT_SECRET"),
             )
 
-
             airbyte_assets = build_airbyte_assets_definitions(
                 workspace=workspace,
                 dagster_airbyte_translator=CustomDagsterAirbyteTranslator()
             )
 
+            defs = dg.Definitions(
+                assets=airbyte_assets,
+                resources={"airbyte": airbyte_workspace},
+            )
+
+        Filter connections by name:
+
+        .. code-block:: python
+
+            from dagster_airbyte import AirbyteCloudWorkspace, build_airbyte_assets_definitions
+
+            import dagster as dg
+
+            airbyte_workspace = AirbyteCloudWorkspace(
+                workspace_id=dg.EnvVar("AIRBYTE_CLOUD_WORKSPACE_ID"),
+                client_id=dg.EnvVar("AIRBYTE_CLOUD_CLIENT_ID"),
+                client_secret=dg.EnvVar("AIRBYTE_CLOUD_CLIENT_SECRET"),
+            )
+
+            airbyte_assets = build_airbyte_assets_definitions(
+                workspace=workspace,
+                connection_selector_fn=lambda connection: connection.name in ["connection1", "connection2"]
+            )
+
             defs = dg.Definitions(
                 assets=airbyte_assets,
                 resources={"airbyte": airbyte_workspace},
             )
     """
     dagster_airbyte_translator = dagster_airbyte_translator or DagsterAirbyteTranslator()
+    connection_selector_fn = connection_selector_fn or (lambda connection: True)
 
     all_asset_specs = workspace.load_asset_specs(
-        dagster_airbyte_translator=dagster_airbyte_translator
+        dagster_airbyte_translator=dagster_airbyte_translator,
+        connection_selector_fn=connection_selector_fn,
     )
 
     connections = {

dagster_airbyte/resources.py

@@ -4,10 +4,10 @@ import logging
 import sys
 import time
 from abc import abstractmethod
-from collections.abc import Mapping, Sequence
+from collections.abc import Iterator, Mapping, Sequence
 from contextlib import contextmanager
 from datetime import datetime, timedelta
-from typing import Any, Optional, cast
+from typing import Any, Callable, Optional, cast
 
 import requests
 from dagster import (
@@ -1142,7 +1142,6 @@ class AirbyteCloudWorkspace(ConfigurableResource):
         default=15,
         description="Time (in seconds) after which the requests to Airbyte are declared timed out.",
     )
-
     _client: AirbyteCloudClient = PrivateAttr(default=None)  # type: ignore
 
     @cached_method
@@ -1197,6 +1196,7 @@ class AirbyteCloudWorkspace(ConfigurableResource):
     def load_asset_specs(
         self,
         dagster_airbyte_translator: Optional[DagsterAirbyteTranslator] = None,
+        connection_selector_fn: Optional[Callable[[AirbyteConnection], bool]] = None,
     ) -> Sequence[AssetSpec]:
         """Returns a list of AssetSpecs representing the Airbyte content in the workspace.
 
@@ -1204,6 +1204,8 @@ class AirbyteCloudWorkspace(ConfigurableResource):
             dagster_airbyte_translator (Optional[DagsterAirbyteTranslator], optional): The translator to use
                 to convert Airbyte content into :py:class:`dagster.AssetSpec`.
                 Defaults to :py:class:`DagsterAirbyteTranslator`.
+            connection_selector_fn (Optional[Callable[[AirbyteConnection], bool]]): A function that allows for filtering
+                which Airbyte connection assets are created for.
 
         Returns:
             List[AssetSpec]: The set of assets representing the Airbyte content in the workspace.
@@ -1228,7 +1230,9 @@ class AirbyteCloudWorkspace(ConfigurableResource):
         dagster_airbyte_translator = dagster_airbyte_translator or DagsterAirbyteTranslator()
 
         return load_airbyte_cloud_asset_specs(
-            workspace=self, dagster_airbyte_translator=dagster_airbyte_translator
+            workspace=self,
+            dagster_airbyte_translator=dagster_airbyte_translator,
+            connection_selector_fn=connection_selector_fn,
         )
 
     def _generate_materialization(
@@ -1317,11 +1321,27 @@ class AirbyteCloudWorkspace(ConfigurableResource):
         if unmaterialized_asset_keys:
             context.log.warning(f"Assets were not materialized: {unmaterialized_asset_keys}")
 
+    @contextmanager
+    def process_config_and_initialize_cm_cached(self) -> Iterator["AirbyteCloudWorkspace"]:
+        # Hack to avoid reconstructing initialized copies of this resource, which invalidates
+        # @cached_method caches. This means that multiple calls to load_airbyte_cloud_asset_specs
+        # will not trigger multiple API calls to fetch the workspace data.
+        # Bespoke impl since @cached_method doesn't play nice with iterators; it's exhausted after
+        # the first call.
+        if hasattr(self, "_initialized"):
+            yield getattr(self, "_initialized")
+        else:
+            with self.process_config_and_initialize_cm() as initialized_workspace:
+                initialized = initialized_workspace
+                setattr(self, "_initialized", initialized)
+                yield initialized
+
 
 @beta
 def load_airbyte_cloud_asset_specs(
     workspace: AirbyteCloudWorkspace,
     dagster_airbyte_translator: Optional[DagsterAirbyteTranslator] = None,
+    connection_selector_fn: Optional[Callable[[AirbyteConnection], bool]] = None,
 ) -> Sequence[AssetSpec]:
     """Returns a list of AssetSpecs representing the Airbyte content in the workspace.
 
@@ -1330,6 +1350,8 @@ def load_airbyte_cloud_asset_specs(
         dagster_airbyte_translator (Optional[DagsterAirbyteTranslator], optional): The translator to use
             to convert Airbyte content into :py:class:`dagster.AssetSpec`.
             Defaults to :py:class:`DagsterAirbyteTranslator`.
+        connection_selector_fn (Optional[Callable[[AirbyteConnection], bool]]): A function that allows for filtering
+            which Airbyte connection assets are created for.
 
     Returns:
         List[AssetSpec]: The set of assets representing the Airbyte content in the workspace.
@@ -1349,13 +1371,32 @@ def load_airbyte_cloud_asset_specs(
                 client_secret=dg.EnvVar("AIRBYTE_CLOUD_CLIENT_SECRET"),
             )
 
-
             airbyte_cloud_specs = load_airbyte_cloud_asset_specs(airbyte_cloud_workspace)
             defs = dg.Definitions(assets=airbyte_cloud_specs)
+
+        Filter connections by name:
+
+        .. code-block:: python
+
+            from dagster_airbyte import AirbyteCloudWorkspace, load_airbyte_cloud_asset_specs
+
+            import dagster as dg
+
+            airbyte_cloud_workspace = AirbyteCloudWorkspace(
+                workspace_id=dg.EnvVar("AIRBYTE_CLOUD_WORKSPACE_ID"),
+                client_id=dg.EnvVar("AIRBYTE_CLOUD_CLIENT_ID"),
+                client_secret=dg.EnvVar("AIRBYTE_CLOUD_CLIENT_SECRET"),
+            )
+
+            airbyte_cloud_specs = load_airbyte_cloud_asset_specs(
+                workspace=airbyte_cloud_workspace,
+                connection_selector_fn=lambda connection: connection.name in ["connection1", "connection2"]
+            )
+            defs = dg.Definitions(assets=airbyte_cloud_specs)
     """
     dagster_airbyte_translator = dagster_airbyte_translator or DagsterAirbyteTranslator()
 
-    with workspace.process_config_and_initialize_cm() as initialized_workspace:
+    with workspace.process_config_and_initialize_cm_cached() as initialized_workspace:
         return [
             spec.merge_attributes(
                 metadata={DAGSTER_AIRBYTE_TRANSLATOR_METADATA_KEY: dagster_airbyte_translator}
@@ -1364,6 +1405,7 @@ def load_airbyte_cloud_asset_specs(
                 AirbyteCloudWorkspaceDefsLoader(
                     workspace=initialized_workspace,
                     translator=dagster_airbyte_translator,
+                    connection_selector_fn=connection_selector_fn,
                 )
                 .build_defs()
                 .assets,
@@ -1373,21 +1415,24 @@ def load_airbyte_cloud_asset_specs(
 
 
 @record
-class AirbyteCloudWorkspaceDefsLoader(StateBackedDefinitionsLoader[Mapping[str, Any]]):
+class AirbyteCloudWorkspaceDefsLoader(StateBackedDefinitionsLoader[AirbyteWorkspaceData]):
     workspace: AirbyteCloudWorkspace
     translator: DagsterAirbyteTranslator
+    connection_selector_fn: Optional[Callable[[AirbyteConnection], bool]]
 
     @property
     def defs_key(self) -> str:
-        return f"{AIRBYTE_RECONSTRUCTION_METADATA_KEY_PREFIX}/{self.workspace.workspace_id}"
+        return f"{AIRBYTE_RECONSTRUCTION_METADATA_KEY_PREFIX}.{self.workspace.workspace_id}"
 
-    def fetch_state(self) -> AirbyteWorkspaceData:  # pyright: ignore[reportIncompatibleMethodOverride]
+    def fetch_state(self) -> AirbyteWorkspaceData:
         return self.workspace.fetch_airbyte_workspace_data()
 
-    def defs_from_state(self, state: AirbyteWorkspaceData) -> Definitions:  # pyright: ignore[reportIncompatibleMethodOverride]
+    def defs_from_state(self, state: AirbyteWorkspaceData) -> Definitions:
         all_asset_specs = [
             self.translator.get_asset_spec(props)
             for props in state.to_airbyte_connection_table_props_data()
+            if not self.connection_selector_fn
+            or self.connection_selector_fn(state.connections_by_id[props.connection_id])
         ]
 
         return Definitions(assets=all_asset_specs)

dagster_airbyte/version.py

@@ -1 +1 @@
-__version__ = "0.26.18rc2"
+__version__ = "0.26.19"

{dagster_airbyte-0.26.18rc2.dist-info → dagster_airbyte-0.26.19.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dagster-airbyte
-Version: 0.26.18rc2
+Version: 0.26.19
 Summary: Package for integrating Airbyte with Dagster.
 Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-airbyte
 Author: Dagster Labs
@@ -14,13 +14,13 @@ Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Operating System :: OS Independent
 Requires-Python: >=3.9,<3.13
 License-File: LICENSE
-Requires-Dist: dagster==1.10.18rc2
+Requires-Dist: dagster==1.10.19
 Requires-Dist: requests
 Provides-Extra: test
 Requires-Dist: requests-mock; extra == "test"
 Requires-Dist: flaky; extra == "test"
 Provides-Extra: managed
-Requires-Dist: dagster-managed-elements==0.26.18rc2; extra == "managed"
+Requires-Dist: dagster-managed-elements==0.26.19; extra == "managed"
 Dynamic: author
 Dynamic: author-email
 Dynamic: classifier

{dagster_airbyte-0.26.18rc2.dist-info → dagster_airbyte-0.26.19.dist-info}/RECORD

@@ -1,23 +1,23 @@
 dagster_airbyte/__init__.py,sha256=Cdn8CiAL5HSFeFjmKIZgXJqccj11xqQFzRfUQlD15b0,1720
 dagster_airbyte/asset_decorator.py,sha256=U6pbeyjE0meeXHed7ZVhaRfAsvLwTvDB14BSyDUYHqY,4565
-dagster_airbyte/asset_defs.py,sha256=X-ong0InnxXe61_H9IZc6ATCACylKb-JQrPbtH-XTHo,50019
+dagster_airbyte/asset_defs.py,sha256=0mSVP7rx7xYku3t3Ou7FHA91_GDtdcSvroA885mb5Ks,51296
 dagster_airbyte/cli.py,sha256=HErteP1MjfHozKKSrznh0yAreKETbXp5NDHzXGsdvvE,425
 dagster_airbyte/ops.py,sha256=oOEczVYpqVRTc1-0osfpR5FZbPjNJDYihgRjk_qtOQA,4229
 dagster_airbyte/py.typed,sha256=la67KBlbjXN-_-DfGNcdOcjYumVpKG_Tkw-8n5dnGB4,8
-dagster_airbyte/resources.py,sha256=ynOI_MbPnKwcW9OhqrbDchmP6RY4eXQ1018PFJokg4g,54090
+dagster_airbyte/resources.py,sha256=TldPjZnbDi6RuKI9X9Y1ZJz14C_fQV85YIQwyqihtZM,56512
 dagster_airbyte/translator.py,sha256=CL2EKeZIahgoFvtZpfEwVa2KmCuvUENXByM6F7369LQ,7529
 dagster_airbyte/types.py,sha256=TYUjI3skjLYeANjesgJ-IAJNu8bAnL1ymsUfz5LsRTE,1565
 dagster_airbyte/utils.py,sha256=SupW5PLwzyZHWTHKe17vh2PlGdcRT7xbYbMCT-ubGDo,4081
-dagster_airbyte/version.py,sha256=3jY34xulRH13mQY1DvMy3EUkXLpEhCUHGkuiPuWl5a4,27
+dagster_airbyte/version.py,sha256=hFTnREFUNV6k-r7gNk8pbgoXT_sSxa-pE2ccr7Zysn4,24
 dagster_airbyte/managed/__init__.py,sha256=6SBtyNOMJ9Cu2UIwFExJHpL_ZVFo3rPMvyIxVOsKvWE,469
 dagster_airbyte/managed/reconciliation.py,sha256=LHtJc2zRwk1gAc86pKVSZ0rnfLNTT9r3usRFQilYCwY,34803
 dagster_airbyte/managed/types.py,sha256=isPfX8L9YwtZAf9Vk4hhxBePLR00AEldsdK2TsM1H2o,14611
 dagster_airbyte/managed/generated/__init__.py,sha256=eYq-yfXEeffuKAVFXY8plD0se1wHjFNVqklpbu9gljw,108
 dagster_airbyte/managed/generated/destinations.py,sha256=x1wmWlXvOJHtfaZva3ErdKuVS--sDvfidSXR5ji9G5w,119692
 dagster_airbyte/managed/generated/sources.py,sha256=y0TPNvcRd8c9mhje-NoXsHeKRPt1nXcpww8mNAtqCps,282685
-dagster_airbyte-0.26.18rc2.dist-info/licenses/LICENSE,sha256=4lsMW-RCvfVD4_F57wrmpe3vX1xwUk_OAKKmV_XT7Z0,11348
-dagster_airbyte-0.26.18rc2.dist-info/METADATA,sha256=xmaR3GiGe7mK7VTT92bJtuNXJja181xsMV2JUlg9qA4,1127
-dagster_airbyte-0.26.18rc2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-dagster_airbyte-0.26.18rc2.dist-info/entry_points.txt,sha256=XrbLOz3LpgPV5fdwMmgdP6Rp1AfSG07KeWIddLqh7Lw,61
-dagster_airbyte-0.26.18rc2.dist-info/top_level.txt,sha256=HLwIRQCzqItn88_KbPP8DNTKKQEBUVKk6NCn4PrCtqY,16
-dagster_airbyte-0.26.18rc2.dist-info/RECORD,,
+dagster_airbyte-0.26.19.dist-info/licenses/LICENSE,sha256=4lsMW-RCvfVD4_F57wrmpe3vX1xwUk_OAKKmV_XT7Z0,11348
+dagster_airbyte-0.26.19.dist-info/METADATA,sha256=0S28Aj89CflD1UAz0j0g0fsqKUgZWSBcmf3Y_8e-qag,1118
+dagster_airbyte-0.26.19.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+dagster_airbyte-0.26.19.dist-info/entry_points.txt,sha256=XrbLOz3LpgPV5fdwMmgdP6Rp1AfSG07KeWIddLqh7Lw,61
+dagster_airbyte-0.26.19.dist-info/top_level.txt,sha256=HLwIRQCzqItn88_KbPP8DNTKKQEBUVKk6NCn4PrCtqY,16
+dagster_airbyte-0.26.19.dist-info/RECORD,,