dagster-airbyte 0.26.18rc2__tar.gz → 0.26.19__tar.gz

{dagster_airbyte-0.26.18rc2/dagster_airbyte.egg-info → dagster_airbyte-0.26.19}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dagster-airbyte
-Version: 0.26.18rc2
+Version: 0.26.19
 Summary: Package for integrating Airbyte with Dagster.
 Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-airbyte
 Author: Dagster Labs
@@ -14,13 +14,13 @@ Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Operating System :: OS Independent
 Requires-Python: >=3.9,<3.13
 License-File: LICENSE
-Requires-Dist: dagster==1.10.18rc2
+Requires-Dist: dagster==1.10.19
 Requires-Dist: requests
 Provides-Extra: test
 Requires-Dist: requests-mock; extra == "test"
 Requires-Dist: flaky; extra == "test"
 Provides-Extra: managed
-Requires-Dist: dagster-managed-elements==0.26.18rc2; extra == "managed"
+Requires-Dist: dagster-managed-elements==0.26.19; extra == "managed"
 Dynamic: author
 Dynamic: author-email
 Dynamic: classifier

{dagster_airbyte-0.26.18rc2 → dagster_airbyte-0.26.19}/dagster_airbyte/asset_defs.py

@@ -40,7 +40,11 @@ from dagster_airbyte.resources import (
     AirbyteResource,
     BaseAirbyteResource,
 )
-from dagster_airbyte.translator import AirbyteMetadataSet, DagsterAirbyteTranslator
+from dagster_airbyte.translator import (
+    AirbyteConnection,
+    AirbyteMetadataSet,
+    DagsterAirbyteTranslator,
+)
 from dagster_airbyte.types import AirbyteTableMetadata
 from dagster_airbyte.utils import (
     clean_name,
@@ -1035,6 +1039,7 @@ def build_airbyte_assets_definitions(
     *,
     workspace: AirbyteCloudWorkspace,
     dagster_airbyte_translator: Optional[DagsterAirbyteTranslator] = None,
+    connection_selector_fn: Optional[Callable[[AirbyteConnection], bool]] = None,
 ) -> Sequence[AssetsDefinition]:
     """The list of AssetsDefinition for all connections in the Airbyte workspace.
 
@@ -1043,6 +1048,8 @@ def build_airbyte_assets_definitions(
         dagster_airbyte_translator (Optional[DagsterAirbyteTranslator], optional): The translator to use
             to convert Airbyte content into :py:class:`dagster.AssetSpec`.
             Defaults to :py:class:`DagsterAirbyteTranslator`.
+        connection_selector_fn (Optional[Callable[[AirbyteConnection], bool]]): A function that allows for filtering
+            which Airbyte connection assets are created for.
 
     Returns:
         List[AssetsDefinition]: The list of AssetsDefinition for all connections in the Airbyte workspace.
@@ -1062,7 +1069,6 @@ def build_airbyte_assets_definitions(
                 client_secret=dg.EnvVar("AIRBYTE_CLOUD_CLIENT_SECRET"),
             )
 
-
             airbyte_assets = build_airbyte_assets_definitions(workspace=workspace)
 
             defs = dg.Definitions(
@@ -1096,21 +1102,46 @@ def build_airbyte_assets_definitions(
                 client_secret=dg.EnvVar("AIRBYTE_CLOUD_CLIENT_SECRET"),
             )
 
-
             airbyte_assets = build_airbyte_assets_definitions(
                 workspace=workspace,
                 dagster_airbyte_translator=CustomDagsterAirbyteTranslator()
             )
 
+            defs = dg.Definitions(
+                assets=airbyte_assets,
+                resources={"airbyte": airbyte_workspace},
+            )
+
+        Filter connections by name:
+
+        .. code-block:: python
+
+            from dagster_airbyte import AirbyteCloudWorkspace, build_airbyte_assets_definitions
+
+            import dagster as dg
+
+            airbyte_workspace = AirbyteCloudWorkspace(
+                workspace_id=dg.EnvVar("AIRBYTE_CLOUD_WORKSPACE_ID"),
+                client_id=dg.EnvVar("AIRBYTE_CLOUD_CLIENT_ID"),
+                client_secret=dg.EnvVar("AIRBYTE_CLOUD_CLIENT_SECRET"),
+            )
+
+            airbyte_assets = build_airbyte_assets_definitions(
+                workspace=workspace,
+                connection_selector_fn=lambda connection: connection.name in ["connection1", "connection2"]
+            )
+
             defs = dg.Definitions(
                 assets=airbyte_assets,
                 resources={"airbyte": airbyte_workspace},
             )
     """
     dagster_airbyte_translator = dagster_airbyte_translator or DagsterAirbyteTranslator()
+    connection_selector_fn = connection_selector_fn or (lambda connection: True)
 
     all_asset_specs = workspace.load_asset_specs(
-        dagster_airbyte_translator=dagster_airbyte_translator
+        dagster_airbyte_translator=dagster_airbyte_translator,
+        connection_selector_fn=connection_selector_fn,
     )
 
     connections = {

{dagster_airbyte-0.26.18rc2 → dagster_airbyte-0.26.19}/dagster_airbyte/resources.py

@@ -4,10 +4,10 @@ import logging
 import sys
 import time
 from abc import abstractmethod
-from collections.abc import Mapping, Sequence
+from collections.abc import Iterator, Mapping, Sequence
 from contextlib import contextmanager
 from datetime import datetime, timedelta
-from typing import Any, Optional, cast
+from typing import Any, Callable, Optional, cast
 
 import requests
 from dagster import (
@@ -1142,7 +1142,6 @@ class AirbyteCloudWorkspace(ConfigurableResource):
         default=15,
         description="Time (in seconds) after which the requests to Airbyte are declared timed out.",
     )
-
     _client: AirbyteCloudClient = PrivateAttr(default=None)  # type: ignore
 
     @cached_method
@@ -1197,6 +1196,7 @@ class AirbyteCloudWorkspace(ConfigurableResource):
     def load_asset_specs(
         self,
         dagster_airbyte_translator: Optional[DagsterAirbyteTranslator] = None,
+        connection_selector_fn: Optional[Callable[[AirbyteConnection], bool]] = None,
     ) -> Sequence[AssetSpec]:
         """Returns a list of AssetSpecs representing the Airbyte content in the workspace.
 
@@ -1204,6 +1204,8 @@ class AirbyteCloudWorkspace(ConfigurableResource):
             dagster_airbyte_translator (Optional[DagsterAirbyteTranslator], optional): The translator to use
                 to convert Airbyte content into :py:class:`dagster.AssetSpec`.
                 Defaults to :py:class:`DagsterAirbyteTranslator`.
+            connection_selector_fn (Optional[Callable[[AirbyteConnection], bool]]): A function that allows for filtering
+                which Airbyte connection assets are created for.
 
         Returns:
             List[AssetSpec]: The set of assets representing the Airbyte content in the workspace.
@@ -1228,7 +1230,9 @@ class AirbyteCloudWorkspace(ConfigurableResource):
         dagster_airbyte_translator = dagster_airbyte_translator or DagsterAirbyteTranslator()
 
         return load_airbyte_cloud_asset_specs(
-            workspace=self, dagster_airbyte_translator=dagster_airbyte_translator
+            workspace=self,
+            dagster_airbyte_translator=dagster_airbyte_translator,
+            connection_selector_fn=connection_selector_fn,
         )
 
     def _generate_materialization(
@@ -1317,11 +1321,27 @@ class AirbyteCloudWorkspace(ConfigurableResource):
         if unmaterialized_asset_keys:
             context.log.warning(f"Assets were not materialized: {unmaterialized_asset_keys}")
 
+    @contextmanager
+    def process_config_and_initialize_cm_cached(self) -> Iterator["AirbyteCloudWorkspace"]:
+        # Hack to avoid reconstructing initialized copies of this resource, which invalidates
+        # @cached_method caches. This means that multiple calls to load_airbyte_cloud_asset_specs
+        # will not trigger multiple API calls to fetch the workspace data.
+        # Bespoke impl since @cached_method doesn't play nice with iterators; it's exhausted after
+        # the first call.
+        if hasattr(self, "_initialized"):
+            yield getattr(self, "_initialized")
+        else:
+            with self.process_config_and_initialize_cm() as initialized_workspace:
+                initialized = initialized_workspace
+                setattr(self, "_initialized", initialized)
+                yield initialized
+
 
 @beta
 def load_airbyte_cloud_asset_specs(
     workspace: AirbyteCloudWorkspace,
     dagster_airbyte_translator: Optional[DagsterAirbyteTranslator] = None,
+    connection_selector_fn: Optional[Callable[[AirbyteConnection], bool]] = None,
 ) -> Sequence[AssetSpec]:
     """Returns a list of AssetSpecs representing the Airbyte content in the workspace.
 
@@ -1330,6 +1350,8 @@ def load_airbyte_cloud_asset_specs(
         dagster_airbyte_translator (Optional[DagsterAirbyteTranslator], optional): The translator to use
             to convert Airbyte content into :py:class:`dagster.AssetSpec`.
             Defaults to :py:class:`DagsterAirbyteTranslator`.
+        connection_selector_fn (Optional[Callable[[AirbyteConnection], bool]]): A function that allows for filtering
+            which Airbyte connection assets are created for.
 
     Returns:
         List[AssetSpec]: The set of assets representing the Airbyte content in the workspace.
@@ -1349,13 +1371,32 @@ def load_airbyte_cloud_asset_specs(
                 client_secret=dg.EnvVar("AIRBYTE_CLOUD_CLIENT_SECRET"),
             )
 
-
             airbyte_cloud_specs = load_airbyte_cloud_asset_specs(airbyte_cloud_workspace)
             defs = dg.Definitions(assets=airbyte_cloud_specs)
+
+        Filter connections by name:
+
+        .. code-block:: python
+
+            from dagster_airbyte import AirbyteCloudWorkspace, load_airbyte_cloud_asset_specs
+
+            import dagster as dg
+
+            airbyte_cloud_workspace = AirbyteCloudWorkspace(
+                workspace_id=dg.EnvVar("AIRBYTE_CLOUD_WORKSPACE_ID"),
+                client_id=dg.EnvVar("AIRBYTE_CLOUD_CLIENT_ID"),
+                client_secret=dg.EnvVar("AIRBYTE_CLOUD_CLIENT_SECRET"),
+            )
+
+            airbyte_cloud_specs = load_airbyte_cloud_asset_specs(
+                workspace=airbyte_cloud_workspace,
+                connection_selector_fn=lambda connection: connection.name in ["connection1", "connection2"]
+            )
+            defs = dg.Definitions(assets=airbyte_cloud_specs)
     """
     dagster_airbyte_translator = dagster_airbyte_translator or DagsterAirbyteTranslator()
 
-    with workspace.process_config_and_initialize_cm() as initialized_workspace:
+    with workspace.process_config_and_initialize_cm_cached() as initialized_workspace:
         return [
             spec.merge_attributes(
                 metadata={DAGSTER_AIRBYTE_TRANSLATOR_METADATA_KEY: dagster_airbyte_translator}
@@ -1364,6 +1405,7 @@ def load_airbyte_cloud_asset_specs(
                 AirbyteCloudWorkspaceDefsLoader(
                     workspace=initialized_workspace,
                     translator=dagster_airbyte_translator,
+                    connection_selector_fn=connection_selector_fn,
                 )
                 .build_defs()
                 .assets,
@@ -1373,21 +1415,24 @@ def load_airbyte_cloud_asset_specs(
 
 
 @record
-class AirbyteCloudWorkspaceDefsLoader(StateBackedDefinitionsLoader[Mapping[str, Any]]):
+class AirbyteCloudWorkspaceDefsLoader(StateBackedDefinitionsLoader[AirbyteWorkspaceData]):
     workspace: AirbyteCloudWorkspace
     translator: DagsterAirbyteTranslator
+    connection_selector_fn: Optional[Callable[[AirbyteConnection], bool]]
 
     @property
     def defs_key(self) -> str:
-        return f"{AIRBYTE_RECONSTRUCTION_METADATA_KEY_PREFIX}/{self.workspace.workspace_id}"
+        return f"{AIRBYTE_RECONSTRUCTION_METADATA_KEY_PREFIX}.{self.workspace.workspace_id}"
 
-    def fetch_state(self) -> AirbyteWorkspaceData:  # pyright: ignore[reportIncompatibleMethodOverride]
+    def fetch_state(self) -> AirbyteWorkspaceData:
         return self.workspace.fetch_airbyte_workspace_data()
 
-    def defs_from_state(self, state: AirbyteWorkspaceData) -> Definitions:  # pyright: ignore[reportIncompatibleMethodOverride]
+    def defs_from_state(self, state: AirbyteWorkspaceData) -> Definitions:
         all_asset_specs = [
             self.translator.get_asset_spec(props)
             for props in state.to_airbyte_connection_table_props_data()
+            if not self.connection_selector_fn
+            or self.connection_selector_fn(state.connections_by_id[props.connection_id])
         ]
 
         return Definitions(assets=all_asset_specs)

dagster_airbyte-0.26.19/dagster_airbyte/version.py

@@ -0,0 +1 @@
+__version__ = "0.26.19"

{dagster_airbyte-0.26.18rc2 → dagster_airbyte-0.26.19/dagster_airbyte.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dagster-airbyte
-Version: 0.26.18rc2
+Version: 0.26.19
 Summary: Package for integrating Airbyte with Dagster.
 Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-airbyte
 Author: Dagster Labs
@@ -14,13 +14,13 @@ Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Operating System :: OS Independent
 Requires-Python: >=3.9,<3.13
 License-File: LICENSE
-Requires-Dist: dagster==1.10.18rc2
+Requires-Dist: dagster==1.10.19
 Requires-Dist: requests
 Provides-Extra: test
 Requires-Dist: requests-mock; extra == "test"
 Requires-Dist: flaky; extra == "test"
 Provides-Extra: managed
-Requires-Dist: dagster-managed-elements==0.26.18rc2; extra == "managed"
+Requires-Dist: dagster-managed-elements==0.26.19; extra == "managed"
 Dynamic: author
 Dynamic: author-email
 Dynamic: classifier

dagster_airbyte-0.26.19/dagster_airbyte.egg-info/requires.txt

@@ -0,0 +1,9 @@
+dagster==1.10.19
+requests
+
+[managed]
+dagster-managed-elements==0.26.19
+
+[test]
+requests-mock
+flaky

{dagster_airbyte-0.26.18rc2 → dagster_airbyte-0.26.19}/setup.py

@@ -36,7 +36,7 @@ setup(
     include_package_data=True,
     python_requires=">=3.9,<3.13",
     install_requires=[
-        "dagster==1.10.18rc2",
+        "dagster==1.10.19",
         "requests",
     ],
     zip_safe=False,
@@ -51,7 +51,7 @@ setup(
             "flaky",
         ],
         "managed": [
-            "dagster-managed-elements==0.26.18rc2",
+            "dagster-managed-elements==0.26.19",
         ],
     },
 )

dagster_airbyte-0.26.18rc2/dagster_airbyte/version.py

@@ -1 +0,0 @@
-__version__ = "0.26.18rc2"

dagster_airbyte-0.26.18rc2/dagster_airbyte.egg-info/requires.txt

@@ -1,9 +0,0 @@
-dagster==1.10.18rc2
-requests
-
-[managed]
-dagster-managed-elements==0.26.18rc2
-
-[test]
-requests-mock
-flaky