dagster-airbyte 0.27.13__py3-none-any.whl → 0.27.15__py3-none-any.whl

dagster_airbyte/components/workspace_component/component.py

@@ -1,36 +1,41 @@
-from collections.abc import Sequence
+from collections import defaultdict
+from collections.abc import Iterable, Sequence
 from functools import cached_property
+from pathlib import Path
 from typing import Annotated, Callable, Optional, Union
 
 import dagster as dg
 import pydantic
 from dagster._annotations import superseded
-from dagster._core.definitions.job_definition import default_job_io_manager
+from dagster._utils.names import clean_name
+from dagster.components.component.state_backed_component import StateBackedComponent
 from dagster.components.resolved.base import resolve_fields
-from dagster.components.utils.translation import TranslationFn, TranslationFnResolver
+from dagster.components.utils.defs_state import (
+    DefsStateConfig,
+    DefsStateConfigArgs,
+    ResolvedDefsStateConfig,
+)
+from dagster.components.utils.translation import (
+    ComponentTranslator,
+    TranslationFn,
+    TranslationFnResolver,
+    create_component_translator_cls,
+)
 from dagster_shared import check
+from dagster_shared.serdes.serdes import deserialize_value
 
-from dagster_airbyte.asset_defs import build_airbyte_assets_definitions
 from dagster_airbyte.components.workspace_component.scaffolder import (
     AirbyteWorkspaceComponentScaffolder,
 )
-from dagster_airbyte.resources import AirbyteCloudWorkspace, AirbyteWorkspace
+from dagster_airbyte.resources import AirbyteCloudWorkspace, AirbyteWorkspace, BaseAirbyteWorkspace
 from dagster_airbyte.translator import (
     AirbyteConnection,
     AirbyteConnectionTableProps,
+    AirbyteMetadataSet,
+    AirbyteWorkspaceData,
     DagsterAirbyteTranslator,
 )
-
-
-class ProxyDagsterAirbyteTranslator(DagsterAirbyteTranslator):
-    def __init__(self, fn: TranslationFn[AirbyteConnectionTableProps]):
-        self.fn = fn
-
-    def get_asset_spec(self, props: AirbyteConnectionTableProps) -> dg.AssetSpec:
-        base_asset_spec = super().get_asset_spec(props)
-        spec = self.fn(base_asset_spec, props)
-
-        return spec
+from dagster_airbyte.utils import DAGSTER_AIRBYTE_TRANSLATOR_METADATA_KEY
 
 
 class BaseAirbyteWorkspaceModel(dg.Model, dg.Resolvable):
@@ -168,7 +173,7 @@ def resolve_airbyte_workspace_type(context: dg.ResolutionContext, model):
 
 
 @dg.scaffold_with(AirbyteWorkspaceComponentScaffolder)
-class AirbyteWorkspaceComponent(dg.Component, dg.Model, dg.Resolvable):
+class AirbyteWorkspaceComponent(StateBackedComponent, dg.Model, dg.Resolvable):
     """Loads Airbyte connections from a given Airbyte workspace as Dagster assets.
     Materializing these assets will trigger a sync of the Airbyte connection, enabling
     you to schedule Airbyte syncs using Dagster.
@@ -197,34 +202,97 @@ class AirbyteWorkspaceComponent(dg.Component, dg.Model, dg.Resolvable):
             TranslationFnResolver(template_vars_for_translation_fn=lambda data: {"props": data}),
         ]
     ] = pydantic.Field(
-        None,
+        default=None,
         description="Function used to translate Airbyte connection table properties into Dagster asset specs.",
     )
+    defs_state: ResolvedDefsStateConfig = DefsStateConfigArgs.legacy_code_server_snapshots()
+
+    @property
+    def defs_state_config(self) -> DefsStateConfig:
+        default_key = f"{self.__class__.__name__}[{self.workspace.workspace_id}]"
+        return DefsStateConfig.from_args(self.defs_state, default_key=default_key)
 
     @cached_property
     def translator(self) -> DagsterAirbyteTranslator:
-        if self.translation:
-            return ProxyDagsterAirbyteTranslator(self.translation)
+        return AirbyteComponentTranslator(self)
+
+    @cached_property
+    def _base_translator(self) -> DagsterAirbyteTranslator:
         return DagsterAirbyteTranslator()
 
-    def build_defs(self, context: dg.ComponentLoadContext) -> dg.Definitions:
-        airbyte_assets = build_airbyte_assets_definitions(
-            workspace=self.workspace,
-            dagster_airbyte_translator=self.translator,
-            connection_selector_fn=self.connection_selector,
+    def get_asset_spec(self, props: AirbyteConnectionTableProps) -> dg.AssetSpec:
+        return self._base_translator.get_asset_spec(props)
+
+    def execute(
+        self, context: dg.AssetExecutionContext, airbyte: BaseAirbyteWorkspace
+    ) -> Iterable[Union[dg.AssetMaterialization, dg.MaterializeResult]]:
+        yield from airbyte.sync_and_poll(context=context)
+
+    def _load_asset_specs(self, state: AirbyteWorkspaceData) -> Sequence[dg.AssetSpec]:
+        connection_selector_fn = self.connection_selector or (lambda connection: True)
+        return [
+            self.translator.get_asset_spec(props).merge_attributes(
+                metadata={DAGSTER_AIRBYTE_TRANSLATOR_METADATA_KEY: self.translator}
+            )
+            for props in state.to_airbyte_connection_table_props_data()
+            if connection_selector_fn(state.connections_by_id[props.connection_id])
+        ]
+
+    def _get_airbyte_assets_def(
+        self, connection_name: str, specs: Sequence[dg.AssetSpec]
+    ) -> dg.AssetsDefinition:
+        @dg.multi_asset(
+            name=f"airbyte_{clean_name(connection_name)}",
+            can_subset=True,
+            specs=specs,
         )
-        assets_with_resource = [
-            airbyte_asset.with_resources(
-                {
-                    "airbyte": self.workspace.get_resource_definition(),
-                    "io_manager": default_job_io_manager,
-                }
+        def _asset(context: dg.AssetExecutionContext):
+            yield from self.execute(context=context, airbyte=self.workspace)
+
+        return _asset
+
+    async def write_state_to_path(self, state_path: Path) -> None:
+        state = self.workspace.fetch_airbyte_workspace_data()
+        state_path.write_text(dg.serialize_value(state))
+
+    def build_defs_from_state(
+        self, context: dg.ComponentLoadContext, state_path: Optional[Path]
+    ) -> dg.Definitions:
+        if state_path is None:
+            return dg.Definitions()
+        state = deserialize_value(state_path.read_text(), AirbyteWorkspaceData)
+
+        # group specs by their connector names
+        specs_by_connection_name = defaultdict(list)
+        for spec in self._load_asset_specs(state):
+            connection_name = check.not_none(
+                AirbyteMetadataSet.extract(spec.metadata).connection_name
             )
-            for airbyte_asset in airbyte_assets
+            specs_by_connection_name[connection_name].append(spec)
+
+        # create one assets definition per connection
+        assets = [
+            self._get_airbyte_assets_def(connection_name, specs)
+            for connection_name, specs in specs_by_connection_name.items()
         ]
-        return dg.Definitions(assets=assets_with_resource)
+        return dg.Definitions(assets=assets)
 
 
 # Subclassing to create the alias to be able to use the superseded decorator.
 @superseded(additional_warn_text="Superseded. Use AirbyteWorkspaceComponent instead.")
 class AirbyteCloudWorkspaceComponent(AirbyteWorkspaceComponent): ...
+
+
+class AirbyteComponentTranslator(
+    create_component_translator_cls(AirbyteWorkspaceComponent, DagsterAirbyteTranslator),
+    ComponentTranslator[AirbyteWorkspaceComponent],
+):
+    def __init__(self, component: AirbyteWorkspaceComponent):
+        self._component = component
+
+    def get_asset_spec(self, props: AirbyteConnectionTableProps) -> dg.AssetSpec:
+        base_asset_spec = super().get_asset_spec(props)
+        if self.component.translation is None:
+            return base_asset_spec
+        else:
+            return self.component.translation(base_asset_spec, props)

dagster_airbyte/version.py

@@ -1 +1 @@
-__version__ = "0.27.13"
+__version__ = "0.27.15"

{dagster_airbyte-0.27.13.dist-info → dagster_airbyte-0.27.15.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dagster-airbyte
-Version: 0.27.13
+Version: 0.27.15
 Summary: Package for integrating Airbyte with Dagster.
 Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-airbyte
 Author: Dagster Labs
@@ -15,13 +15,13 @@ Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Operating System :: OS Independent
 Requires-Python: >=3.9,<3.14
 License-File: LICENSE
-Requires-Dist: dagster==1.11.13
+Requires-Dist: dagster==1.11.15
 Requires-Dist: requests
 Provides-Extra: test
 Requires-Dist: requests-mock; extra == "test"
 Requires-Dist: flaky; extra == "test"
 Provides-Extra: managed
-Requires-Dist: dagster-managed-elements==0.27.13; extra == "managed"
+Requires-Dist: dagster-managed-elements==0.27.15; extra == "managed"
 Dynamic: author
 Dynamic: author-email
 Dynamic: classifier

{dagster_airbyte-0.27.13.dist-info → dagster_airbyte-0.27.15.dist-info}/RECORD

@@ -8,10 +8,10 @@ dagster_airbyte/resources.py,sha256=AXN_mtNl_JKauoOY29wyuiv9jjAiCNF9M4baVCQPMYo,
 dagster_airbyte/translator.py,sha256=RY2LhGPACIfyd2zOSH1swdLz4koX6_bvclW-alS3Bic,7547
 dagster_airbyte/types.py,sha256=TYUjI3skjLYeANjesgJ-IAJNu8bAnL1ymsUfz5LsRTE,1565
 dagster_airbyte/utils.py,sha256=wG9119kXi87JgcOjK7iNozr-svZocJBQYoHBMmnXZcE,4092
-dagster_airbyte/version.py,sha256=rEMnLvYM5w34kWC8YIc4QXFgiezRHgYr2j_AJji8ui8,24
+dagster_airbyte/version.py,sha256=nh4rkWKwcS4_2NEvEKWozrztgCUEn_1ODIolwNpGd8g,24
 dagster_airbyte/components/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dagster_airbyte/components/workspace_component/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-dagster_airbyte/components/workspace_component/component.py,sha256=TIl8Ja6BHwpyRmkzkn-o75sH1dOUbbHtTftAb68Ag_c,8445
+dagster_airbyte/components/workspace_component/component.py,sha256=nGOQXivXS-Zc69_K1aq2xATT06Roci3fAJ-5x1xOqgU,11238
 dagster_airbyte/components/workspace_component/scaffolder.py,sha256=zk5HDJ6C8zcfUWKslJVyqyBB0LV3JzNAocY2ptT517s,1046
 dagster_airbyte/managed/__init__.py,sha256=6SBtyNOMJ9Cu2UIwFExJHpL_ZVFo3rPMvyIxVOsKvWE,469
 dagster_airbyte/managed/reconciliation.py,sha256=xoVfqPBpNSldkiqOLIPnc7ei8CppHKWtzv8bvxjdqlI,34859
@@ -19,9 +19,9 @@ dagster_airbyte/managed/types.py,sha256=isPfX8L9YwtZAf9Vk4hhxBePLR00AEldsdK2TsM1
 dagster_airbyte/managed/generated/__init__.py,sha256=eYq-yfXEeffuKAVFXY8plD0se1wHjFNVqklpbu9gljw,108
 dagster_airbyte/managed/generated/destinations.py,sha256=x1wmWlXvOJHtfaZva3ErdKuVS--sDvfidSXR5ji9G5w,119692
 dagster_airbyte/managed/generated/sources.py,sha256=y0TPNvcRd8c9mhje-NoXsHeKRPt1nXcpww8mNAtqCps,282685
-dagster_airbyte-0.27.13.dist-info/licenses/LICENSE,sha256=4lsMW-RCvfVD4_F57wrmpe3vX1xwUk_OAKKmV_XT7Z0,11348
-dagster_airbyte-0.27.13.dist-info/METADATA,sha256=WWUOmQj-uKukv-8S2boTU0nKxUIdLi-l_zinaNRkOyw,1169
-dagster_airbyte-0.27.13.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-dagster_airbyte-0.27.13.dist-info/entry_points.txt,sha256=096yvfMP-gNsCgDg9vDQtinis5QGpD-e_kHEhcHaML8,120
-dagster_airbyte-0.27.13.dist-info/top_level.txt,sha256=HLwIRQCzqItn88_KbPP8DNTKKQEBUVKk6NCn4PrCtqY,16
-dagster_airbyte-0.27.13.dist-info/RECORD,,
+dagster_airbyte-0.27.15.dist-info/licenses/LICENSE,sha256=4lsMW-RCvfVD4_F57wrmpe3vX1xwUk_OAKKmV_XT7Z0,11348
+dagster_airbyte-0.27.15.dist-info/METADATA,sha256=lvl1x_vvakCJSW40mnHZe-kahcduGWPQZzjMYQ2CObk,1169
+dagster_airbyte-0.27.15.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+dagster_airbyte-0.27.15.dist-info/entry_points.txt,sha256=096yvfMP-gNsCgDg9vDQtinis5QGpD-e_kHEhcHaML8,120
+dagster_airbyte-0.27.15.dist-info/top_level.txt,sha256=HLwIRQCzqItn88_KbPP8DNTKKQEBUVKk6NCn4PrCtqY,16
+dagster_airbyte-0.27.15.dist-info/RECORD,,