dagster-airbyte 0.26.19__tar.gz → 0.26.20__tar.gz

{dagster_airbyte-0.26.19/dagster_airbyte.egg-info → dagster_airbyte-0.26.20}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dagster-airbyte
-Version: 0.26.19
+Version: 0.26.20
 Summary: Package for integrating Airbyte with Dagster.
 Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-airbyte
 Author: Dagster Labs
@@ -14,13 +14,13 @@ Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Operating System :: OS Independent
 Requires-Python: >=3.9,<3.13
 License-File: LICENSE
-Requires-Dist: dagster==1.10.19
+Requires-Dist: dagster==1.10.20
 Requires-Dist: requests
 Provides-Extra: test
 Requires-Dist: requests-mock; extra == "test"
 Requires-Dist: flaky; extra == "test"
 Provides-Extra: managed
-Requires-Dist: dagster-managed-elements==0.26.19; extra == "managed"
+Requires-Dist: dagster-managed-elements==0.26.20; extra == "managed"
 Dynamic: author
 Dynamic: author-email
 Dynamic: classifier

{dagster_airbyte-0.26.19 → dagster_airbyte-0.26.20}/dagster_airbyte/__init__.py

@@ -1,5 +1,9 @@
 from dagster_shared.libraries import DagsterLibraryRegistry
 
+from dagster_airbyte.components.workspace_component.component import (
+    AirbyteCloudWorkspaceComponent as AirbyteCloudWorkspaceComponent,
+)
+
 try:
     from dagster_airbyte.managed import (
         AirbyteConnection as AirbyteConnection,

{dagster_airbyte-0.26.19 → dagster_airbyte-0.26.20}/dagster_airbyte/asset_defs.py

@@ -1158,7 +1158,7 @@ def build_airbyte_assets_definitions(
         @airbyte_assets(
             connection_id=connection_id,
             workspace=workspace,
-            name=clean_name(connection_name),
+            name=f"airbyte_{clean_name(connection_name)}",
             dagster_airbyte_translator=dagster_airbyte_translator,
         )
         def _asset_fn(context: AssetExecutionContext, airbyte: AirbyteCloudWorkspace):

dagster_airbyte-0.26.20/dagster_airbyte/components/workspace_component/component.py

@@ -0,0 +1,155 @@
+from collections.abc import Sequence
+from functools import cached_property
+from typing import Annotated, Callable, Optional, Union
+
+import dagster as dg
+import pydantic
+from dagster._core.definitions.job_definition import default_job_io_manager
+from dagster.components.resolved.base import resolve_fields
+from dagster.components.utils import TranslatorResolvingInfo
+from dagster_shared import check
+from typing_extensions import TypeAlias
+
+from dagster_airbyte.asset_defs import build_airbyte_assets_definitions
+from dagster_airbyte.components.workspace_component.scaffolder import (
+    AirbyteCloudWorkspaceComponentScaffolder,
+)
+from dagster_airbyte.resources import AirbyteCloudWorkspace
+from dagster_airbyte.translator import (
+    AirbyteConnection,
+    AirbyteConnectionTableProps,
+    DagsterAirbyteTranslator,
+)
+
+
+def resolve_translation(context: dg.ResolutionContext, model):
+    info = TranslatorResolvingInfo(
+        "props",
+        asset_attributes=model,
+        resolution_context=context,
+        model_key="translation",
+    )
+    return lambda base_asset_spec, props: info.get_asset_spec(
+        base_asset_spec,
+        {
+            "props": props,
+            "spec": base_asset_spec,
+        },
+    )
+
+
+TranslationFn: TypeAlias = Callable[[dg.AssetSpec, AirbyteConnectionTableProps], dg.AssetSpec]
+ResolvedTranslationFn: TypeAlias = Annotated[
+    TranslationFn,
+    dg.Resolver(
+        resolve_translation,
+        model_field_type=Union[str, dg.AssetAttributesModel],
+    ),
+]
+
+
+class ProxyDagsterAirbyteTranslator(DagsterAirbyteTranslator):
+    def __init__(self, fn: TranslationFn):
+        self.fn = fn
+
+    def get_asset_spec(self, props: AirbyteConnectionTableProps) -> dg.AssetSpec:
+        base_asset_spec = super().get_asset_spec(props)
+        spec = self.fn(base_asset_spec, props)
+
+        return spec
+
+
+class AirbyteCloudWorkspaceModel(dg.Model):
+    workspace_id: Annotated[str, pydantic.Field(..., description="The Airbyte Cloud workspace ID.")]
+    client_id: Annotated[
+        str, pydantic.Field(..., description="Client ID used to authenticate to Airbyte Cloud.")
+    ]
+    client_secret: Annotated[
+        str, pydantic.Field(..., description="Client secret used to authenticate to Airbyte Cloud.")
+    ]
+
+
+class AirbyteConnectionSelectorByName(dg.Model):
+    by_name: Annotated[
+        Sequence[str],
+        pydantic.Field(..., description="A list of connection names to include in the collection."),
+    ]
+
+
+class AirbyteConnectionSelectorById(dg.Model):
+    by_id: Annotated[
+        Sequence[str],
+        pydantic.Field(..., description="A list of connection IDs to include in the collection."),
+    ]
+
+
+def resolve_connection_selector(
+    context: dg.ResolutionContext, model
+) -> Optional[Callable[[AirbyteConnection], bool]]:
+    if isinstance(model, str):
+        model = context.resolve_value(model)
+
+    if isinstance(model, AirbyteConnectionSelectorByName):
+        return lambda connection: connection.name in model.by_name
+    elif isinstance(model, AirbyteConnectionSelectorById):
+        return lambda connection: connection.id in model.by_id
+    else:
+        check.failed(f"Unknown connection target type: {type(model)}")
+
+
+@dg.scaffold_with(AirbyteCloudWorkspaceComponentScaffolder)
+class AirbyteCloudWorkspaceComponent(dg.Component, dg.Model, dg.Resolvable):
+    """Loads Airbyte Cloud connections from a given Airbyte Cloud workspace as Dagster assets.
+    Materializing these assets will trigger a sync of the Airbyte Cloud connection, enabling
+    you to schedule Airbyte Cloud syncs using Dagster.
+    """
+
+    workspace: Annotated[
+        AirbyteCloudWorkspace,
+        dg.Resolver(
+            lambda context, model: AirbyteCloudWorkspace(
+                **resolve_fields(model, AirbyteCloudWorkspace, context)  # type: ignore
+            )
+        ),
+    ]
+    connection_selector: Annotated[
+        Optional[Callable[[AirbyteConnection], bool]],
+        dg.Resolver(
+            resolve_connection_selector,
+            model_field_type=Union[
+                str, AirbyteConnectionSelectorByName, AirbyteConnectionSelectorById
+            ],
+            description="Function used to select Airbyte Cloud connections to pull into Dagster.",
+        ),
+    ] = None
+    translation: Optional[ResolvedTranslationFn] = pydantic.Field(
+        None,
+        description="Function used to translate Airbyte Cloud connection table properties into Dagster asset specs.",
+    )
+
+    @cached_property
+    def workspace_resource(self) -> AirbyteCloudWorkspace:
+        return self.workspace
+
+    @cached_property
+    def translator(self) -> DagsterAirbyteTranslator:
+        if self.translation:
+            return ProxyDagsterAirbyteTranslator(self.translation)
+        return DagsterAirbyteTranslator()
+
+    def build_defs(self, context: dg.ComponentLoadContext) -> dg.Definitions:
+        airbyte_assets = build_airbyte_assets_definitions(
+            workspace=self.workspace_resource,
+            dagster_airbyte_translator=self.translator,
+            connection_selector_fn=self.connection_selector,
+        )
+        assets_with_resource = [
+            airbyte_asset.with_resources(
+                {
+                    "airbyte": self.workspace_resource.get_resource_definition(),
+                    "io_manager": default_job_io_manager,
+                }
+            )
+            for airbyte_asset in airbyte_assets
+        ]
+        return dg.Definitions(assets=assets_with_resource)

dagster_airbyte-0.26.20/dagster_airbyte/components/workspace_component/scaffolder.py

@@ -0,0 +1,30 @@
+from typing import Optional
+
+from dagster.components.component.component_scaffolder import Scaffolder
+from dagster.components.component_scaffolding import scaffold_component
+from dagster.components.scaffold.scaffold import ScaffoldRequest
+from pydantic import BaseModel
+
+
+class AirbyteScaffolderParams(BaseModel):
+    workspace_id: Optional[str] = None
+    client_id: Optional[str] = None
+    client_secret: Optional[str] = None
+
+
+class AirbyteCloudWorkspaceComponentScaffolder(Scaffolder[AirbyteScaffolderParams]):
+    @classmethod
+    def get_scaffold_params(cls) -> type[AirbyteScaffolderParams]:
+        return AirbyteScaffolderParams
+
+    def scaffold(self, request: ScaffoldRequest[AirbyteScaffolderParams]) -> None:
+        scaffold_component(
+            request,
+            {
+                "workspace": {
+                    "workspace_id": request.params.workspace_id,
+                    "client_id": request.params.client_id,
+                    "client_secret": request.params.client_secret,
+                }
+            },
+        )

dagster_airbyte-0.26.20/dagster_airbyte/version.py

@@ -0,0 +1 @@
+__version__ = "0.26.20"

{dagster_airbyte-0.26.19 → dagster_airbyte-0.26.20/dagster_airbyte.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dagster-airbyte
-Version: 0.26.19
+Version: 0.26.20
 Summary: Package for integrating Airbyte with Dagster.
 Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-airbyte
 Author: Dagster Labs
@@ -14,13 +14,13 @@ Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Operating System :: OS Independent
 Requires-Python: >=3.9,<3.13
 License-File: LICENSE
-Requires-Dist: dagster==1.10.19
+Requires-Dist: dagster==1.10.20
 Requires-Dist: requests
 Provides-Extra: test
 Requires-Dist: requests-mock; extra == "test"
 Requires-Dist: flaky; extra == "test"
 Provides-Extra: managed
-Requires-Dist: dagster-managed-elements==0.26.19; extra == "managed"
+Requires-Dist: dagster-managed-elements==0.26.20; extra == "managed"
 Dynamic: author
 Dynamic: author-email
 Dynamic: classifier

{dagster_airbyte-0.26.19 → dagster_airbyte-0.26.20}/dagster_airbyte.egg-info/SOURCES.txt

@@ -21,6 +21,10 @@ dagster_airbyte.egg-info/entry_points.txt
 dagster_airbyte.egg-info/not-zip-safe
 dagster_airbyte.egg-info/requires.txt
 dagster_airbyte.egg-info/top_level.txt
+dagster_airbyte/components/__init__.py
+dagster_airbyte/components/workspace_component/__init__.py
+dagster_airbyte/components/workspace_component/component.py
+dagster_airbyte/components/workspace_component/scaffolder.py
 dagster_airbyte/managed/__init__.py
 dagster_airbyte/managed/reconciliation.py
 dagster_airbyte/managed/types.py

{dagster_airbyte-0.26.19 → dagster_airbyte-0.26.20}/dagster_airbyte.egg-info/entry_points.txt

@@ -1,2 +1,5 @@
 [console_scripts]
 dagster-airbyte = dagster_airbyte.cli:main
+
+[dagster_dg_cli.plugin]
+dagster_airbyte = dagster_airbyte

dagster_airbyte-0.26.20/dagster_airbyte.egg-info/requires.txt

@@ -0,0 +1,9 @@
+dagster==1.10.20
+requests
+
+[managed]
+dagster-managed-elements==0.26.20
+
+[test]
+requests-mock
+flaky

{dagster_airbyte-0.26.19 → dagster_airbyte-0.26.20}/setup.py

@@ -36,14 +36,17 @@ setup(
     include_package_data=True,
     python_requires=">=3.9,<3.13",
     install_requires=[
-        "dagster==1.10.19",
+        "dagster==1.10.20",
         "requests",
     ],
     zip_safe=False,
     entry_points={
         "console_scripts": [
             "dagster-airbyte = dagster_airbyte.cli:main",
-        ]
+        ],
+        "dagster_dg_cli.plugin": [
+            "dagster_airbyte = dagster_airbyte",
+        ],
     },
     extras_require={
         "test": [
@@ -51,7 +54,7 @@ setup(
             "flaky",
         ],
         "managed": [
-            "dagster-managed-elements==0.26.19",
+            "dagster-managed-elements==0.26.20",
         ],
     },
 )

dagster_airbyte-0.26.19/dagster_airbyte/version.py

@@ -1 +0,0 @@
-__version__ = "0.26.19"

dagster_airbyte-0.26.19/dagster_airbyte.egg-info/requires.txt

@@ -1,9 +0,0 @@
-dagster==1.10.19
-requests
-
-[managed]
-dagster-managed-elements==0.26.19
-
-[test]
-requests-mock
-flaky