dagster-airbyte 0.27.10__tar.gz → 0.27.11__tar.gz

{dagster_airbyte-0.27.10/dagster_airbyte.egg-info → dagster_airbyte-0.27.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dagster-airbyte
-Version: 0.27.10
+Version: 0.27.11
 Summary: Package for integrating Airbyte with Dagster.
 Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-airbyte
 Author: Dagster Labs
@@ -15,13 +15,13 @@ Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Operating System :: OS Independent
 Requires-Python: >=3.9,<3.14
 License-File: LICENSE
-Requires-Dist: dagster==1.11.10
+Requires-Dist: dagster==1.11.11
 Requires-Dist: requests
 Provides-Extra: test
 Requires-Dist: requests-mock; extra == "test"
 Requires-Dist: flaky; extra == "test"
 Provides-Extra: managed
-Requires-Dist: dagster-managed-elements==0.27.10; extra == "managed"
+Requires-Dist: dagster-managed-elements==0.27.11; extra == "managed"
 Dynamic: author
 Dynamic: author-email
 Dynamic: classifier

{dagster_airbyte-0.27.10 → dagster_airbyte-0.27.11}/dagster_airbyte/__init__.py

@@ -2,6 +2,7 @@ from dagster_shared.libraries import DagsterLibraryRegistry
 
 from dagster_airbyte.components.workspace_component.component import (
     AirbyteCloudWorkspaceComponent as AirbyteCloudWorkspaceComponent,
+    AirbyteWorkspaceComponent as AirbyteWorkspaceComponent,
 )
 
 try:
@@ -29,8 +30,10 @@ from dagster_airbyte.resources import (
     AirbyteCloudResource as AirbyteCloudResource,
     AirbyteCloudWorkspace as AirbyteCloudWorkspace,
     AirbyteResource as AirbyteResource,
+    AirbyteWorkspace as AirbyteWorkspace,
     airbyte_cloud_resource as airbyte_cloud_resource,
     airbyte_resource as airbyte_resource,
+    load_airbyte_asset_specs as load_airbyte_asset_specs,
     load_airbyte_cloud_asset_specs as load_airbyte_cloud_asset_specs,
 )
 from dagster_airbyte.translator import (

{dagster_airbyte-0.27.10 → dagster_airbyte-0.27.11}/dagster_airbyte/asset_decorator.py

@@ -1,10 +1,10 @@
-from typing import Any, Callable, Optional
+from typing import Any, Callable, Optional, Union
 
 from dagster import AssetsDefinition, multi_asset
 from dagster._annotations import beta
 from dagster._core.errors import DagsterInvariantViolationError
 
-from dagster_airbyte.resources import AirbyteCloudWorkspace
+from dagster_airbyte.resources import AirbyteCloudWorkspace, AirbyteWorkspace
 from dagster_airbyte.translator import AirbyteMetadataSet, DagsterAirbyteTranslator
 
 
@@ -12,7 +12,7 @@ from dagster_airbyte.translator import AirbyteMetadataSet, DagsterAirbyteTransla
 def airbyte_assets(
     *,
     connection_id: str,
-    workspace: AirbyteCloudWorkspace,
+    workspace: Union[AirbyteWorkspace, AirbyteCloudWorkspace],
     name: Optional[str] = None,
     group_name: Optional[str] = None,
     dagster_airbyte_translator: Optional[DagsterAirbyteTranslator] = None,
@@ -21,7 +21,7 @@ def airbyte_assets(
 
     Args:
         connection_id (str): The Airbyte Connection ID.
-        workspace (AirbyteCloudWorkspace): The Airbyte workspace to fetch assets from.
+        workspace (Union[AirbyteWorkspace, AirbyteCloudWorkspace]): The Airbyte workspace to fetch assets from.
         name (Optional[str], optional): The name of the op.
         group_name (Optional[str], optional): The name of the asset group.
         dagster_airbyte_translator (Optional[DagsterAirbyteTranslator], optional): The translator to use

{dagster_airbyte-0.27.10 → dagster_airbyte-0.27.11}/dagster_airbyte/asset_defs.py

@@ -38,7 +38,9 @@ from dagster_airbyte.resources import (
     AirbyteCloudResource,
     AirbyteCloudWorkspace,
     AirbyteResource,
+    AirbyteWorkspace,
     BaseAirbyteResource,
+    BaseAirbyteWorkspace,
 )
 from dagster_airbyte.translator import (
     AirbyteConnection,
@@ -1040,14 +1042,14 @@ def load_assets_from_airbyte_instance(
 @beta
 def build_airbyte_assets_definitions(
     *,
-    workspace: AirbyteCloudWorkspace,
+    workspace: Union[AirbyteWorkspace, AirbyteCloudWorkspace],
     dagster_airbyte_translator: Optional[DagsterAirbyteTranslator] = None,
     connection_selector_fn: Optional[Callable[[AirbyteConnection], bool]] = None,
 ) -> Sequence[AssetsDefinition]:
     """The list of AssetsDefinition for all connections in the Airbyte workspace.
 
     Args:
-        workspace (AirbyteCloudWorkspace): The Airbyte workspace to fetch assets from.
+        workspace (Union[AirbyteWorkspace, AirbyteCloudWorkspace]): The Airbyte workspace to fetch assets from.
         dagster_airbyte_translator (Optional[DagsterAirbyteTranslator], optional): The translator to use
             to convert Airbyte content into :py:class:`dagster.AssetSpec`.
             Defaults to :py:class:`DagsterAirbyteTranslator`.
@@ -1164,7 +1166,7 @@ def build_airbyte_assets_definitions(
             name=f"airbyte_{clean_name(connection_name)}",
             dagster_airbyte_translator=dagster_airbyte_translator,
         )
-        def _asset_fn(context: AssetExecutionContext, airbyte: AirbyteCloudWorkspace):
+        def _asset_fn(context: AssetExecutionContext, airbyte: BaseAirbyteWorkspace):
             yield from airbyte.sync_and_poll(context=context)
 
         _asset_fns.append(_asset_fn)

dagster_airbyte-0.27.11/dagster_airbyte/components/workspace_component/component.py

@@ -0,0 +1,230 @@
+from collections.abc import Sequence
+from functools import cached_property
+from typing import Annotated, Callable, Optional, Union
+
+import dagster as dg
+import pydantic
+from dagster._annotations import superseded
+from dagster._core.definitions.job_definition import default_job_io_manager
+from dagster.components.resolved.base import resolve_fields
+from dagster.components.utils.translation import TranslationFn, TranslationFnResolver
+from dagster_shared import check
+
+from dagster_airbyte.asset_defs import build_airbyte_assets_definitions
+from dagster_airbyte.components.workspace_component.scaffolder import (
+    AirbyteWorkspaceComponentScaffolder,
+)
+from dagster_airbyte.resources import AirbyteCloudWorkspace, AirbyteWorkspace
+from dagster_airbyte.translator import (
+    AirbyteConnection,
+    AirbyteConnectionTableProps,
+    DagsterAirbyteTranslator,
+)
+
+
+class ProxyDagsterAirbyteTranslator(DagsterAirbyteTranslator):
+    def __init__(self, fn: TranslationFn[AirbyteConnectionTableProps]):
+        self.fn = fn
+
+    def get_asset_spec(self, props: AirbyteConnectionTableProps) -> dg.AssetSpec:
+        base_asset_spec = super().get_asset_spec(props)
+        spec = self.fn(base_asset_spec, props)
+
+        return spec
+
+
+class BaseAirbyteWorkspaceModel(dg.Model, dg.Resolvable):
+    request_max_retries: Annotated[
+        int,
+        pydantic.Field(
+            default=3,
+            description=(
+                "The maximum number of times requests to the Airbyte API should be retried "
+                "before failing."
+            ),
+        ),
+    ]
+    request_retry_delay: Annotated[
+        float,
+        pydantic.Field(
+            default=0.25,
+            description="Time (in seconds) to wait between each request retry.",
+        ),
+    ]
+    request_timeout: Annotated[
+        int,
+        pydantic.Field(
+            default=15,
+            description="Time (in seconds) after which the requests to Airbyte are declared timed out.",
+        ),
+    ]
+
+
+class AirbyteWorkspaceModel(BaseAirbyteWorkspaceModel):
+    rest_api_base_url: Annotated[
+        str,
+        pydantic.Field(
+            ...,
+            description=(
+                "The base URL for the Airbyte REST API. "
+                "For Airbyte Cloud, leave this as the default. "
+                "For self-managed Airbyte, this is usually <your Airbyte host>/api/public/v1."
+            ),
+            examples=[
+                "http://localhost:8000/api/public/v1",
+                "https://my-airbyte-server.com/api/public/v1",
+                "http://airbyte-airbyte-server-svc.airbyte.svc.cluster.local:8001/api/public/v1",
+            ],
+        ),
+    ]
+    configuration_api_base_url: Annotated[
+        str,
+        pydantic.Field(
+            ...,
+            description=(
+                "The base URL for the Airbyte Configuration API. "
+                "For Airbyte Cloud, leave this as the default. "
+                "For self-managed Airbyte, this is usually <your Airbyte host>/api/v1."
+            ),
+            examples=[
+                "http://localhost:8000/api/v1",
+                "https://my-airbyte-server.com/api/v1",
+                "http://airbyte-airbyte-server-svc.airbyte.svc.cluster.local:8001/api/v1",
+            ],
+        ),
+    ]
+    workspace_id: Annotated[str, pydantic.Field(..., description="The Airbyte workspace ID.")]
+    client_id: Annotated[
+        Optional[str],
+        pydantic.Field(None, description="Client ID used to authenticate to Airbyte."),
+    ]
+    client_secret: Annotated[
+        Optional[str],
+        pydantic.Field(None, description="Client secret used to authenticate to Airbyte."),
+    ]
+    username: Annotated[
+        Optional[str],
+        pydantic.Field(
+            None,
+            description="Username used to authenticate to Airbyte. Used for self-managed Airbyte with basic auth.",
+        ),
+    ]
+    password: Annotated[
+        Optional[str],
+        pydantic.Field(
+            None,
+            description="Password used to authenticate to Airbyte. Used for self-managed Airbyte with basic auth.",
+        ),
+    ]
+
+
+class AirbyteCloudWorkspaceModel(BaseAirbyteWorkspaceModel):
+    workspace_id: Annotated[str, pydantic.Field(..., description="The Airbyte workspace ID.")]
+    client_id: Annotated[
+        Optional[str],
+        pydantic.Field(..., description="Client ID used to authenticate to Airbyte."),
+    ]
+    client_secret: Annotated[
+        Optional[str],
+        pydantic.Field(..., description="Client secret used to authenticate to Airbyte."),
+    ]
+
+
+class AirbyteConnectionSelectorByName(dg.Model):
+    by_name: Annotated[
+        Sequence[str],
+        pydantic.Field(..., description="A list of connection names to include in the collection."),
+    ]
+
+
+class AirbyteConnectionSelectorById(dg.Model):
+    by_id: Annotated[
+        Sequence[str],
+        pydantic.Field(..., description="A list of connection IDs to include in the collection."),
+    ]
+
+
+def resolve_connection_selector(
+    context: dg.ResolutionContext, model
+) -> Optional[Callable[[AirbyteConnection], bool]]:
+    if isinstance(model, str):
+        model = context.resolve_value(model)
+
+    if isinstance(model, AirbyteConnectionSelectorByName):
+        return lambda connection: connection.name in model.by_name
+    elif isinstance(model, AirbyteConnectionSelectorById):
+        return lambda connection: connection.id in model.by_id
+    else:
+        check.failed(f"Unknown connection target type: {type(model)}")
+
+
+def resolve_airbyte_workspace_type(context: dg.ResolutionContext, model):
+    if isinstance(model, AirbyteWorkspaceModel):
+        return AirbyteWorkspace(**resolve_fields(model, AirbyteWorkspaceModel, context))
+    elif isinstance(model, AirbyteCloudWorkspaceModel):
+        return AirbyteCloudWorkspace(**resolve_fields(model, AirbyteCloudWorkspaceModel, context))
+    else:
+        check.failed(f"Unknown Airbyte workspace type: {type(model)}")
+
+
+@dg.scaffold_with(AirbyteWorkspaceComponentScaffolder)
+class AirbyteWorkspaceComponent(dg.Component, dg.Model, dg.Resolvable):
+    """Loads Airbyte connections from a given Airbyte workspace as Dagster assets.
+    Materializing these assets will trigger a sync of the Airbyte connection, enabling
+    you to schedule Airbyte syncs using Dagster.
+    """
+
+    workspace: Annotated[
+        Union[AirbyteWorkspace, AirbyteCloudWorkspace],
+        dg.Resolver(
+            resolve_airbyte_workspace_type,
+            model_field_type=Union[AirbyteWorkspaceModel, AirbyteCloudWorkspaceModel],
+        ),
+    ]
+    connection_selector: Annotated[
+        Optional[Callable[[AirbyteConnection], bool]],
+        dg.Resolver(
+            resolve_connection_selector,
+            model_field_type=Union[
+                str, AirbyteConnectionSelectorByName, AirbyteConnectionSelectorById
+            ],
+            description="Function used to select Airbyte connections to pull into Dagster.",
+        ),
+    ] = None
+    translation: Optional[
+        Annotated[
+            TranslationFn[AirbyteConnectionTableProps],
+            TranslationFnResolver(template_vars_for_translation_fn=lambda data: {"props": data}),
+        ]
+    ] = pydantic.Field(
+        None,
+        description="Function used to translate Airbyte connection table properties into Dagster asset specs.",
+    )
+
+    @cached_property
+    def translator(self) -> DagsterAirbyteTranslator:
+        if self.translation:
+            return ProxyDagsterAirbyteTranslator(self.translation)
+        return DagsterAirbyteTranslator()
+
+    def build_defs(self, context: dg.ComponentLoadContext) -> dg.Definitions:
+        airbyte_assets = build_airbyte_assets_definitions(
+            workspace=self.workspace,
+            dagster_airbyte_translator=self.translator,
+            connection_selector_fn=self.connection_selector,
+        )
+        assets_with_resource = [
+            airbyte_asset.with_resources(
+                {
+                    "airbyte": self.workspace.get_resource_definition(),
+                    "io_manager": default_job_io_manager,
+                }
+            )
+            for airbyte_asset in airbyte_assets
+        ]
+        return dg.Definitions(assets=assets_with_resource)
+
+
+# Subclassing to create the alias to be able to use the superseded decorator.
+@superseded(additional_warn_text="Superseded. Use AirbyteWorkspaceComponent instead.")
+class AirbyteCloudWorkspaceComponent(AirbyteWorkspaceComponent): ...

{dagster_airbyte-0.27.10 → dagster_airbyte-0.27.11}/dagster_airbyte/components/workspace_component/scaffolder.py

@@ -12,7 +12,7 @@ class AirbyteScaffolderParams(BaseModel):
     client_secret: Optional[str] = None
 
 
-class AirbyteCloudWorkspaceComponentScaffolder(Scaffolder[AirbyteScaffolderParams]):
+class AirbyteWorkspaceComponentScaffolder(Scaffolder[AirbyteScaffolderParams]):
     @classmethod
     def get_scaffold_params(cls) -> type[AirbyteScaffolderParams]:
         return AirbyteScaffolderParams