dagster-airbyte 0.23.7__py3-none-any.whl → 0.25.10__py3-none-any.whl

dagster_airbyte/resources.py

@@ -4,42 +4,71 @@ import logging
 import sys
 import time
 from abc import abstractmethod
+from collections.abc import Mapping, Sequence
 from contextlib import contextmanager
-from typing import Any, Dict, List, Mapping, Optional, cast
+from datetime import datetime, timedelta
+from typing import Any, Optional, cast
 
 import requests
 from dagster import (
+    AssetExecutionContext,
+    AssetMaterialization,
     ConfigurableResource,
+    Definitions,
     Failure,
+    InitResourceContext,
+    MaterializeResult,
     _check as check,
     get_dagster_logger,
     resource,
 )
+from dagster._annotations import experimental, public
 from dagster._config.pythonic_config import infer_schema_from_config_class
+from dagster._core.definitions.asset_spec import AssetSpec
+from dagster._core.definitions.definitions_load_context import StateBackedDefinitionsLoader
 from dagster._core.definitions.resource_definition import dagster_maintained_resource
+from dagster._model import DagsterModel
+from dagster._record import record
 from dagster._utils.cached_method import cached_method
 from dagster._utils.merger import deep_merge_dicts
-from pydantic import Field
+from pydantic import Field, PrivateAttr
 from requests.exceptions import RequestException
 
+from dagster_airbyte.translator import (
+    AirbyteConnection,
+    AirbyteConnectionTableProps,
+    AirbyteDestination,
+    AirbyteJob,
+    AirbyteJobStatusType,
+    AirbyteMetadataSet,
+    AirbyteWorkspaceData,
+    DagsterAirbyteTranslator,
+)
 from dagster_airbyte.types import AirbyteOutput
+from dagster_airbyte.utils import (
+    DAGSTER_AIRBYTE_TRANSLATOR_METADATA_KEY,
+    get_airbyte_connection_table_name,
+    get_translator_from_airbyte_assets,
+)
+
+AIRBYTE_REST_API_BASE = "https://api.airbyte.com"
+AIRBYTE_REST_API_VERSION = "v1"
+
+AIRBYTE_CONFIGURATION_API_BASE = "https://cloud.airbyte.com/api"
+AIRBYTE_CONFIGURATION_API_VERSION = "v1"
 
 DEFAULT_POLL_INTERVAL_SECONDS = 10
 
+# The access token expire every 3 minutes in Airbyte Cloud.
+# Refresh is needed after 2.5 minutes to avoid the "token expired" error message.
+AIRBYTE_CLOUD_REFRESH_TIMEDELTA_SECONDS = 150
 
-class AirbyteState:
-    RUNNING = "running"
-    SUCCEEDED = "succeeded"
-    CANCELLED = "cancelled"
-    PENDING = "pending"
-    FAILED = "failed"
-    ERROR = "error"
-    INCOMPLETE = "incomplete"
+AIRBYTE_RECONSTRUCTION_METADATA_KEY_PREFIX = "dagster-airbyte/reconstruction_metadata"
 
 
 class AirbyteResourceState:
     def __init__(self) -> None:
-        self.request_cache: Dict[str, Optional[Mapping[str, object]]] = {}
+        self.request_cache: dict[str, Optional[Mapping[str, object]]] = {}
         # Int in case we nest contexts
         self.cache_enabled = 0
 
@@ -94,7 +123,11 @@ class BaseAirbyteResource(ConfigurableResource):
         raise NotImplementedError()
 
     def make_request(
-        self, endpoint: str, data: Optional[Mapping[str, object]] = None, method: str = "POST"
+        self,
+        endpoint: str,
+        data: Optional[Mapping[str, object]] = None,
+        method: str = "POST",
+        include_additional_request_params: bool = True,
     ) -> Optional[Mapping[str, object]]:
         """Creates and sends a request to the desired Airbyte REST API endpoint.
 
@@ -111,7 +144,7 @@ class BaseAirbyteResource(ConfigurableResource):
         num_retries = 0
         while True:
             try:
-                request_args: Dict[str, Any] = dict(
+                request_args: dict[str, Any] = dict(
                     method=method,
                     url=url,
                     headers=headers,
@@ -120,10 +153,11 @@ class BaseAirbyteResource(ConfigurableResource):
                 if data:
                     request_args["json"] = data
 
-                request_args = deep_merge_dicts(
-                    request_args,
-                    self.all_additional_request_params,
-                )
+                if include_additional_request_params:
+                    request_args = deep_merge_dicts(
+                        request_args,
+                        self.all_additional_request_params,
+                    )
 
                 response = requests.request(
                     **request_args,
@@ -183,7 +217,7 @@ class BaseAirbyteResource(ConfigurableResource):
         """
         connection_details = self.get_connection_details(connection_id)
         job_details = self.start_sync(connection_id)
-        job_info = cast(Dict[str, object], job_details.get("job", {}))
+        job_info = cast(dict[str, object], job_details.get("job", {}))
         job_id = cast(int, job_info.get("id"))
 
         self._log.info(f"Job {job_id} initialized for connection_id={connection_id}.")
@@ -201,7 +235,7 @@ class BaseAirbyteResource(ConfigurableResource):
                     )
                 time.sleep(poll_interval or self.poll_interval)
                 job_details = self.get_job_status(connection_id, job_id)
-                attempts = cast(List, job_details.get("attempts", []))
+                attempts = cast(list, job_details.get("attempts", []))
                 cur_attempt = len(attempts)
                 # spit out the available Airbyte log info
                 if cur_attempt:
@@ -218,16 +252,20 @@ class BaseAirbyteResource(ConfigurableResource):
                         logged_lines = 0
                         logged_attempts += 1
 
-                job_info = cast(Dict[str, object], job_details.get("job", {}))
+                job_info = cast(dict[str, object], job_details.get("job", {}))
                 state = job_info.get("status")
 
-                if state in (AirbyteState.RUNNING, AirbyteState.PENDING, AirbyteState.INCOMPLETE):
+                if state in (
+                    AirbyteJobStatusType.RUNNING,
+                    AirbyteJobStatusType.PENDING,
+                    AirbyteJobStatusType.INCOMPLETE,
+                ):
                     continue
-                elif state == AirbyteState.SUCCEEDED:
+                elif state == AirbyteJobStatusType.SUCCEEDED:
                     break
-                elif state == AirbyteState.ERROR:
+                elif state == AirbyteJobStatusType.ERROR:
                     raise Failure(f"Job failed: {job_id}")
-                elif state == AirbyteState.CANCELLED:
+                elif state == AirbyteJobStatusType.CANCELLED:
                     raise Failure(f"Job was cancelled: {job_id}")
                 else:
                     raise Failure(f"Encountered unexpected state `{state}` for job_id {job_id}")
@@ -235,7 +273,12 @@ class BaseAirbyteResource(ConfigurableResource):
             # if Airbyte sync has not completed, make sure to cancel it so that it doesn't outlive
             # the python process
             if (
-                state not in (AirbyteState.SUCCEEDED, AirbyteState.ERROR, AirbyteState.CANCELLED)
+                state
+                not in (
+                    AirbyteJobStatusType.SUCCEEDED,
+                    AirbyteJobStatusType.ERROR,
+                    AirbyteJobStatusType.CANCELLED,
+                )
                 and self.cancel_sync_on_run_termination
             ):
                 self.cancel_job(job_id)
@@ -244,7 +287,7 @@ class BaseAirbyteResource(ConfigurableResource):
 
 
 class AirbyteCloudResource(BaseAirbyteResource):
-    """This resource allows users to programatically interface with the Airbyte Cloud API to launch
+    """This resource allows users to programmatically interface with the Airbyte Cloud API to launch
     syncs and monitor their progress.
 
     **Examples:**
@@ -255,7 +298,8 @@ class AirbyteCloudResource(BaseAirbyteResource):
         from dagster_airbyte import AirbyteResource
 
         my_airbyte_resource = AirbyteCloudResource(
-            api_key=EnvVar("AIRBYTE_API_KEY"),
+            client_id=EnvVar("AIRBYTE_CLIENT_ID"),
+            client_secret=EnvVar("AIRBYTE_CLIENT_SECRET"),
         )
 
         airbyte_assets = build_airbyte_assets(
@@ -269,7 +313,15 @@ class AirbyteCloudResource(BaseAirbyteResource):
         )
     """
 
-    api_key: str = Field(..., description="The Airbyte Cloud API key.")
+    client_id: str = Field(..., description="The Airbyte Cloud client ID.")
+    client_secret: str = Field(..., description="The Airbyte Cloud client secret.")
+
+    _access_token_value: Optional[str] = PrivateAttr(default=None)
+    _access_token_timestamp: Optional[float] = PrivateAttr(default=None)
+
+    def setup_for_execution(self, context: InitResourceContext) -> None:
+        # Refresh access token when the resource is initialized
+        self._refresh_access_token()
 
     @property
     def api_base_url(self) -> str:
@@ -277,7 +329,32 @@ class AirbyteCloudResource(BaseAirbyteResource):
 
     @property
     def all_additional_request_params(self) -> Mapping[str, Any]:
-        return {"headers": {"Authorization": f"Bearer {self.api_key}", "User-Agent": "dagster"}}
+        # Make sure the access token is refreshed before using it when calling the API.
+        if self._needs_refreshed_access_token():
+            self._refresh_access_token()
+        return {
+            "headers": {
+                "Authorization": f"Bearer {self._access_token_value}",
+                "User-Agent": "dagster",
+            }
+        }
+
+    def make_request(
+        self,
+        endpoint: str,
+        data: Optional[Mapping[str, object]] = None,
+        method: str = "POST",
+        include_additional_request_params: bool = True,
+    ) -> Optional[Mapping[str, object]]:
+        # Make sure the access token is refreshed before using it when calling the API.
+        if include_additional_request_params and self._needs_refreshed_access_token():
+            self._refresh_access_token()
+        return super().make_request(
+            endpoint=endpoint,
+            data=data,
+            method=method,
+            include_additional_request_params=include_additional_request_params,
+        )
 
     def start_sync(self, connection_id: str) -> Mapping[str, object]:
         job_sync = check.not_none(
@@ -306,6 +383,31 @@ class AirbyteCloudResource(BaseAirbyteResource):
         # Airbyte Cloud does not support streaming logs yet
         return False
 
+    def _refresh_access_token(self) -> None:
+        response = check.not_none(
+            self.make_request(
+                endpoint="/applications/token",
+                data={
+                    "client_id": self.client_id,
+                    "client_secret": self.client_secret,
+                },
+                # Must not pass the bearer access token when refreshing it.
+                include_additional_request_params=False,
+            )
+        )
+        self._access_token_value = str(response["access_token"])
+        self._access_token_timestamp = datetime.now().timestamp()
+
+    def _needs_refreshed_access_token(self) -> bool:
+        return (
+            not self._access_token_value
+            or not self._access_token_timestamp
+            or self._access_token_timestamp
+            <= datetime.timestamp(
+                datetime.now() - timedelta(seconds=AIRBYTE_CLOUD_REFRESH_TIMEDELTA_SECONDS)
+            )
+        )
+
 
 class AirbyteResource(BaseAirbyteResource):
     """This resource allows users to programatically interface with the Airbyte REST API to launch
@@ -469,7 +571,7 @@ class AirbyteResource(BaseAirbyteResource):
 
     def get_default_workspace(self) -> str:
         workspaces = cast(
-            List[Dict[str, Any]],
+            list[dict[str, Any]],
             check.not_none(self.make_request_cached(endpoint="/workspaces/list", data={})).get(
                 "workspaces", []
             ),
@@ -481,7 +583,7 @@ class AirbyteResource(BaseAirbyteResource):
         definitions = check.not_none(
             self.make_request_cached(endpoint="/source_definitions/list", data={})
         )
-        source_definitions = cast(List[Dict[str, Any]], definitions["sourceDefinitions"])
+        source_definitions = cast(list[dict[str, Any]], definitions["sourceDefinitions"])
 
         return next(
             (
@@ -495,7 +597,7 @@ class AirbyteResource(BaseAirbyteResource):
     def get_destination_definition_by_name(self, name: str):
         name_lower = name.lower()
         definitions = cast(
-            Dict[str, List[Dict[str, str]]],
+            dict[str, list[dict[str, str]]],
             check.not_none(
                 self.make_request_cached(endpoint="/destination_definitions/list", data={})
             ),
@@ -511,7 +613,7 @@ class AirbyteResource(BaseAirbyteResource):
 
     def get_source_catalog_id(self, source_id: str):
         result = cast(
-            Dict[str, Any],
+            dict[str, Any],
             check.not_none(
                 self.make_request(endpoint="/sources/discover_schema", data={"sourceId": source_id})
             ),
@@ -520,7 +622,7 @@ class AirbyteResource(BaseAirbyteResource):
 
     def get_source_schema(self, source_id: str) -> Mapping[str, Any]:
         return cast(
-            Dict[str, Any],
+            dict[str, Any],
             check.not_none(
                 self.make_request(endpoint="/sources/discover_schema", data={"sourceId": source_id})
             ),
@@ -532,7 +634,7 @@ class AirbyteResource(BaseAirbyteResource):
         # Airbyte API changed source of truth for normalization in PR
         # https://github.com/airbytehq/airbyte/pull/21005
         norm_dest_def_spec: bool = cast(
-            Dict[str, Any],
+            dict[str, Any],
             check.not_none(
                 self.make_request_cached(
                     endpoint="/destination_definition_specifications/get",
@@ -546,7 +648,7 @@ class AirbyteResource(BaseAirbyteResource):
 
         norm_dest_def: bool = (
             cast(
-                Dict[str, Any],
+                dict[str, Any],
                 check.not_none(
                     self.make_request_cached(
                         endpoint="/destination_definitions/get",
@@ -579,7 +681,7 @@ class AirbyteResource(BaseAirbyteResource):
                     },
                 )
             )
-            job = next((job for job in cast(List, out["jobs"]) if job["job"]["id"] == job_id), None)
+            job = next((job for job in cast(list, out["jobs"]) if job["job"]["id"] == job_id), None)
 
             return check.not_none(job)
 
@@ -614,7 +716,7 @@ class AirbyteResource(BaseAirbyteResource):
         """
         connection_details = self.get_connection_details(connection_id)
         job_details = self.start_sync(connection_id)
-        job_info = cast(Dict[str, object], job_details.get("job", {}))
+        job_info = cast(dict[str, object], job_details.get("job", {}))
         job_id = cast(int, job_info.get("id"))
 
         self._log.info(f"Job {job_id} initialized for connection_id={connection_id}.")
@@ -632,7 +734,7 @@ class AirbyteResource(BaseAirbyteResource):
                     )
                 time.sleep(poll_interval or self.poll_interval)
                 job_details = self.get_job_status(connection_id, job_id)
-                attempts = cast(List, job_details.get("attempts", []))
+                attempts = cast(list, job_details.get("attempts", []))
                 cur_attempt = len(attempts)
                 # spit out the available Airbyte log info
                 if cur_attempt:
@@ -649,16 +751,20 @@ class AirbyteResource(BaseAirbyteResource):
                         logged_lines = 0
                         logged_attempts += 1
 
-                job_info = cast(Dict[str, object], job_details.get("job", {}))
+                job_info = cast(dict[str, object], job_details.get("job", {}))
                 state = job_info.get("status")
 
-                if state in (AirbyteState.RUNNING, AirbyteState.PENDING, AirbyteState.INCOMPLETE):
+                if state in (
+                    AirbyteJobStatusType.RUNNING,
+                    AirbyteJobStatusType.PENDING,
+                    AirbyteJobStatusType.INCOMPLETE,
+                ):
                     continue
-                elif state == AirbyteState.SUCCEEDED:
+                elif state == AirbyteJobStatusType.SUCCEEDED:
                     break
-                elif state == AirbyteState.ERROR:
+                elif state == AirbyteJobStatusType.ERROR:
                     raise Failure(f"Job failed: {job_id}")
-                elif state == AirbyteState.CANCELLED:
+                elif state == AirbyteJobStatusType.CANCELLED:
                     raise Failure(f"Job was cancelled: {job_id}")
                 else:
                     raise Failure(f"Encountered unexpected state `{state}` for job_id {job_id}")
@@ -666,7 +772,12 @@ class AirbyteResource(BaseAirbyteResource):
             # if Airbyte sync has not completed, make sure to cancel it so that it doesn't outlive
             # the python process
             if (
-                state not in (AirbyteState.SUCCEEDED, AirbyteState.ERROR, AirbyteState.CANCELLED)
+                state
+                not in (
+                    AirbyteJobStatusType.SUCCEEDED,
+                    AirbyteJobStatusType.ERROR,
+                    AirbyteJobStatusType.CANCELLED,
+                )
                 and self.cancel_sync_on_run_termination
             ):
                 self.cancel_job(job_id)
@@ -721,3 +832,552 @@ def airbyte_cloud_resource(context) -> AirbyteCloudResource:
 
     """
     return AirbyteCloudResource.from_resource_context(context)
+
+
+# -------------
+# Resources v2
+# -------------
+
+
+@experimental
+class AirbyteCloudClient(DagsterModel):
+    """This class exposes methods on top of the Airbyte APIs for Airbyte Cloud."""
+
+    workspace_id: str = Field(..., description="The Airbyte workspace ID")
+    client_id: str = Field(..., description="The Airbyte client ID.")
+    client_secret: str = Field(..., description="The Airbyte client secret.")
+    request_max_retries: int = Field(
+        ...,
+        description=(
+            "The maximum number of times requests to the Airbyte API should be retried "
+            "before failing."
+        ),
+    )
+    request_retry_delay: float = Field(
+        ...,
+        description="Time (in seconds) to wait between each request retry.",
+    )
+    request_timeout: int = Field(
+        ...,
+        description="Time (in seconds) after which the requests to Airbyte are declared timed out.",
+    )
+
+    _access_token_value: Optional[str] = PrivateAttr(default=None)
+    _access_token_timestamp: Optional[float] = PrivateAttr(default=None)
+
+    @property
+    @cached_method
+    def _log(self) -> logging.Logger:
+        return get_dagster_logger()
+
+    @property
+    def rest_api_base_url(self) -> str:
+        return f"{AIRBYTE_REST_API_BASE}/{AIRBYTE_REST_API_VERSION}"
+
+    @property
+    def configuration_api_base_url(self) -> str:
+        return f"{AIRBYTE_CONFIGURATION_API_BASE}/{AIRBYTE_CONFIGURATION_API_VERSION}"
+
+    @property
+    def all_additional_request_params(self) -> Mapping[str, Any]:
+        return {**self.authorization_request_params, **self.user_agent_request_params}
+
+    @property
+    def authorization_request_params(self) -> Mapping[str, Any]:
+        # Make sure the access token is refreshed before using it when calling the API.
+        if self._needs_refreshed_access_token():
+            self._refresh_access_token()
+        return {
+            "Authorization": f"Bearer {self._access_token_value}",
+        }
+
+    @property
+    def user_agent_request_params(self) -> Mapping[str, Any]:
+        return {
+            "User-Agent": "dagster",
+        }
+
+    def _refresh_access_token(self) -> None:
+        response = check.not_none(
+            self._make_request(
+                method="POST",
+                endpoint="applications/token",
+                base_url=self.rest_api_base_url,
+                data={
+                    "client_id": self.client_id,
+                    "client_secret": self.client_secret,
+                },
+                # Must not pass the bearer access token when refreshing it.
+                include_additional_request_params=False,
+            )
+        )
+        self._access_token_value = str(response["access_token"])
+        self._access_token_timestamp = datetime.now().timestamp()
+
+    def _needs_refreshed_access_token(self) -> bool:
+        return (
+            not self._access_token_value
+            or not self._access_token_timestamp
+            or self._access_token_timestamp
+            <= (
+                datetime.now() - timedelta(seconds=AIRBYTE_CLOUD_REFRESH_TIMEDELTA_SECONDS)
+            ).timestamp()
+        )
+
+    def _get_session(self, include_additional_request_params: bool) -> requests.Session:
+        headers = {"accept": "application/json"}
+        if include_additional_request_params:
+            headers = {
+                **headers,
+                **self.all_additional_request_params,
+            }
+        session = requests.Session()
+        session.headers.update(headers)
+        return session
+
+    def _make_request(
+        self,
+        method: str,
+        endpoint: str,
+        base_url: str,
+        data: Optional[Mapping[str, Any]] = None,
+        params: Optional[Mapping[str, Any]] = None,
+        include_additional_request_params: bool = True,
+    ) -> Mapping[str, Any]:
+        """Creates and sends a request to the desired Airbyte REST API endpoint.
+
+        Args:
+            method (str): The http method to use for this request (e.g. "POST", "GET", "PATCH").
+            endpoint (str): The Airbyte API endpoint to send this request to.
+            base_url (str): The base url to the Airbyte API to use.
+            data (Optional[Dict[str, Any]]): JSON-formatted data string to be included in the request.
+            params (Optional[Dict[str, Any]]): JSON-formatted query params to be included in the request.
+            include_additional_request_params (bool): Whether to include authorization and user-agent headers
+            to the request parameters. Defaults to True.
+
+        Returns:
+            Dict[str, Any]: Parsed json data from the response to this request
+        """
+        url = f"{base_url}/{endpoint}"
+
+        num_retries = 0
+        while True:
+            try:
+                session = self._get_session(
+                    include_additional_request_params=include_additional_request_params
+                )
+                response = session.request(
+                    method=method, url=url, json=data, params=params, timeout=self.request_timeout
+                )
+                response.raise_for_status()
+                return response.json()
+            except RequestException as e:
+                self._log.error(
+                    f"Request to Airbyte API failed for url {url} with method {method} : {e}"
+                )
+                if num_retries == self.request_max_retries:
+                    break
+                num_retries += 1
+                time.sleep(self.request_retry_delay)
+
+        raise Failure(f"Max retries ({self.request_max_retries}) exceeded with url: {url}.")
+
+    def get_connections(self) -> Mapping[str, Any]:
+        """Fetches all connections of an Airbyte workspace from the Airbyte REST API."""
+        return self._make_request(
+            method="GET",
+            endpoint="connections",
+            base_url=self.rest_api_base_url,
+            params={"workspaceIds": self.workspace_id},
+        )
+
+    def get_connection_details(self, connection_id) -> Mapping[str, Any]:
+        """Fetches details about a given connection from the Airbyte Configuration API.
+        The Airbyte Configuration API is an internal and may change in the future.
+        """
+        # Using the Airbyte Configuration API to get the connection details, including streams and their configs.
+        # https://airbyte-public-api-docs.s3.us-east-2.amazonaws.com/rapidoc-api-docs.html#post-/v1/connections/get
+        # https://github.com/airbytehq/airbyte-platform/blob/v1.0.0/airbyte-api/server-api/src/main/openapi/config.yaml
+        return self._make_request(
+            method="POST",
+            endpoint="connections/get",
+            base_url=self.configuration_api_base_url,
+            data={"connectionId": connection_id},
+        )
+
+    def get_destination_details(self, destination_id: str) -> Mapping[str, Any]:
+        """Fetches details about a given destination from the Airbyte REST API."""
+        return self._make_request(
+            method="GET",
+            endpoint=f"destinations/{destination_id}",
+            base_url=self.rest_api_base_url,
+        )
+
+    def start_sync_job(self, connection_id: str) -> Mapping[str, Any]:
+        return self._make_request(
+            method="POST",
+            endpoint="jobs",
+            base_url=self.rest_api_base_url,
+            data={
+                "connectionId": connection_id,
+                "jobType": "sync",
+            },
+        )
+
+    def get_job_details(self, job_id: int) -> Mapping[str, Any]:
+        return self._make_request(
+            method="GET", endpoint=f"jobs/{job_id}", base_url=self.rest_api_base_url
+        )
+
+    def cancel_job(self, job_id: int) -> Mapping[str, Any]:
+        return self._make_request(
+            method="DELETE", endpoint=f"jobs/{job_id}", base_url=self.rest_api_base_url
+        )
+
+    def sync_and_poll(
+        self,
+        connection_id: str,
+        poll_interval: Optional[float] = None,
+        poll_timeout: Optional[float] = None,
+        cancel_on_termination: bool = True,
+    ) -> AirbyteOutput:
+        """Initializes a sync operation for the given connection, and polls until it completes.
+
+        Args:
+            connection_id (str): The Airbyte Connection ID. You can retrieve this value from the
+                "Connection" tab of a given connection in the Airbyte UI.
+            poll_interval (float): The time (in seconds) that will be waited between successive polls.
+            poll_timeout (float): The maximum time that will wait before this operation is timed
+                out. By default, this will never time out.
+            cancel_on_termination (bool): Whether to cancel a sync in Airbyte if the Dagster runner is terminated.
+                This may be useful to disable if using Airbyte sources that cannot be cancelled and
+                resumed easily, or if your Dagster deployment may experience runner interruptions
+                that do not impact your Airbyte deployment.
+
+        Returns:
+            :py:class:`~AirbyteOutput`:
+                Details of the sync job.
+        """
+        connection_details = self.get_connection_details(connection_id)
+        start_job_details = self.start_sync_job(connection_id)
+        job = AirbyteJob.from_job_details(job_details=start_job_details)
+
+        self._log.info(f"Job {job.id} initialized for connection_id={connection_id}.")
+        poll_start = datetime.now()
+        poll_interval = (
+            poll_interval if poll_interval is not None else DEFAULT_POLL_INTERVAL_SECONDS
+        )
+        try:
+            while True:
+                if poll_timeout and datetime.now() > poll_start + timedelta(seconds=poll_timeout):
+                    raise Failure(
+                        f"Timeout: Airbyte job {job.id} is not ready after the timeout"
+                        f" {poll_timeout} seconds"
+                    )
+
+                time.sleep(poll_interval)
+                # We return these job details in the AirbyteOutput when the job succeeds
+                poll_job_details = self.get_job_details(job.id)
+                job = AirbyteJob.from_job_details(job_details=poll_job_details)
+                if job.status in (
+                    AirbyteJobStatusType.RUNNING,
+                    AirbyteJobStatusType.PENDING,
+                    AirbyteJobStatusType.INCOMPLETE,
+                ):
+                    continue
+                elif job.status == AirbyteJobStatusType.SUCCEEDED:
+                    break
+                elif job.status in [AirbyteJobStatusType.ERROR, AirbyteJobStatusType.FAILED]:
+                    raise Failure(f"Job failed: {job.id}")
+                elif job.status == AirbyteJobStatusType.CANCELLED:
+                    raise Failure(f"Job was cancelled: {job.id}")
+                else:
+                    raise Failure(
+                        f"Encountered unexpected state `{job.status}` for job_id {job.id}"
+                    )
+        finally:
+            # if Airbyte sync has not completed, make sure to cancel it so that it doesn't outlive
+            # the python process
+            if cancel_on_termination and job.status not in (
+                AirbyteJobStatusType.SUCCEEDED,
+                AirbyteJobStatusType.ERROR,
+                AirbyteJobStatusType.CANCELLED,
+                AirbyteJobStatusType.FAILED,
+            ):
+                self.cancel_job(job.id)
+
+        return AirbyteOutput(job_details=poll_job_details, connection_details=connection_details)
+
+
+@experimental
+class AirbyteCloudWorkspace(ConfigurableResource):
+    """This class represents a Airbyte Cloud workspace and provides utilities
+    to interact with Airbyte APIs.
+    """
+
+    workspace_id: str = Field(..., description="The Airbyte Cloud workspace ID")
+    client_id: str = Field(..., description="The Airbyte Cloud client ID.")
+    client_secret: str = Field(..., description="The Airbyte Cloud client secret.")
+    request_max_retries: int = Field(
+        default=3,
+        description=(
+            "The maximum number of times requests to the Airbyte API should be retried "
+            "before failing."
+        ),
+    )
+    request_retry_delay: float = Field(
+        default=0.25,
+        description="Time (in seconds) to wait between each request retry.",
+    )
+    request_timeout: int = Field(
+        default=15,
+        description="Time (in seconds) after which the requests to Airbyte are declared timed out.",
+    )
+
+    _client: AirbyteCloudClient = PrivateAttr(default=None)  # type: ignore
+
+    @cached_method
+    def get_client(self) -> AirbyteCloudClient:
+        return AirbyteCloudClient(
+            workspace_id=self.workspace_id,
+            client_id=self.client_id,
+            client_secret=self.client_secret,
+            request_max_retries=self.request_max_retries,
+            request_retry_delay=self.request_retry_delay,
+            request_timeout=self.request_timeout,
+        )
+
+    def fetch_airbyte_workspace_data(
+        self,
+    ) -> AirbyteWorkspaceData:
+        """Retrieves all Airbyte content from the workspace and returns it as a AirbyteWorkspaceData object.
+
+        Returns:
+            AirbyteWorkspaceData: A snapshot of the Airbyte workspace's content.
+        """
+        connections_by_id = {}
+        destinations_by_id = {}
+
+        client = self.get_client()
+        connections = client.get_connections()["data"]
+
+        for partial_connection_details in connections:
+            full_connection_details = client.get_connection_details(
+                connection_id=partial_connection_details["connectionId"]
+            )
+            connection = AirbyteConnection.from_connection_details(
+                connection_details=full_connection_details
+            )
+            connections_by_id[connection.id] = connection
+
+            destination_details = client.get_destination_details(
+                destination_id=connection.destination_id
+            )
+            destination = AirbyteDestination.from_destination_details(
+                destination_details=destination_details
+            )
+            destinations_by_id[destination.id] = destination
+
+        return AirbyteWorkspaceData(
+            connections_by_id=connections_by_id,
+            destinations_by_id=destinations_by_id,
+        )
+
+    @cached_method
+    def load_asset_specs(
+        self,
+        dagster_airbyte_translator: Optional[DagsterAirbyteTranslator] = None,
+    ) -> Sequence[AssetSpec]:
+        """Returns a list of AssetSpecs representing the Airbyte content in the workspace.
+
+        Args:
+            dagster_airbyte_translator (Optional[DagsterAirbyteTranslator], optional): The translator to use
+                to convert Airbyte content into :py:class:`dagster.AssetSpec`.
+                Defaults to :py:class:`DagsterAirbyteTranslator`.
+
+        Returns:
+            List[AssetSpec]: The set of assets representing the Airbyte content in the workspace.
+
+        Examples:
+            Loading the asset specs for a given Airbyte workspace:
+            .. code-block:: python
+
+                from dagster_airbyte import AirbyteCloudWorkspace
+
+                import dagster as dg
+
+                airbyte_workspace = AirbyteCloudWorkspace(
+                    workspace_id=dg.EnvVar("AIRBYTE_CLOUD_WORKSPACE_ID"),
+                    client_id=dg.EnvVar("AIRBYTE_CLOUD_CLIENT_ID"),
+                    client_secret=dg.EnvVar("AIRBYTE_CLOUD_CLIENT_SECRET"),
+                )
+
+                airbyte_specs = airbyte_workspace.load_asset_specs()
+                defs = dg.Definitions(assets=airbyte_specs, resources={"airbyte": airbyte_workspace}
+        """
+        dagster_airbyte_translator = dagster_airbyte_translator or DagsterAirbyteTranslator()
+
+        return load_airbyte_cloud_asset_specs(
+            workspace=self, dagster_airbyte_translator=dagster_airbyte_translator
+        )
+
+    def _generate_materialization(
+        self,
+        airbyte_output: AirbyteOutput,
+        dagster_airbyte_translator: DagsterAirbyteTranslator,
+    ):
+        connection = AirbyteConnection.from_connection_details(
+            connection_details=airbyte_output.connection_details
+        )
+
+        for stream in connection.streams.values():
+            if stream.selected:
+                connection_table_name = get_airbyte_connection_table_name(
+                    stream_prefix=connection.stream_prefix,
+                    stream_name=stream.name,
+                )
+                stream_asset_spec = dagster_airbyte_translator.get_asset_spec(
+                    props=AirbyteConnectionTableProps(
+                        table_name=connection_table_name,
+                        stream_prefix=connection.stream_prefix,
+                        stream_name=stream.name,
+                        json_schema=stream.json_schema,
+                        connection_id=connection.id,
+                        connection_name=connection.name,
+                        destination_type=None,
+                        database=None,
+                        schema=None,
+                    )
+                )
+
+                yield AssetMaterialization(
+                    asset_key=stream_asset_spec.key,
+                    description=(
+                        f"Table generated via Airbyte Cloud sync "
+                        f"for connection {connection.name}: {connection_table_name}"
+                    ),
+                    metadata=stream_asset_spec.metadata,
+                )
+
+    @public
+    @experimental
+    def sync_and_poll(self, context: AssetExecutionContext):
+        """Executes a sync and poll process to materialize Airbyte Cloud assets.
+            This method can only be used in the context of an asset execution.
+
+        Args:
+            context (AssetExecutionContext): The execution context
+                from within `@airbyte_assets`.
+
+        Returns:
+            Iterator[Union[AssetMaterialization, MaterializeResult]]: An iterator of MaterializeResult
+                or AssetMaterialization.
+        """
+        assets_def = context.assets_def
+        dagster_airbyte_translator = get_translator_from_airbyte_assets(assets_def)
+        connection_id = next(
+            check.not_none(AirbyteMetadataSet.extract(spec.metadata).connection_id)
+            for spec in assets_def.specs
+        )
+
+        client = self.get_client()
+        airbyte_output = client.sync_and_poll(
+            connection_id=connection_id,
+        )
+
+        materialized_asset_keys = set()
+        for materialization in self._generate_materialization(
+            airbyte_output=airbyte_output, dagster_airbyte_translator=dagster_airbyte_translator
+        ):
+            # Scan through all tables actually created, if it was expected then emit a MaterializeResult.
+            # Otherwise, emit a runtime AssetMaterialization.
+            if materialization.asset_key in context.selected_asset_keys:
+                yield MaterializeResult(
+                    asset_key=materialization.asset_key, metadata=materialization.metadata
+                )
+                materialized_asset_keys.add(materialization.asset_key)
+            else:
+                context.log.warning(
+                    f"An unexpected asset was materialized: {materialization.asset_key}. "
+                    f"Yielding a materialization event."
+                )
+                yield materialization
+
+        unmaterialized_asset_keys = context.selected_asset_keys - materialized_asset_keys
+        if unmaterialized_asset_keys:
+            context.log.warning(f"Assets were not materialized: {unmaterialized_asset_keys}")
+
+
+@experimental
+def load_airbyte_cloud_asset_specs(
+    workspace: AirbyteCloudWorkspace,
+    dagster_airbyte_translator: Optional[DagsterAirbyteTranslator] = None,
+) -> Sequence[AssetSpec]:
+    """Returns a list of AssetSpecs representing the Airbyte content in the workspace.
+
+    Args:
+        workspace (AirbyteCloudWorkspace): The Airbyte Cloud workspace to fetch assets from.
+        dagster_airbyte_translator (Optional[DagsterAirbyteTranslator], optional): The translator to use
+            to convert Airbyte content into :py:class:`dagster.AssetSpec`.
+            Defaults to :py:class:`DagsterAirbyteTranslator`.
+
+    Returns:
+        List[AssetSpec]: The set of assets representing the Airbyte content in the workspace.
+
+    Examples:
+        Loading the asset specs for a given Airbyte Cloud workspace:
+
+        .. code-block:: python
+
+            from dagster_airbyte import AirbyteCloudWorkspace, load_airbyte_cloud_asset_specs
+
+            import dagster as dg
+
+            airbyte_cloud_workspace = AirbyteCloudWorkspace(
+                workspace_id=dg.EnvVar("AIRBYTE_CLOUD_WORKSPACE_ID"),
+                client_id=dg.EnvVar("AIRBYTE_CLOUD_CLIENT_ID"),
+                client_secret=dg.EnvVar("AIRBYTE_CLOUD_CLIENT_SECRET"),
+            )
+
+
+            airbyte_cloud_specs = load_airbyte_cloud_asset_specs(airbyte_cloud_workspace)
+            defs = dg.Definitions(assets=airbyte_cloud_specs)
+    """
+    dagster_airbyte_translator = dagster_airbyte_translator or DagsterAirbyteTranslator()
+
+    with workspace.process_config_and_initialize_cm() as initialized_workspace:
+        return [
+            spec.merge_attributes(
+                metadata={DAGSTER_AIRBYTE_TRANSLATOR_METADATA_KEY: dagster_airbyte_translator}
+            )
+            for spec in check.is_list(
+                AirbyteCloudWorkspaceDefsLoader(
+                    workspace=initialized_workspace,
+                    translator=dagster_airbyte_translator,
+                )
+                .build_defs()
+                .assets,
+                AssetSpec,
+            )
+        ]
+
+
+@record
+class AirbyteCloudWorkspaceDefsLoader(StateBackedDefinitionsLoader[Mapping[str, Any]]):
+    workspace: AirbyteCloudWorkspace
+    translator: DagsterAirbyteTranslator
+
+    @property
+    def defs_key(self) -> str:
+        return f"{AIRBYTE_RECONSTRUCTION_METADATA_KEY_PREFIX}/{self.workspace.workspace_id}"
+
+    def fetch_state(self) -> AirbyteWorkspaceData:
+        return self.workspace.fetch_airbyte_workspace_data()
+
+    def defs_from_state(self, state: AirbyteWorkspaceData) -> Definitions:
+        all_asset_specs = [
+            self.translator.get_asset_spec(props)
+            for props in state.to_airbyte_connection_table_props_data()
+        ]
+
+        return Definitions(assets=all_asset_specs)