dagster-airbyte 0.23.7__py3-none-any.whl → 0.25.10__py3-none-any.whl

dagster_airbyte/asset_defs.py

@@ -1,28 +1,15 @@
 import hashlib
 import inspect
 import os
-import re
 from abc import abstractmethod
+from collections.abc import Iterable, Mapping, Sequence
 from functools import partial
 from itertools import chain
-from typing import (
-    Any,
-    Callable,
-    Dict,
-    Iterable,
-    List,
-    Mapping,
-    NamedTuple,
-    Optional,
-    Sequence,
-    Set,
-    Tuple,
-    Union,
-    cast,
-)
+from typing import Any, Callable, NamedTuple, Optional, Union, cast
 
 import yaml
 from dagster import (
+    AssetExecutionContext,
     AssetKey,
     AssetOut,
     AutoMaterializePolicy,
@@ -33,21 +20,30 @@ from dagster import (
     SourceAsset,
     _check as check,
 )
+from dagster._annotations import experimental
 from dagster._core.definitions import AssetsDefinition, multi_asset
 from dagster._core.definitions.cacheable_assets import (
     AssetsDefinitionCacheableData,
     CacheableAssetsDefinition,
 )
 from dagster._core.definitions.events import CoercibleToAssetKey, CoercibleToAssetKeyPrefix
-from dagster._core.definitions.metadata import MetadataValue, TableSchemaMetadataValue
+from dagster._core.definitions.metadata.metadata_set import TableMetadataSet
 from dagster._core.definitions.metadata.table import TableSchema
 from dagster._core.errors import DagsterInvalidDefinitionError, DagsterInvalidInvocationError
 from dagster._core.execution.context.init import build_init_resource_context
 from dagster._utils.merger import merge_dicts
 
-from dagster_airbyte.resources import AirbyteCloudResource, AirbyteResource, BaseAirbyteResource
+from dagster_airbyte.asset_decorator import airbyte_assets
+from dagster_airbyte.resources import (
+    AirbyteCloudResource,
+    AirbyteCloudWorkspace,
+    AirbyteResource,
+    BaseAirbyteResource,
+)
+from dagster_airbyte.translator import AirbyteMetadataSet, DagsterAirbyteTranslator
 from dagster_airbyte.types import AirbyteTableMetadata
 from dagster_airbyte.utils import (
+    clean_name,
     generate_materializations,
     generate_table_schema,
     is_basic_normalization_operation,
@@ -61,9 +57,13 @@ def _table_to_output_name_fn(table: str) -> str:
 def _build_airbyte_asset_defn_metadata(
     connection_id: str,
     destination_tables: Sequence[str],
+    destination_raw_table_names_by_table: Mapping[str, str],
+    destination_database: Optional[str],
+    destination_schema: Optional[str],
     table_to_asset_key_fn: Callable[[str], AssetKey],
     asset_key_prefix: Optional[Sequence[str]] = None,
-    normalization_tables: Optional[Mapping[str, Set[str]]] = None,
+    normalization_tables: Optional[Mapping[str, set[str]]] = None,
+    normalization_raw_table_names_by_table: Optional[Mapping[str, str]] = None,
     upstream_assets: Optional[Iterable[AssetKey]] = None,
     group_name: Optional[str] = None,
     io_manager_key: Optional[str] = None,
@@ -92,7 +92,7 @@ def _build_airbyte_asset_defn_metadata(
         for table in tables
     }
 
-    internal_deps: Dict[str, Set[AssetKey]] = {}
+    internal_deps: dict[str, set[AssetKey]] = {}
 
     metadata_encodable_normalization_tables = (
         {k: list(v) for k, v in normalization_tables.items()} if normalization_tables else {}
@@ -111,6 +111,30 @@ def _build_airbyte_asset_defn_metadata(
     for table in destination_tables:
         internal_deps[table] = set(upstream_assets or [])
 
+    table_names: dict[str, str] = {}
+    for table in destination_tables:
+        if destination_database and destination_schema and table:
+            # Use the destination raw table name to create the table name
+            table_names[table] = ".".join(
+                [
+                    destination_database,
+                    destination_schema,
+                    destination_raw_table_names_by_table[table],
+                ]
+            )
+            if normalization_tables and normalization_raw_table_names_by_table:
+                for normalization_table in normalization_tables.get(table, set()):
+                    table_names[normalization_table] = ".".join(
+                        [
+                            destination_database,
+                            destination_schema,
+                            destination_raw_table_names_by_table[table],
+                            normalization_raw_table_names_by_table[normalization_table],
+                        ]
+                    )
+
+    schema_by_table_name = schema_by_table_name if schema_by_table_name else {}
+
     return AssetsDefinitionCacheableData(
         keys_by_input_name=(
             {asset_key.path[-1]: asset_key for asset_key in upstream_assets}
@@ -124,11 +148,14 @@ def _build_airbyte_asset_defn_metadata(
         can_subset=False,
         metadata_by_output_name=(
             {
-                table: {"table_schema": MetadataValue.table_schema(schema_by_table_name[table])}
+                table: {
+                    **TableMetadataSet(
+                        column_schema=schema_by_table_name.get(table),
+                        table_name=table_names.get(table),
+                    ),
+                }
                 for table in tables
             }
-            if schema_by_table_name
-            else None
         ),
         freshness_policies_by_output_name=(
             {output: freshness_policy for output in outputs} if freshness_policy else None
@@ -155,21 +182,18 @@ def _build_airbyte_assets_from_metadata(
     metadata = cast(Mapping[str, Any], assets_defn_meta.extra_metadata)
     connection_id = cast(str, metadata["connection_id"])
     group_name = cast(Optional[str], metadata["group_name"])
-    destination_tables = cast(List[str], metadata["destination_tables"])
-    normalization_tables = cast(Mapping[str, List[str]], metadata["normalization_tables"])
+    destination_tables = cast(list[str], metadata["destination_tables"])
+    normalization_tables = cast(Mapping[str, list[str]], metadata["normalization_tables"])
     io_manager_key = cast(Optional[str], metadata["io_manager_key"])
 
     @multi_asset(
-        name=f"airbyte_sync_{connection_id[:5]}",
+        name=f"airbyte_sync_{connection_id.replace('-', '_')}",
         deps=list((assets_defn_meta.keys_by_input_name or {}).values()),
         outs={
             k: AssetOut(
                 key=v,
                 metadata=(
-                    {
-                        k: cast(TableSchemaMetadataValue, v)
-                        for k, v in assets_defn_meta.metadata_by_output_name.get(k, {}).items()
-                    }
+                    assets_defn_meta.metadata_by_output_name.get(k)
                     if assets_defn_meta.metadata_by_output_name
                     else None
                 ),
@@ -224,11 +248,13 @@ def _build_airbyte_assets_from_metadata(
 def build_airbyte_assets(
     connection_id: str,
     destination_tables: Sequence[str],
+    destination_database: Optional[str] = None,
+    destination_schema: Optional[str] = None,
     asset_key_prefix: Optional[Sequence[str]] = None,
     group_name: Optional[str] = None,
-    normalization_tables: Optional[Mapping[str, Set[str]]] = None,
+    normalization_tables: Optional[Mapping[str, set[str]]] = None,
     deps: Optional[Iterable[Union[CoercibleToAssetKey, AssetsDefinition, SourceAsset]]] = None,
-    upstream_assets: Optional[Set[AssetKey]] = None,
+    upstream_assets: Optional[set[AssetKey]] = None,
     schema_by_table_name: Optional[Mapping[str, TableSchema]] = None,
     freshness_policy: Optional[FreshnessPolicy] = None,
     stream_to_asset_map: Optional[Mapping[str, str]] = None,
@@ -242,6 +268,8 @@ def build_airbyte_assets(
         destination_tables (List[str]): The names of the tables that you want to be represented
             in the Dagster asset graph for this sync. This will generally map to the name of the
             stream in Airbyte, unless a stream prefix has been specified in Airbyte.
+        destination_database (Optional[str]): The name of the destination database.
+        destination_schema (Optional[str]): The name of the destination schema.
         normalization_tables (Optional[Mapping[str, List[str]]]): If you are using Airbyte's
             normalization feature, you may specify a mapping of destination table to a list of
             derived tables that will be created by the normalization process.
@@ -268,13 +296,34 @@ def build_airbyte_assets(
     tables = chain.from_iterable(
         chain([destination_tables], normalization_tables.values() if normalization_tables else [])
     )
+
+    table_names: dict[str, str] = {}
+    for table in destination_tables:
+        if destination_database and destination_schema and table:
+            table_names[table] = ".".join([destination_database, destination_schema, table])
+            if normalization_tables:
+                for normalization_table in normalization_tables.get(table, set()):
+                    table_names[normalization_table] = ".".join(
+                        [
+                            destination_database,
+                            destination_schema,
+                            table,
+                            normalization_table,
+                        ]
+                    )
+
+    schema_by_table_name = schema_by_table_name if schema_by_table_name else {}
+
     outputs = {
         table: AssetOut(
             key=AssetKey([*asset_key_prefix, table]),
             metadata=(
-                {"table_schema": MetadataValue.table_schema(schema_by_table_name[table])}
-                if schema_by_table_name
-                else None
+                {
+                    **TableMetadataSet(
+                        column_schema=schema_by_table_name.get(table),
+                        table_name=table_names.get(table),
+                    ),
+                }
             ),
             freshness_policy=freshness_policy,
             auto_materialize_policy=auto_materialize_policy,
@@ -300,7 +349,7 @@ def build_airbyte_assets(
         internal_deps[table] = set(upstream_deps) if upstream_deps else set()
 
     @multi_asset(
-        name=f"airbyte_sync_{connection_id[:5]}",
+        name=f"airbyte_sync_{connection_id.replace('-', '_')}",
         deps=upstream_deps,
         outs=outputs,
         internal_asset_deps=internal_deps,
@@ -376,7 +425,7 @@ def _get_normalization_tables_for_schema(
     For more information on Airbyte's normalization process, see:
     https://docs.airbyte.com/understanding-airbyte/basic-normalization/#nesting
     """
-    out: Dict[str, AirbyteTableMetadata] = {}
+    out: dict[str, AirbyteTableMetadata] = {}
     # Object types are broken into a new table, as long as they have children
 
     sub_schemas = _get_sub_schemas(schema)
@@ -388,7 +437,7 @@ def _get_normalization_tables_for_schema(
 
         if "object" in schema_types and len(sub_schema.get("properties", {})) > 0:
             out[prefix + key] = AirbyteTableMetadata(
-                schema=generate_table_schema(sub_schema.get("properties", {}))
+                raw_table_name=key, schema=generate_table_schema(sub_schema.get("properties", {}))
             )
             for k, v in sub_schema["properties"].items():
                 out = merge_dicts(
@@ -397,7 +446,8 @@ def _get_normalization_tables_for_schema(
         # Array types are also broken into a new table
         elif "array" in schema_types:
             out[prefix + key] = AirbyteTableMetadata(
-                schema=generate_table_schema(sub_schema.get("items", {}).get("properties", {}))
+                raw_table_name=key,
+                schema=generate_table_schema(sub_schema.get("items", {}).get("properties", {})),
             )
             if sub_schema.get("items", {}).get("properties"):
                 for k, v in sub_schema["items"]["properties"].items():
@@ -408,11 +458,6 @@ def _get_normalization_tables_for_schema(
     return out
 
 
-def _clean_name(name: str) -> str:
-    """Cleans an input to be a valid Dagster asset name."""
-    return re.sub(r"[^a-z0-9]+", "_", name.lower())
-
-
 class AirbyteConnectionMetadata(
     NamedTuple(
         "_AirbyteConnectionMetadata",
@@ -420,7 +465,8 @@ class AirbyteConnectionMetadata(
             ("name", str),
             ("stream_prefix", str),
             ("has_basic_normalization", bool),
-            ("stream_data", List[Mapping[str, Any]]),
+            ("stream_data", list[Mapping[str, Any]]),
+            ("destination", Mapping[str, Any]),
         ],
     )
 ):
@@ -435,7 +481,10 @@ class AirbyteConnectionMetadata(
 
     @classmethod
     def from_api_json(
-        cls, contents: Mapping[str, Any], operations: Mapping[str, Any]
+        cls,
+        contents: Mapping[str, Any],
+        operations: Mapping[str, Any],
+        destination: Mapping[str, Any],
     ) -> "AirbyteConnectionMetadata":
         return cls(
             name=contents["name"],
@@ -445,10 +494,13 @@ class AirbyteConnectionMetadata(
                 for op in operations.get("operations", [])
             ),
             stream_data=contents.get("syncCatalog", {}).get("streams", []),
+            destination=destination,
         )
 
     @classmethod
-    def from_config(cls, contents: Mapping[str, Any]) -> "AirbyteConnectionMetadata":
+    def from_config(
+        cls, contents: Mapping[str, Any], destination: Mapping[str, Any]
+    ) -> "AirbyteConnectionMetadata":
         config_contents = cast(Mapping[str, Any], contents.get("configuration"))
         check.invariant(
             config_contents is not None, "Airbyte connection config is missing 'configuration' key"
@@ -462,6 +514,7 @@ class AirbyteConnectionMetadata(
                 for op in config_contents.get("operations", [])
             ),
             stream_data=config_contents.get("sync_catalog", {}).get("streams", []),
+            destination=destination,
         )
 
     def parse_stream_tables(
@@ -471,7 +524,7 @@ class AirbyteConnectionMetadata(
         tables associated with each enabled stream and values representing any affiliated
         tables created by Airbyte's normalization process, if enabled.
         """
-        tables: Dict[str, AirbyteTableMetadata] = {}
+        tables: dict[str, AirbyteTableMetadata] = {}
 
         enabled_streams = [
             stream for stream in self.stream_data if stream.get("config", {}).get("selected", False)
@@ -486,7 +539,7 @@ class AirbyteConnectionMetadata(
                 if "json_schema" in stream["stream"]
                 else stream["stream"]["jsonSchema"]
             )
-            normalization_tables: Dict[str, AirbyteTableMetadata] = {}
+            normalization_tables: dict[str, AirbyteTableMetadata] = {}
             schema_props = schema.get("properties", schema.get("items", {}).get("properties", {}))
             if self.has_basic_normalization and return_normalization_tables:
                 for k, v in schema_props.items():
@@ -496,6 +549,7 @@ class AirbyteConnectionMetadata(
                         prefixed_norm_table_name = f"{self.stream_prefix}{normalization_table_name}"
                         normalization_tables[prefixed_norm_table_name] = meta
             tables[prefixed_name] = AirbyteTableMetadata(
+                raw_table_name=name,
                 schema=generate_table_schema(schema_props),
                 normalization_tables=normalization_tables,
             )
@@ -513,7 +567,7 @@ def _get_schema_by_table_name(
                 [
                     (k, v.schema)
                     for k, v in cast(
-                        Dict[str, AirbyteTableMetadata], meta.normalization_tables
+                        dict[str, AirbyteTableMetadata], meta.normalization_tables
                     ).items()
                 ]
                 for meta in stream_table_metadata.values()
@@ -564,25 +618,46 @@ class AirbyteCoreCacheableAssetsDefinition(CacheableAssetsDefinition):
         super().__init__(unique_id=f"airbyte-{contents.hexdigest()}")
 
     @abstractmethod
-    def _get_connections(self) -> Sequence[Tuple[str, AirbyteConnectionMetadata]]:
+    def _get_connections(self) -> Sequence[tuple[str, AirbyteConnectionMetadata]]:
         pass
 
     def compute_cacheable_data(self) -> Sequence[AssetsDefinitionCacheableData]:
-        asset_defn_data: List[AssetsDefinitionCacheableData] = []
+        asset_defn_data: list[AssetsDefinitionCacheableData] = []
         for connection_id, connection in self._get_connections():
             stream_table_metadata = connection.parse_stream_tables(
                 self._create_assets_for_normalization_tables
             )
             schema_by_table_name = _get_schema_by_table_name(stream_table_metadata)
 
+            destination_database = connection.destination.get("configuration", {}).get("database")
+            destination_schema = connection.destination.get("configuration", {}).get("schema")
+
             table_to_asset_key = partial(self._connection_to_asset_key_fn, connection)
+
+            destination_tables = list(stream_table_metadata.keys())
+            destination_raw_table_names_by_table = {
+                table: metadata.raw_table_name for table, metadata in stream_table_metadata.items()
+            }
+            normalization_tables = {
+                table: set(metadata.normalization_tables.keys())
+                for table, metadata in stream_table_metadata.items()
+            }
+            normalization_raw_table_names_by_table = {
+                normalization_table: metadata.normalization_tables[
+                    normalization_table
+                ].raw_table_name
+                for table, metadata in stream_table_metadata.items()
+                for normalization_table in normalization_tables[table]
+            }
+
             asset_data_for_conn = _build_airbyte_asset_defn_metadata(
                 connection_id=connection_id,
-                destination_tables=list(stream_table_metadata.keys()),
-                normalization_tables={
-                    table: set(metadata.normalization_tables.keys())
-                    for table, metadata in stream_table_metadata.items()
-                },
+                destination_tables=destination_tables,
+                destination_raw_table_names_by_table=destination_raw_table_names_by_table,
+                destination_database=destination_database,
+                destination_schema=destination_schema,
+                normalization_tables=normalization_tables,
+                normalization_raw_table_names_by_table=normalization_raw_table_names_by_table,
                 asset_key_prefix=self._key_prefix,
                 group_name=(
                     self._connection_meta_to_group_fn(connection)
@@ -661,11 +736,11 @@ class AirbyteInstanceCacheableAssetsDefinition(AirbyteCoreCacheableAssetsDefinit
             )
             self._airbyte_instance: AirbyteResource = self._partially_initialized_airbyte_instance
 
-    def _get_connections(self) -> Sequence[Tuple[str, AirbyteConnectionMetadata]]:
+    def _get_connections(self) -> Sequence[tuple[str, AirbyteConnectionMetadata]]:
         workspace_id = self._workspace_id
         if not workspace_id:
             workspaces = cast(
-                List[Dict[str, Any]],
+                list[dict[str, Any]],
                 check.not_none(
                     self._airbyte_instance.make_request(endpoint="/workspaces/list", data={})
                 ).get("workspaces", []),
@@ -677,7 +752,7 @@ class AirbyteInstanceCacheableAssetsDefinition(AirbyteCoreCacheableAssetsDefinit
             workspace_id = workspaces[0].get("workspaceId")
 
         connections = cast(
-            List[Dict[str, Any]],
+            list[dict[str, Any]],
             check.not_none(
                 self._airbyte_instance.make_request(
                     endpoint="/connections/list", data={"workspaceId": workspace_id}
@@ -685,12 +760,12 @@ class AirbyteInstanceCacheableAssetsDefinition(AirbyteCoreCacheableAssetsDefinit
             ).get("connections", []),
         )
 
-        output_connections: List[Tuple[str, AirbyteConnectionMetadata]] = []
+        output_connections: list[tuple[str, AirbyteConnectionMetadata]] = []
         for connection_json in connections:
             connection_id = cast(str, connection_json.get("connectionId"))
 
             operations_json = cast(
-                Dict[str, Any],
+                dict[str, Any],
                 check.not_none(
                     self._airbyte_instance.make_request(
                         endpoint="/operations/list",
@@ -698,7 +773,21 @@ class AirbyteInstanceCacheableAssetsDefinition(AirbyteCoreCacheableAssetsDefinit
                     )
                 ),
             )
-            connection = AirbyteConnectionMetadata.from_api_json(connection_json, operations_json)
+
+            destination_id = cast(str, connection_json.get("destinationId"))
+            destination_json = cast(
+                dict[str, Any],
+                check.not_none(
+                    self._airbyte_instance.make_request(
+                        endpoint="/destinations/get",
+                        data={"destinationId": destination_id},
+                    )
+                ),
+            )
+
+            connection = AirbyteConnectionMetadata.from_api_json(
+                connection_json, operations_json, destination_json
+            )
 
             # Filter out connections that don't match the filter function
             if self._connection_filter and not self._connection_filter(connection):
@@ -749,16 +838,26 @@ class AirbyteYAMLCacheableAssetsDefinition(AirbyteCoreCacheableAssetsDefinition)
         self._project_dir = project_dir
         self._connection_directories = connection_directories
 
-    def _get_connections(self) -> Sequence[Tuple[str, AirbyteConnectionMetadata]]:
+    def _get_connections(self) -> Sequence[tuple[str, AirbyteConnectionMetadata]]:
         connections_dir = os.path.join(self._project_dir, "connections")
 
-        output_connections: List[Tuple[str, AirbyteConnectionMetadata]] = []
+        output_connections: list[tuple[str, AirbyteConnectionMetadata]] = []
 
         connection_directories = self._connection_directories or os.listdir(connections_dir)
         for connection_name in connection_directories:
             connection_dir = os.path.join(connections_dir, connection_name)
             with open(os.path.join(connection_dir, "configuration.yaml"), encoding="utf-8") as f:
-                connection = AirbyteConnectionMetadata.from_config(yaml.safe_load(f.read()))
+                connection_data = yaml.safe_load(f.read())
+
+            destination_configuration_path = cast(
+                str, connection_data.get("destination_configuration_path")
+            )
+            with open(
+                os.path.join(self._project_dir, destination_configuration_path), encoding="utf-8"
+            ) as f:
+                destination_data = yaml.safe_load(f.read())
+
+            connection = AirbyteConnectionMetadata.from_config(connection_data, destination_data)
 
             # Filter out connections that don't match the filter function
             if self._connection_filter and not self._connection_filter(connection):
@@ -800,7 +899,7 @@ def load_assets_from_airbyte_instance(
     workspace_id: Optional[str] = None,
     key_prefix: Optional[CoercibleToAssetKeyPrefix] = None,
     create_assets_for_normalization_tables: bool = True,
-    connection_to_group_fn: Optional[Callable[[str], Optional[str]]] = _clean_name,
+    connection_to_group_fn: Optional[Callable[[str], Optional[str]]] = clean_name,
     connection_meta_to_group_fn: Optional[
         Callable[[AirbyteConnectionMetadata], Optional[str]]
     ] = None,
@@ -905,7 +1004,7 @@ def load_assets_from_airbyte_instance(
     check.invariant(
         not connection_meta_to_group_fn
         or not connection_to_group_fn
-        or connection_to_group_fn == _clean_name,
+        or connection_to_group_fn == clean_name,
         "Cannot specify both connection_meta_to_group_fn and connection_to_group_fn",
     )
 
@@ -926,123 +1025,115 @@ def load_assets_from_airbyte_instance(
     )
 
 
-def load_assets_from_airbyte_project(
-    project_dir: str,
-    workspace_id: Optional[str] = None,
-    key_prefix: Optional[CoercibleToAssetKeyPrefix] = None,
-    create_assets_for_normalization_tables: bool = True,
-    connection_to_group_fn: Optional[Callable[[str], Optional[str]]] = _clean_name,
-    connection_meta_to_group_fn: Optional[
-        Callable[[AirbyteConnectionMetadata], Optional[str]]
-    ] = None,
-    io_manager_key: Optional[str] = None,
-    connection_to_io_manager_key_fn: Optional[Callable[[str], Optional[str]]] = None,
-    connection_filter: Optional[Callable[[AirbyteConnectionMetadata], bool]] = None,
-    connection_directories: Optional[Sequence[str]] = None,
-    connection_to_asset_key_fn: Optional[
-        Callable[[AirbyteConnectionMetadata, str], AssetKey]
-    ] = None,
-    connection_to_freshness_policy_fn: Optional[
-        Callable[[AirbyteConnectionMetadata], Optional[FreshnessPolicy]]
-    ] = None,
-    connection_to_auto_materialize_policy_fn: Optional[
-        Callable[[AirbyteConnectionMetadata], Optional[AutoMaterializePolicy]]
-    ] = None,
-) -> CacheableAssetsDefinition:
-    """Loads an Airbyte project into a set of Dagster assets.
+# -----------------------
+# Reworked assets factory
+# -----------------------
 
-    Point to the root folder of an Airbyte project synced using the Octavia CLI. For
-    more information, see https://github.com/airbytehq/airbyte/tree/master/octavia-cli#octavia-import-all.
+
+@experimental
+def build_airbyte_assets_definitions(
+    *,
+    workspace: AirbyteCloudWorkspace,
+    dagster_airbyte_translator: Optional[DagsterAirbyteTranslator] = None,
+) -> Sequence[AssetsDefinition]:
+    """The list of AssetsDefinition for all connections in the Airbyte workspace.
 
     Args:
-        project_dir (str): The path to the root of your Airbyte project, containing sources, destinations,
-            and connections folders.
-        workspace_id (Optional[str]): The ID of the Airbyte workspace to load connections from. Only
-            required if multiple workspace state YAMLfiles exist in the project.
-        key_prefix (Optional[CoercibleToAssetKeyPrefix]): A prefix for the asset keys created.
-        create_assets_for_normalization_tables (bool): If True, assets will be created for tables
-            created by Airbyte's normalization feature. If False, only the destination tables
-            will be created. Defaults to True.
-        connection_to_group_fn (Optional[Callable[[str], Optional[str]]]): Function which returns an asset
-            group name for a given Airbyte connection name. If None, no groups will be created. Defaults
-            to a basic sanitization function.
-        connection_meta_to_group_fn (Optional[Callable[[AirbyteConnectionMetadata], Optional[str]]]): Function
-            which returns an asset group name for a given Airbyte connection metadata. If None and connection_to_group_fn
-            is None, no groups will be created. Defaults to None.
-        io_manager_key (Optional[str]): The I/O manager key to use for all assets. Defaults to "io_manager".
-            Use this if all assets should be loaded from the same source, otherwise use connection_to_io_manager_key_fn.
-        connection_to_io_manager_key_fn (Optional[Callable[[str], Optional[str]]]): Function which returns an
-            I/O manager key for a given Airbyte connection name. When other ops are downstream of the loaded assets,
-            the IOManager specified determines how the inputs to those ops are loaded. Defaults to "io_manager".
-        connection_filter (Optional[Callable[[AirbyteConnectionMetadata], bool]]): Optional function which
-            takes in connection metadata and returns False if the connection should be excluded from the output assets.
-        connection_directories (Optional[List[str]]): Optional list of connection directories to load assets from.
-            If omitted, all connections in the Airbyte project are loaded. May be faster than connection_filter
-            if the project has many connections or if the connection yaml files are large.
-        connection_to_asset_key_fn (Optional[Callable[[AirbyteConnectionMetadata, str], AssetKey]]): Optional function which
-            takes in connection metadata and table name and returns an asset key for the table. If None, the default asset
-            key is based on the table name. Any asset key prefix will be applied to the output of this function.
-        connection_to_freshness_policy_fn (Optional[Callable[[AirbyteConnectionMetadata], Optional[FreshnessPolicy]]]):
-            Optional function which takes in connection metadata and returns a freshness policy for the connection's assets.
-            If None, no freshness policies will be applied to the assets.
-        connection_to_auto_materialize_policy_fn (Optional[Callable[[AirbyteConnectionMetadata], Optional[AutoMaterializePolicy]]]):
-            Optional function which takes in connection metadata and returns an auto materialization policy for the connection's assets.
-            If None, no auto materialization policies will be applied to the assets.
+        workspace (AirbyteCloudWorkspace): The Airbyte workspace to fetch assets from.
+        dagster_airbyte_translator (Optional[DagsterAirbyteTranslator], optional): The translator to use
+            to convert Airbyte content into :py:class:`dagster.AssetSpec`.
+            Defaults to :py:class:`DagsterAirbyteTranslator`.
 
-    **Examples:**
+    Returns:
+        List[AssetsDefinition]: The list of AssetsDefinition for all connections in the Airbyte workspace.
 
-    Loading all Airbyte connections as assets:
+    Examples:
+        Sync the tables of a Airbyte connection:
 
-    .. code-block:: python
+        .. code-block:: python
 
-        from dagster_airbyte import load_assets_from_airbyte_project
+            from dagster_airbyte import AirbyteCloudWorkspace, build_airbyte_assets_definitions
 
-        airbyte_assets = load_assets_from_airbyte_project(
-            project_dir="path/to/airbyte/project",
-        )
+            import dagster as dg
 
-    Filtering the set of loaded connections:
+            airbyte_workspace = AirbyteCloudWorkspace(
+                workspace_id=dg.EnvVar("AIRBYTE_CLOUD_WORKSPACE_ID"),
+                client_id=dg.EnvVar("AIRBYTE_CLOUD_CLIENT_ID"),
+                client_secret=dg.EnvVar("AIRBYTE_CLOUD_CLIENT_SECRET"),
+            )
 
-    .. code-block:: python
 
-        from dagster_airbyte import load_assets_from_airbyte_project
+            airbyte_assets = build_airbyte_assets_definitions(workspace=workspace)
 
-        airbyte_assets = load_assets_from_airbyte_project(
-            project_dir="path/to/airbyte/project",
-            connection_filter=lambda meta: "snowflake" in meta.name,
-        )
+            defs = dg.Definitions(
+                assets=airbyte_assets,
+                resources={"airbyte": airbyte_workspace},
+            )
+
+        Sync the tables of a Airbyte connection with a custom translator:
+
+        .. code-block:: python
+
+            from dagster_airbyte import (
+                DagsterAirbyteTranslator,
+                AirbyteConnectionTableProps,
+                AirbyteCloudWorkspace,
+                build_airbyte_assets_definitions
+            )
+
+            import dagster as dg
+
+            class CustomDagsterAirbyteTranslator(DagsterAirbyteTranslator):
+                def get_asset_spec(self, props: AirbyteConnectionTableProps) -> dg.AssetSpec:
+                    default_spec = super().get_asset_spec(props)
+                    return default_spec.merge_attributes(
+                        metadata={"custom": "metadata"},
+                    )
+
+            airbyte_workspace = AirbyteCloudWorkspace(
+                workspace_id=dg.EnvVar("AIRBYTE_CLOUD_WORKSPACE_ID"),
+                client_id=dg.EnvVar("AIRBYTE_CLOUD_CLIENT_ID"),
+                client_secret=dg.EnvVar("AIRBYTE_CLOUD_CLIENT_SECRET"),
+            )
+
+
+            airbyte_assets = build_airbyte_assets_definitions(
+                workspace=workspace,
+                dagster_airbyte_translator=CustomDagsterAirbyteTranslator()
+            )
+
+            defs = dg.Definitions(
+                assets=airbyte_assets,
+                resources={"airbyte": airbyte_workspace},
+            )
     """
-    if isinstance(key_prefix, str):
-        key_prefix = [key_prefix]
-    key_prefix = check.list_param(key_prefix or [], "key_prefix", of_type=str)
+    dagster_airbyte_translator = dagster_airbyte_translator or DagsterAirbyteTranslator()
 
-    check.invariant(
-        not io_manager_key or not connection_to_io_manager_key_fn,
-        "Cannot specify both io_manager_key and connection_to_io_manager_key_fn",
+    all_asset_specs = workspace.load_asset_specs(
+        dagster_airbyte_translator=dagster_airbyte_translator
     )
-    if not connection_to_io_manager_key_fn:
-        connection_to_io_manager_key_fn = lambda _: io_manager_key
 
-    check.invariant(
-        not connection_meta_to_group_fn
-        or not connection_to_group_fn
-        or connection_to_group_fn == _clean_name,
-        "Cannot specify both connection_meta_to_group_fn and connection_to_group_fn",
-    )
+    connections = {
+        (
+            check.not_none(AirbyteMetadataSet.extract(spec.metadata).connection_id),
+            check.not_none(AirbyteMetadataSet.extract(spec.metadata).connection_name),
+        )
+        for spec in all_asset_specs
+    }
 
-    if not connection_meta_to_group_fn and connection_to_group_fn:
-        connection_meta_to_group_fn = lambda meta: connection_to_group_fn(meta.name)
+    _asset_fns = []
+    for connection_id, connection_name in connections:
 
-    return AirbyteYAMLCacheableAssetsDefinition(
-        project_dir=project_dir,
-        workspace_id=workspace_id,
-        key_prefix=key_prefix,
-        create_assets_for_normalization_tables=create_assets_for_normalization_tables,
-        connection_meta_to_group_fn=connection_meta_to_group_fn,
-        connection_to_io_manager_key_fn=connection_to_io_manager_key_fn,
-        connection_filter=connection_filter,
-        connection_directories=connection_directories,
-        connection_to_asset_key_fn=connection_to_asset_key_fn,
-        connection_to_freshness_policy_fn=connection_to_freshness_policy_fn,
-        connection_to_auto_materialize_policy_fn=connection_to_auto_materialize_policy_fn,
-    )
+        @airbyte_assets(
+            connection_id=connection_id,
+            workspace=workspace,
+            name=clean_name(connection_name),
+            group_name=clean_name(connection_name),
+            dagster_airbyte_translator=dagster_airbyte_translator,
+        )
+        def _asset_fn(context: AssetExecutionContext, airbyte: AirbyteCloudWorkspace):
+            yield from airbyte.sync_and_poll(context=context)
+
+        _asset_fns.append(_asset_fn)
+
+    return _asset_fns