dagster-airbyte 0.24.10__py3-none-any.whl → 0.24.11__py3-none-any.whl

dagster_airbyte/asset_defs.py

@@ -40,7 +40,7 @@ from dagster._core.definitions.cacheable_assets import (
     CacheableAssetsDefinition,
 )
 from dagster._core.definitions.events import CoercibleToAssetKey, CoercibleToAssetKeyPrefix
-from dagster._core.definitions.metadata import MetadataValue, TableSchemaMetadataValue
+from dagster._core.definitions.metadata.metadata_set import TableMetadataSet
 from dagster._core.definitions.metadata.table import TableSchema
 from dagster._core.errors import DagsterInvalidDefinitionError, DagsterInvalidInvocationError
 from dagster._core.execution.context.init import build_init_resource_context
@@ -62,9 +62,13 @@ def _table_to_output_name_fn(table: str) -> str:
 def _build_airbyte_asset_defn_metadata(
     connection_id: str,
     destination_tables: Sequence[str],
+    destination_raw_table_names_by_table: Mapping[str, str],
+    destination_database: Optional[str],
+    destination_schema: Optional[str],
     table_to_asset_key_fn: Callable[[str], AssetKey],
     asset_key_prefix: Optional[Sequence[str]] = None,
     normalization_tables: Optional[Mapping[str, Set[str]]] = None,
+    normalization_raw_table_names_by_table: Optional[Mapping[str, str]] = None,
     upstream_assets: Optional[Iterable[AssetKey]] = None,
     group_name: Optional[str] = None,
     io_manager_key: Optional[str] = None,
@@ -112,6 +116,30 @@ def _build_airbyte_asset_defn_metadata(
     for table in destination_tables:
         internal_deps[table] = set(upstream_assets or [])
 
+    relation_identifiers: Dict[str, str] = {}
+    for table in destination_tables:
+        if destination_database and destination_schema and table:
+            # Use the destination raw table name to create the relation identifier
+            relation_identifiers[table] = ".".join(
+                [
+                    destination_database,
+                    destination_schema,
+                    destination_raw_table_names_by_table[table],
+                ]
+            )
+            if normalization_tables and normalization_raw_table_names_by_table:
+                for normalization_table in normalization_tables.get(table, set()):
+                    relation_identifiers[normalization_table] = ".".join(
+                        [
+                            destination_database,
+                            destination_schema,
+                            destination_raw_table_names_by_table[table],
+                            normalization_raw_table_names_by_table[normalization_table],
+                        ]
+                    )
+
+    schema_by_table_name = schema_by_table_name if schema_by_table_name else {}
+
     return AssetsDefinitionCacheableData(
         keys_by_input_name=(
             {asset_key.path[-1]: asset_key for asset_key in upstream_assets}
@@ -125,11 +153,14 @@ def _build_airbyte_asset_defn_metadata(
         can_subset=False,
         metadata_by_output_name=(
             {
-                table: {"table_schema": MetadataValue.table_schema(schema_by_table_name[table])}
+                table: {
+                    **TableMetadataSet(
+                        column_schema=schema_by_table_name.get(table),
+                        relation_identifier=relation_identifiers.get(table),
+                    ),
+                }
                 for table in tables
             }
-            if schema_by_table_name
-            else None
         ),
         freshness_policies_by_output_name=(
             {output: freshness_policy for output in outputs} if freshness_policy else None
@@ -167,10 +198,7 @@ def _build_airbyte_assets_from_metadata(
             k: AssetOut(
                 key=v,
                 metadata=(
-                    {
-                        k: cast(TableSchemaMetadataValue, v)
-                        for k, v in assets_defn_meta.metadata_by_output_name.get(k, {}).items()
-                    }
+                    assets_defn_meta.metadata_by_output_name.get(k)
                     if assets_defn_meta.metadata_by_output_name
                     else None
                 ),
@@ -225,6 +253,8 @@ def _build_airbyte_assets_from_metadata(
 def build_airbyte_assets(
     connection_id: str,
     destination_tables: Sequence[str],
+    destination_database: Optional[str] = None,
+    destination_schema: Optional[str] = None,
     asset_key_prefix: Optional[Sequence[str]] = None,
     group_name: Optional[str] = None,
     normalization_tables: Optional[Mapping[str, Set[str]]] = None,
@@ -243,6 +273,8 @@ def build_airbyte_assets(
         destination_tables (List[str]): The names of the tables that you want to be represented
             in the Dagster asset graph for this sync. This will generally map to the name of the
             stream in Airbyte, unless a stream prefix has been specified in Airbyte.
+        destination_database (Optional[str]): The name of the destination database.
+        destination_schema (Optional[str]): The name of the destination schema.
         normalization_tables (Optional[Mapping[str, List[str]]]): If you are using Airbyte's
             normalization feature, you may specify a mapping of destination table to a list of
             derived tables that will be created by the normalization process.
@@ -269,13 +301,36 @@ def build_airbyte_assets(
     tables = chain.from_iterable(
         chain([destination_tables], normalization_tables.values() if normalization_tables else [])
     )
+
+    relation_identifiers: Dict[str, str] = {}
+    for table in destination_tables:
+        if destination_database and destination_schema and table:
+            relation_identifiers[table] = ".".join(
+                [destination_database, destination_schema, table]
+            )
+            if normalization_tables:
+                for normalization_table in normalization_tables.get(table, set()):
+                    relation_identifiers[normalization_table] = ".".join(
+                        [
+                            destination_database,
+                            destination_schema,
+                            table,
+                            normalization_table,
+                        ]
+                    )
+
+    schema_by_table_name = schema_by_table_name if schema_by_table_name else {}
+
     outputs = {
         table: AssetOut(
             key=AssetKey([*asset_key_prefix, table]),
             metadata=(
-                {"table_schema": MetadataValue.table_schema(schema_by_table_name[table])}
-                if schema_by_table_name
-                else None
+                {
+                    **TableMetadataSet(
+                        column_schema=schema_by_table_name.get(table),
+                        relation_identifier=relation_identifiers.get(table),
+                    ),
+                }
             ),
             freshness_policy=freshness_policy,
             auto_materialize_policy=auto_materialize_policy,
@@ -389,7 +444,7 @@ def _get_normalization_tables_for_schema(
 
         if "object" in schema_types and len(sub_schema.get("properties", {})) > 0:
             out[prefix + key] = AirbyteTableMetadata(
-                schema=generate_table_schema(sub_schema.get("properties", {}))
+                raw_table_name=key, schema=generate_table_schema(sub_schema.get("properties", {}))
             )
             for k, v in sub_schema["properties"].items():
                 out = merge_dicts(
@@ -398,7 +453,8 @@ def _get_normalization_tables_for_schema(
         # Array types are also broken into a new table
         elif "array" in schema_types:
             out[prefix + key] = AirbyteTableMetadata(
-                schema=generate_table_schema(sub_schema.get("items", {}).get("properties", {}))
+                raw_table_name=key,
+                schema=generate_table_schema(sub_schema.get("items", {}).get("properties", {})),
             )
             if sub_schema.get("items", {}).get("properties"):
                 for k, v in sub_schema["items"]["properties"].items():
@@ -422,6 +478,7 @@ class AirbyteConnectionMetadata(
             ("stream_prefix", str),
             ("has_basic_normalization", bool),
             ("stream_data", List[Mapping[str, Any]]),
+            ("destination", Mapping[str, Any]),
         ],
     )
 ):
@@ -436,7 +493,10 @@ class AirbyteConnectionMetadata(
 
     @classmethod
     def from_api_json(
-        cls, contents: Mapping[str, Any], operations: Mapping[str, Any]
+        cls,
+        contents: Mapping[str, Any],
+        operations: Mapping[str, Any],
+        destination: Mapping[str, Any],
     ) -> "AirbyteConnectionMetadata":
         return cls(
             name=contents["name"],
@@ -446,10 +506,13 @@ class AirbyteConnectionMetadata(
                 for op in operations.get("operations", [])
             ),
             stream_data=contents.get("syncCatalog", {}).get("streams", []),
+            destination=destination,
         )
 
     @classmethod
-    def from_config(cls, contents: Mapping[str, Any]) -> "AirbyteConnectionMetadata":
+    def from_config(
+        cls, contents: Mapping[str, Any], destination: Mapping[str, Any]
+    ) -> "AirbyteConnectionMetadata":
         config_contents = cast(Mapping[str, Any], contents.get("configuration"))
         check.invariant(
             config_contents is not None, "Airbyte connection config is missing 'configuration' key"
@@ -463,6 +526,7 @@ class AirbyteConnectionMetadata(
                 for op in config_contents.get("operations", [])
             ),
             stream_data=config_contents.get("sync_catalog", {}).get("streams", []),
+            destination=destination,
         )
 
     def parse_stream_tables(
@@ -497,6 +561,7 @@ class AirbyteConnectionMetadata(
                         prefixed_norm_table_name = f"{self.stream_prefix}{normalization_table_name}"
                         normalization_tables[prefixed_norm_table_name] = meta
             tables[prefixed_name] = AirbyteTableMetadata(
+                raw_table_name=name,
                 schema=generate_table_schema(schema_props),
                 normalization_tables=normalization_tables,
             )
@@ -576,14 +641,35 @@ class AirbyteCoreCacheableAssetsDefinition(CacheableAssetsDefinition):
             )
             schema_by_table_name = _get_schema_by_table_name(stream_table_metadata)
 
+            destination_database = connection.destination.get("configuration", {}).get("database")
+            destination_schema = connection.destination.get("configuration", {}).get("schema")
+
             table_to_asset_key = partial(self._connection_to_asset_key_fn, connection)
+
+            destination_tables = list(stream_table_metadata.keys())
+            destination_raw_table_names_by_table = {
+                table: metadata.raw_table_name for table, metadata in stream_table_metadata.items()
+            }
+            normalization_tables = {
+                table: set(metadata.normalization_tables.keys())
+                for table, metadata in stream_table_metadata.items()
+            }
+            normalization_raw_table_names_by_table = {
+                normalization_table: metadata.normalization_tables[
+                    normalization_table
+                ].raw_table_name
+                for table, metadata in stream_table_metadata.items()
+                for normalization_table in normalization_tables[table]
+            }
+
             asset_data_for_conn = _build_airbyte_asset_defn_metadata(
                 connection_id=connection_id,
-                destination_tables=list(stream_table_metadata.keys()),
-                normalization_tables={
-                    table: set(metadata.normalization_tables.keys())
-                    for table, metadata in stream_table_metadata.items()
-                },
+                destination_tables=destination_tables,
+                destination_raw_table_names_by_table=destination_raw_table_names_by_table,
+                destination_database=destination_database,
+                destination_schema=destination_schema,
+                normalization_tables=normalization_tables,
+                normalization_raw_table_names_by_table=normalization_raw_table_names_by_table,
                 asset_key_prefix=self._key_prefix,
                 group_name=(
                     self._connection_meta_to_group_fn(connection)
@@ -699,7 +785,21 @@ class AirbyteInstanceCacheableAssetsDefinition(AirbyteCoreCacheableAssetsDefinit
                     )
                 ),
             )
-            connection = AirbyteConnectionMetadata.from_api_json(connection_json, operations_json)
+
+            destination_id = cast(str, connection_json.get("destinationId"))
+            destination_json = cast(
+                Dict[str, Any],
+                check.not_none(
+                    self._airbyte_instance.make_request(
+                        endpoint="/destinations/get",
+                        data={"destinationId": destination_id},
+                    )
+                ),
+            )
+
+            connection = AirbyteConnectionMetadata.from_api_json(
+                connection_json, operations_json, destination_json
+            )
 
             # Filter out connections that don't match the filter function
             if self._connection_filter and not self._connection_filter(connection):
@@ -759,7 +859,17 @@ class AirbyteYAMLCacheableAssetsDefinition(AirbyteCoreCacheableAssetsDefinition)
         for connection_name in connection_directories:
             connection_dir = os.path.join(connections_dir, connection_name)
             with open(os.path.join(connection_dir, "configuration.yaml"), encoding="utf-8") as f:
-                connection = AirbyteConnectionMetadata.from_config(yaml.safe_load(f.read()))
+                connection_data = yaml.safe_load(f.read())
+
+            destination_configuration_path = cast(
+                str, connection_data.get("destination_configuration_path")
+            )
+            with open(
+                os.path.join(self._project_dir, destination_configuration_path), encoding="utf-8"
+            ) as f:
+                destination_data = yaml.safe_load(f.read())
+
+            connection = AirbyteConnectionMetadata.from_config(connection_data, destination_data)
 
             # Filter out connections that don't match the filter function
             if self._connection_filter and not self._connection_filter(connection):

dagster_airbyte/types.py

@@ -6,10 +6,14 @@ from dagster._core.definitions.metadata.table import TableSchema
 class AirbyteTableMetadata:
     def __init__(
         self,
+        raw_table_name: str,
         schema: TableSchema,
         normalization_tables: Optional[Mapping[str, "AirbyteTableMetadata"]] = None,
     ):
-        """Contains metadata about an Airbyte table, including its schema and any created normalization tables."""
+        """Contains metadata about an Airbyte table, including its destination raw table name,
+        schema and any created normalization tables.
+        """
+        self.raw_table_name = raw_table_name
         self.schema = schema
         self.normalization_tables = normalization_tables or dict()
 

dagster_airbyte/version.py

@@ -1 +1 @@
-__version__ = "0.24.10"
+__version__ = "0.24.11"

{dagster_airbyte-0.24.10.dist-info → dagster_airbyte-0.24.11.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dagster-airbyte
-Version: 0.24.10
+Version: 0.24.11
 Summary: Package for integrating Airbyte with Dagster.
 Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-airbyte
 Author: Dagster Labs
@@ -15,10 +15,10 @@ Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Operating System :: OS Independent
 Requires-Python: >=3.8,<3.13
 License-File: LICENSE
-Requires-Dist: dagster ==1.8.10
+Requires-Dist: dagster ==1.8.11
 Requires-Dist: requests
 Provides-Extra: managed
-Requires-Dist: dagster-managed-elements ==0.24.10 ; extra == 'managed'
+Requires-Dist: dagster-managed-elements ==0.24.11 ; extra == 'managed'
 Provides-Extra: test
 Requires-Dist: requests-mock ; extra == 'test'
 

{dagster_airbyte-0.24.10.dist-info → dagster_airbyte-0.24.11.dist-info}/RECORD

@@ -1,21 +1,21 @@
 dagster_airbyte/__init__.py,sha256=eXDY4rzDz0PKzPAaO0zWw6uP8RQY-SgU0EXa7GMdM2k,1305
-dagster_airbyte/asset_defs.py,sha256=GAKJWVjrrG6IX2uqlK8zeDueBjbs5irkzig-siXB4XY,48539
+dagster_airbyte/asset_defs.py,sha256=_jG8PpusrnkIKxow8VsQte8sEZa_IbCDTvBpQoBwuww,53307
 dagster_airbyte/cli.py,sha256=HErteP1MjfHozKKSrznh0yAreKETbXp5NDHzXGsdvvE,425
 dagster_airbyte/ops.py,sha256=pq6mp7vN2wXgo3gJMuWaAcxTmfkZ7d1zWzPyL_auSEY,4208
 dagster_airbyte/py.typed,sha256=la67KBlbjXN-_-DfGNcdOcjYumVpKG_Tkw-8n5dnGB4,8
 dagster_airbyte/resources.py,sha256=FoHI_Eiw2F6tnI1KZMfkgXhThdU6DplHp6INsbjUFsg,29802
-dagster_airbyte/types.py,sha256=fwqUv_MZCegwHhSELgUqm1H1JVUb-m83CyXFkd0r-ko,1425
+dagster_airbyte/types.py,sha256=w1DyTcXyuzrG3wfkOPYFtwj7snHcgqf-dC7_pRjiE1Q,1544
 dagster_airbyte/utils.py,sha256=cFKCkGFAvwr17KFTeqpVtQRDsNo4zpqw9yr2-1YSJeI,2823
-dagster_airbyte/version.py,sha256=Myj5XbZsr7lAMNmFf6wAiKVPI6RdUkFCauJ1_IiXz-4,24
+dagster_airbyte/version.py,sha256=XoAmHmUmn_stAwPPPU1_t-UuvmY7DsPg9-UreBts-RU,24
 dagster_airbyte/managed/__init__.py,sha256=6SBtyNOMJ9Cu2UIwFExJHpL_ZVFo3rPMvyIxVOsKvWE,469
 dagster_airbyte/managed/reconciliation.py,sha256=HgrLT-Xs8vWY9SfbdBXuorMf60KCn5Qz7bPITW5MxJo,34862
 dagster_airbyte/managed/types.py,sha256=ja056Wm7_ZFw1XGSNmdxmBy2TcOxbnylJCpRA2ng2TE,14596
 dagster_airbyte/managed/generated/__init__.py,sha256=eYq-yfXEeffuKAVFXY8plD0se1wHjFNVqklpbu9gljw,108
 dagster_airbyte/managed/generated/destinations.py,sha256=x1wmWlXvOJHtfaZva3ErdKuVS--sDvfidSXR5ji9G5w,119692
 dagster_airbyte/managed/generated/sources.py,sha256=wyNoGJiNvW8mjRRs6b-_lWFs0Fgy-MZlRaxiN6bP-4s,282691
-dagster_airbyte-0.24.10.dist-info/LICENSE,sha256=TMatHW4_G9ldRdodEAp-l2Xa2WvsdeOh60E3v1R2jis,11349
-dagster_airbyte-0.24.10.dist-info/METADATA,sha256=w35TLIldBfcStMCQnHUpIUovBS4tuePN4ukHHfFFzA8,929
-dagster_airbyte-0.24.10.dist-info/WHEEL,sha256=yQN5g4mg4AybRjkgi-9yy4iQEFibGQmlz78Pik5Or-A,92
-dagster_airbyte-0.24.10.dist-info/entry_points.txt,sha256=XrbLOz3LpgPV5fdwMmgdP6Rp1AfSG07KeWIddLqh7Lw,61
-dagster_airbyte-0.24.10.dist-info/top_level.txt,sha256=HLwIRQCzqItn88_KbPP8DNTKKQEBUVKk6NCn4PrCtqY,16
-dagster_airbyte-0.24.10.dist-info/RECORD,,
+dagster_airbyte-0.24.11.dist-info/LICENSE,sha256=TMatHW4_G9ldRdodEAp-l2Xa2WvsdeOh60E3v1R2jis,11349
+dagster_airbyte-0.24.11.dist-info/METADATA,sha256=-ggc_Tx6b8G3ZLsUb3ZEum_Um0RdPVBATFvCPorkh44,929
+dagster_airbyte-0.24.11.dist-info/WHEEL,sha256=yQN5g4mg4AybRjkgi-9yy4iQEFibGQmlz78Pik5Or-A,92
+dagster_airbyte-0.24.11.dist-info/entry_points.txt,sha256=XrbLOz3LpgPV5fdwMmgdP6Rp1AfSG07KeWIddLqh7Lw,61
+dagster_airbyte-0.24.11.dist-info/top_level.txt,sha256=HLwIRQCzqItn88_KbPP8DNTKKQEBUVKk6NCn4PrCtqY,16
+dagster_airbyte-0.24.11.dist-info/RECORD,,