dagster-airbyte 0.20.17__py3-none-any.whl → 0.21.0__py3-none-any.whl

dagster_airbyte/asset_defs.py

@@ -226,6 +226,7 @@ def build_airbyte_assets(
     upstream_assets: Optional[Set[AssetKey]] = None,
     schema_by_table_name: Optional[Mapping[str, TableSchema]] = None,
     freshness_policy: Optional[FreshnessPolicy] = None,
+    stream_to_asset_map: Optional[Mapping[str, str]] = None,
 ) -> Sequence[AssetsDefinition]:
     """Builds a set of assets representing the tables created by an Airbyte sync operation.
 
@@ -244,6 +245,8 @@ def build_airbyte_assets(
             A list of assets to add as sources.
         upstream_assets (Optional[Set[AssetKey]]): Deprecated, use deps instead. A list of assets to add as sources.
         freshness_policy (Optional[FreshnessPolicy]): A freshness policy to apply to the assets
+        stream_to_asset_map (Optional[Mapping[str, str]]): A mapping of an Airbyte stream name to a Dagster asset.
+            This allows the use of the "prefix" setting in Airbyte with special characters that aren't valid asset names.
     """
     if upstream_assets is not None and deps is not None:
         raise DagsterInvalidDefinitionError(
@@ -314,7 +317,9 @@ def build_airbyte_assets(
                             output_name=_table_to_output_name_fn(dependent_table),
                         )
         else:
-            for materialization in generate_materializations(ab_output, asset_key_prefix):
+            for materialization in generate_materializations(
+                ab_output, asset_key_prefix, stream_to_asset_map
+            ):
                 table_name = materialization.asset_key.path[-1]
                 if table_name in destination_tables:
                     yield Output(

dagster_airbyte/utils.py

@@ -1,4 +1,4 @@
-from typing import Any, Iterator, Mapping, Sequence
+from typing import Any, Iterator, Mapping, Optional, Sequence
 
 from dagster import AssetMaterialization, MetadataValue
 from dagster._core.definitions.metadata.table import TableColumn, TableSchema
@@ -46,7 +46,9 @@ def _get_attempt(attempt: dict):
 
 
 def generate_materializations(
-    output: AirbyteOutput, asset_key_prefix: Sequence[str]
+    output: AirbyteOutput,
+    asset_key_prefix: Sequence[str],
+    stream_to_asset_map: Optional[Mapping[str, str]] = None,
 ) -> Iterator[AssetMaterialization]:
     prefix = output.connection_details.get("prefix") or ""
     # all the streams that are set to be sync'd by this connection
@@ -64,11 +66,12 @@ def generate_materializations(
         s["streamName"]: s.get("stats", {})
         for s in _get_attempt(output.job_details.get("attempts", [{}])[-1]).get("streamStats", [])
     }
+    stream_to_asset_map = stream_to_asset_map if stream_to_asset_map else {}
     for stream_name, stream_props in all_stream_props.items():
         yield _materialization_for_stream(
-            stream_name,
+            stream_to_asset_map.get(stream_name, stream_name),
             stream_props,
-            # if no records are sync'd, no stats will be avaiable for this stream
+            # if no records are sync'd, no stats will be available for this stream
             all_stream_stats.get(stream_name, {}),
             asset_key_prefix=asset_key_prefix,
         )

dagster_airbyte/version.py

@@ -1 +1 @@
-__version__ = "0.20.17"
+__version__ = "0.21.0"

{dagster_airbyte-0.20.17.dist-info → dagster_airbyte-0.21.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dagster-airbyte
-Version: 0.20.17
+Version: 0.21.0
 Summary: Package for integrating Airbyte with Dagster.
 Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-airbyte
 Author: Dagster Labs
@@ -13,10 +13,10 @@ Classifier: Programming Language :: Python :: 3.11
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Operating System :: OS Independent
 License-File: LICENSE
-Requires-Dist: dagster (==1.4.17)
+Requires-Dist: dagster (==1.5.0)
 Requires-Dist: requests
 Provides-Extra: managed
-Requires-Dist: dagster-managed-elements (==0.20.17) ; extra == 'managed'
+Requires-Dist: dagster-managed-elements (==0.21.0) ; extra == 'managed'
 Provides-Extra: test
 Requires-Dist: requests-mock ; extra == 'test'
 

{dagster_airbyte-0.20.17.dist-info → dagster_airbyte-0.21.0.dist-info}/RECORD

@@ -1,20 +1,20 @@
 dagster_airbyte/__init__.py,sha256=KRUcHzOxDEmo9oO7qRwJNrRnxK28PeHTeUemMS4z2Gc,1215
-dagster_airbyte/asset_defs.py,sha256=vjvJLnyWIsA9IxiLDB1AtgjGZe-SMCQofTJ3m8nCWWU,45192
+dagster_airbyte/asset_defs.py,sha256=aSq28JFYhQ3q6jLOfITFPko5CwtoqKuGPPONob7ydwo,45541
 dagster_airbyte/cli.py,sha256=HErteP1MjfHozKKSrznh0yAreKETbXp5NDHzXGsdvvE,425
 dagster_airbyte/ops.py,sha256=OaD6DA9Iz-T6hroSAQcrruqZWgR8ECxksupg0CQR_DU,4128
 dagster_airbyte/resources.py,sha256=I1QOe3rkCoySWL-M3WQr75J-mOhC8FRXFvS6ktjuXUU,26926
 dagster_airbyte/types.py,sha256=fwqUv_MZCegwHhSELgUqm1H1JVUb-m83CyXFkd0r-ko,1425
-dagster_airbyte/utils.py,sha256=wd_38t9rZelUogIA7xKjz8tbWfHUwdkyFTBTPI3kCIU,2639
-dagster_airbyte/version.py,sha256=eM75ih3rRMeRs2KAFT1C-wtFC_fhUYsPhS2X2Aa8ILk,24
+dagster_airbyte/utils.py,sha256=arYFolnfcJ5w-jRLDtwEHQ5vg545_QRxwlLJ2R0ZJcA,2831
+dagster_airbyte/version.py,sha256=-reNiXr25nUU7em7_IKJzimCI10W4nA8ouxAiOr4FaQ,23
 dagster_airbyte/managed/__init__.py,sha256=azuKhCI8eUsmEGBL2M2d6qkqQ0vhsV9H5IioP83kbm8,423
 dagster_airbyte/managed/reconciliation.py,sha256=-_zXsjDZ9aod82kzEytgW9127gJMKBuQxYfpYb0Sp2o,33784
 dagster_airbyte/managed/types.py,sha256=Aq6065Jcimf9uK0wbP_EsdTWMYyG5LEnn7sGZ6J1KBA,14209
 dagster_airbyte/managed/generated/__init__.py,sha256=1ChWhXU1uyX_9rClMD6XHzkriKOc0fjIQntsoI3G99g,76
 dagster_airbyte/managed/generated/destinations.py,sha256=afBHge9FQENIBVywHcBoUcSyGqrLDjQpv1XgHkks7Vs,119751
 dagster_airbyte/managed/generated/sources.py,sha256=hoahliCUupXBg07GnCZugjKhxyutjXZOfU0fOFv_Z68,282784
-dagster_airbyte-0.20.17.dist-info/LICENSE,sha256=TMatHW4_G9ldRdodEAp-l2Xa2WvsdeOh60E3v1R2jis,11349
-dagster_airbyte-0.20.17.dist-info/METADATA,sha256=aLJ0LulCEuR_c9dlq2g2oLEuvf81q8ftrHW9I90ccHY,853
-dagster_airbyte-0.20.17.dist-info/WHEEL,sha256=p46_5Uhzqz6AzeSosiOnxK-zmFja1i22CrQCjmYe8ec,92
-dagster_airbyte-0.20.17.dist-info/entry_points.txt,sha256=XrbLOz3LpgPV5fdwMmgdP6Rp1AfSG07KeWIddLqh7Lw,61
-dagster_airbyte-0.20.17.dist-info/top_level.txt,sha256=HLwIRQCzqItn88_KbPP8DNTKKQEBUVKk6NCn4PrCtqY,16
-dagster_airbyte-0.20.17.dist-info/RECORD,,
+dagster_airbyte-0.21.0.dist-info/LICENSE,sha256=TMatHW4_G9ldRdodEAp-l2Xa2WvsdeOh60E3v1R2jis,11349
+dagster_airbyte-0.21.0.dist-info/METADATA,sha256=6EGu_12sdlnCFVZNl2AJG6w5seZoxVASU9UeM5ZCQO8,850
+dagster_airbyte-0.21.0.dist-info/WHEEL,sha256=p46_5Uhzqz6AzeSosiOnxK-zmFja1i22CrQCjmYe8ec,92
+dagster_airbyte-0.21.0.dist-info/entry_points.txt,sha256=XrbLOz3LpgPV5fdwMmgdP6Rp1AfSG07KeWIddLqh7Lw,61
+dagster_airbyte-0.21.0.dist-info/top_level.txt,sha256=HLwIRQCzqItn88_KbPP8DNTKKQEBUVKk6NCn4PrCtqY,16
+dagster_airbyte-0.21.0.dist-info/RECORD,,