dagster-airbyte 0.20.17__tar.gz → 0.21.0__tar.gz

{dagster-airbyte-0.20.17/dagster_airbyte.egg-info → dagster-airbyte-0.21.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dagster-airbyte
-Version: 0.20.17
+Version: 0.21.0
 Summary: Package for integrating Airbyte with Dagster.
 Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-airbyte
 Author: Dagster Labs

{dagster-airbyte-0.20.17 → dagster-airbyte-0.21.0}/dagster_airbyte/asset_defs.py

@@ -226,6 +226,7 @@ def build_airbyte_assets(
     upstream_assets: Optional[Set[AssetKey]] = None,
     schema_by_table_name: Optional[Mapping[str, TableSchema]] = None,
     freshness_policy: Optional[FreshnessPolicy] = None,
+    stream_to_asset_map: Optional[Mapping[str, str]] = None,
 ) -> Sequence[AssetsDefinition]:
     """Builds a set of assets representing the tables created by an Airbyte sync operation.
 
@@ -244,6 +245,8 @@ def build_airbyte_assets(
             A list of assets to add as sources.
         upstream_assets (Optional[Set[AssetKey]]): Deprecated, use deps instead. A list of assets to add as sources.
         freshness_policy (Optional[FreshnessPolicy]): A freshness policy to apply to the assets
+        stream_to_asset_map (Optional[Mapping[str, str]]): A mapping of an Airbyte stream name to a Dagster asset.
+            This allows the use of the "prefix" setting in Airbyte with special characters that aren't valid asset names.
     """
     if upstream_assets is not None and deps is not None:
         raise DagsterInvalidDefinitionError(
@@ -314,7 +317,9 @@ def build_airbyte_assets(
                             output_name=_table_to_output_name_fn(dependent_table),
                         )
         else:
-            for materialization in generate_materializations(ab_output, asset_key_prefix):
+            for materialization in generate_materializations(
+                ab_output, asset_key_prefix, stream_to_asset_map
+            ):
                 table_name = materialization.asset_key.path[-1]
                 if table_name in destination_tables:
                     yield Output(

{dagster-airbyte-0.20.17 → dagster-airbyte-0.21.0}/dagster_airbyte/utils.py

@@ -1,4 +1,4 @@
-from typing import Any, Iterator, Mapping, Sequence
+from typing import Any, Iterator, Mapping, Optional, Sequence
 
 from dagster import AssetMaterialization, MetadataValue
 from dagster._core.definitions.metadata.table import TableColumn, TableSchema
@@ -46,7 +46,9 @@ def _get_attempt(attempt: dict):
 
 
 def generate_materializations(
-    output: AirbyteOutput, asset_key_prefix: Sequence[str]
+    output: AirbyteOutput,
+    asset_key_prefix: Sequence[str],
+    stream_to_asset_map: Optional[Mapping[str, str]] = None,
 ) -> Iterator[AssetMaterialization]:
     prefix = output.connection_details.get("prefix") or ""
     # all the streams that are set to be sync'd by this connection
@@ -64,11 +66,12 @@ def generate_materializations(
         s["streamName"]: s.get("stats", {})
         for s in _get_attempt(output.job_details.get("attempts", [{}])[-1]).get("streamStats", [])
     }
+    stream_to_asset_map = stream_to_asset_map if stream_to_asset_map else {}
     for stream_name, stream_props in all_stream_props.items():
         yield _materialization_for_stream(
-            stream_name,
+            stream_to_asset_map.get(stream_name, stream_name),
             stream_props,
-            # if no records are sync'd, no stats will be avaiable for this stream
+            # if no records are sync'd, no stats will be available for this stream
             all_stream_stats.get(stream_name, {}),
             asset_key_prefix=asset_key_prefix,
         )

dagster-airbyte-0.21.0/dagster_airbyte/version.py

@@ -0,0 +1 @@
+__version__ = "0.21.0"

{dagster-airbyte-0.20.17 → dagster-airbyte-0.21.0/dagster_airbyte.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dagster-airbyte
-Version: 0.20.17
+Version: 0.21.0
 Summary: Package for integrating Airbyte with Dagster.
 Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-airbyte
 Author: Dagster Labs

dagster-airbyte-0.21.0/dagster_airbyte.egg-info/requires.txt

@@ -0,0 +1,8 @@
+dagster==1.5.0
+requests
+
+[managed]
+dagster-managed-elements==0.21.0
+
+[test]
+requests-mock

{dagster-airbyte-0.20.17 → dagster-airbyte-0.21.0}/setup.py

@@ -35,7 +35,7 @@ setup(
     ],
     packages=find_packages(exclude=["dagster_airbyte_tests*"]),
     install_requires=[
-        "dagster==1.4.17",
+        "dagster==1.5.0",
         "requests",
     ],
     zip_safe=False,
@@ -49,7 +49,7 @@ setup(
             "requests-mock",
         ],
         "managed": [
-            "dagster-managed-elements==0.20.17",
+            "dagster-managed-elements==0.21.0",
         ],
     },
 )

dagster-airbyte-0.20.17/dagster_airbyte/version.py

@@ -1 +0,0 @@
-__version__ = "0.20.17"

dagster-airbyte-0.20.17/dagster_airbyte.egg-info/requires.txt

@@ -1,8 +0,0 @@
-dagster==1.4.17
-requests
-
-[managed]
-dagster-managed-elements==0.20.17
-
-[test]
-requests-mock