dagster-airbyte 0.20.4__tar.gz → 0.20.6__tar.gz

{dagster-airbyte-0.20.4/dagster_airbyte.egg-info → dagster-airbyte-0.20.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dagster-airbyte
-Version: 0.20.4
+Version: 0.20.6
 Summary: Package for integrating Airbyte with Dagster.
 Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-airbyte
 Author: Elementl

{dagster-airbyte-0.20.4 → dagster-airbyte-0.20.6}/dagster_airbyte/asset_defs.py

@@ -54,6 +54,10 @@ from dagster_airbyte.utils import (
 )
 
 
+def _table_to_output_name_fn(table: str) -> str:
+    return table.replace("-", "_")
+
+
 def _build_airbyte_asset_defn_metadata(
     connection_id: str,
     destination_tables: Sequence[str],
@@ -82,7 +86,10 @@ def _build_airbyte_asset_defn_metadata(
     )
 
     outputs = {
-        table: AssetKey([*asset_key_prefix, *table_to_asset_key_fn(table).path]) for table in tables
+        _table_to_output_name_fn(table): AssetKey(
+            [*asset_key_prefix, *table_to_asset_key_fn(table).path]
+        )
+        for table in tables
     }
 
     internal_deps: Dict[str, Set[AssetKey]] = {}
@@ -105,28 +112,32 @@ def _build_airbyte_asset_defn_metadata(
         internal_deps[table] = set(upstream_assets or [])
 
     return AssetsDefinitionCacheableData(
-        keys_by_input_name={asset_key.path[-1]: asset_key for asset_key in upstream_assets}
-        if upstream_assets
-        else {},
+        keys_by_input_name=(
+            {asset_key.path[-1]: asset_key for asset_key in upstream_assets}
+            if upstream_assets
+            else {}
+        ),
         keys_by_output_name=outputs,
         internal_asset_deps=internal_deps,
         group_name=group_name,
         key_prefix=asset_key_prefix,
         can_subset=False,
-        metadata_by_output_name={
-            table: {"table_schema": MetadataValue.table_schema(schema_by_table_name[table])}
-            for table in tables
-        }
-        if schema_by_table_name
-        else None,
-        freshness_policies_by_output_name={output: freshness_policy for output in outputs}
-        if freshness_policy
-        else None,
-        auto_materialize_policies_by_output_name={
-            output: auto_materialize_policy for output in outputs
-        }
-        if auto_materialize_policy
-        else None,
+        metadata_by_output_name=(
+            {
+                table: {"table_schema": MetadataValue.table_schema(schema_by_table_name[table])}
+                for table in tables
+            }
+            if schema_by_table_name
+            else None
+        ),
+        freshness_policies_by_output_name=(
+            {output: freshness_policy for output in outputs} if freshness_policy else None
+        ),
+        auto_materialize_policies_by_output_name=(
+            {output: auto_materialize_policy for output in outputs}
+            if auto_materialize_policy
+            else None
+        ),
         extra_metadata={
             "connection_id": connection_id,
             "group_name": group_name,
@@ -154,16 +165,20 @@ def _build_airbyte_assets_from_metadata(
         outs={
             k: AssetOut(
                 key=v,
-                metadata={
-                    k: cast(TableSchemaMetadataValue, v)
-                    for k, v in assets_defn_meta.metadata_by_output_name.get(k, {}).items()
-                }
-                if assets_defn_meta.metadata_by_output_name
-                else None,
+                metadata=(
+                    {
+                        k: cast(TableSchemaMetadataValue, v)
+                        for k, v in assets_defn_meta.metadata_by_output_name.get(k, {}).items()
+                    }
+                    if assets_defn_meta.metadata_by_output_name
+                    else None
+                ),
                 io_manager_key=io_manager_key,
-                freshness_policy=assets_defn_meta.freshness_policies_by_output_name.get(k)
-                if assets_defn_meta.freshness_policies_by_output_name
-                else None,
+                freshness_policy=(
+                    assets_defn_meta.freshness_policies_by_output_name.get(k)
+                    if assets_defn_meta.freshness_policies_by_output_name
+                    else None
+                ),
                 dagster_type=Nothing,
             )
             for k, v in (assets_defn_meta.keys_by_output_name or {}).items()
@@ -184,7 +199,7 @@ def _build_airbyte_assets_from_metadata(
             if table_name in destination_tables:
                 yield Output(
                     value=None,
-                    output_name=table_name,
+                    output_name=_table_to_output_name_fn(table_name),
                     metadata=materialization.metadata,
                 )
                 # Also materialize any normalization tables affiliated with this destination
@@ -193,7 +208,7 @@ def _build_airbyte_assets_from_metadata(
                     for dependent_table in normalization_tables.get(table_name, set()):
                         yield Output(
                             value=None,
-                            output_name=dependent_table,
+                            output_name=_table_to_output_name_fn(dependent_table),
                         )
             else:
                 yield materialization
@@ -246,9 +261,11 @@ def build_airbyte_assets(
     outputs = {
         table: AssetOut(
             key=AssetKey([*asset_key_prefix, table]),
-            metadata={"table_schema": MetadataValue.table_schema(schema_by_table_name[table])}
-            if schema_by_table_name
-            else None,
+            metadata=(
+                {"table_schema": MetadataValue.table_schema(schema_by_table_name[table])}
+                if schema_by_table_name
+                else None
+            ),
             freshness_policy=freshness_policy,
         )
         for table in tables
@@ -288,13 +305,13 @@ def build_airbyte_assets(
             for table_name in destination_tables:
                 yield Output(
                     value=None,
-                    output_name=table_name,
+                    output_name=_table_to_output_name_fn(table_name),
                 )
                 if normalization_tables:
                     for dependent_table in normalization_tables.get(table_name, set()):
                         yield Output(
                             value=None,
-                            output_name=dependent_table,
+                            output_name=_table_to_output_name_fn(dependent_table),
                         )
         else:
             for materialization in generate_materializations(ab_output, asset_key_prefix):
@@ -302,7 +319,7 @@ def build_airbyte_assets(
                 if table_name in destination_tables:
                     yield Output(
                         value=None,
-                        output_name=table_name,
+                        output_name=_table_to_output_name_fn(table_name),
                         metadata=materialization.metadata,
                     )
                     # Also materialize any normalization tables affiliated with this destination
@@ -311,7 +328,7 @@ def build_airbyte_assets(
                         for dependent_table in normalization_tables.get(table_name, set()):
                             yield Output(
                                 value=None,
-                                output_name=dependent_table,
+                                output_name=_table_to_output_name_fn(dependent_table),
                             )
                 else:
                     yield materialization
@@ -330,8 +347,7 @@ def _get_schema_types(schema: Mapping[str, Any]) -> Sequence[str]:
 
 
 def _get_sub_schemas(schema: Mapping[str, Any]) -> Sequence[Mapping[str, Any]]:
-    """Returns a list of sub-schema definitions for a given schema. This is used to handle union types.
-    """
+    """Returns a list of sub-schema definitions for a given schema. This is used to handle union types."""
     return schema.get("anyOf") or schema.get("oneOf") or [schema]
 
 
@@ -516,9 +532,9 @@ class AirbyteCoreCacheableAssetsDefinition(CacheableAssetsDefinition):
         self._connection_to_group_fn = connection_to_group_fn
         self._connection_to_io_manager_key_fn = connection_to_io_manager_key_fn
         self._connection_filter = connection_filter
-        self._connection_to_asset_key_fn: Callable[
-            [AirbyteConnectionMetadata, str], AssetKey
-        ] = connection_to_asset_key_fn or (lambda _, table: AssetKey(path=[table]))
+        self._connection_to_asset_key_fn: Callable[[AirbyteConnectionMetadata, str], AssetKey] = (
+            connection_to_asset_key_fn or (lambda _, table: AssetKey(path=[table]))
+        )
         self._connection_to_freshness_policy_fn = connection_to_freshness_policy_fn or (
             lambda _: None
         )
@@ -555,12 +571,16 @@ class AirbyteCoreCacheableAssetsDefinition(CacheableAssetsDefinition):
                     for table, metadata in stream_table_metadata.items()
                 },
                 asset_key_prefix=self._key_prefix,
-                group_name=self._connection_to_group_fn(connection.name)
-                if self._connection_to_group_fn
-                else None,
-                io_manager_key=self._connection_to_io_manager_key_fn(connection.name)
-                if self._connection_to_io_manager_key_fn
-                else None,
+                group_name=(
+                    self._connection_to_group_fn(connection.name)
+                    if self._connection_to_group_fn
+                    else None
+                ),
+                io_manager_key=(
+                    self._connection_to_io_manager_key_fn(connection.name)
+                    if self._connection_to_io_manager_key_fn
+                    else None
+                ),
                 schema_by_table_name=schema_by_table_name,
                 table_to_asset_key_fn=table_to_asset_key,
                 freshness_policy=self._connection_to_freshness_policy_fn(connection),

{dagster-airbyte-0.20.4 → dagster-airbyte-0.20.6}/dagster_airbyte/managed/reconciliation.py

@@ -125,9 +125,11 @@ def conn_dict(conn: Optional[AirbyteConnection]) -> Mapping[str, Any]:
         "destination": conn.destination.name if conn.destination else "Unknown",
         "normalize data": conn.normalize_data,
         "streams": {k: v.to_json() for k, v in conn.stream_config.items()},
-        "destination namespace": conn.destination_namespace.name
-        if isinstance(conn.destination_namespace, AirbyteDestinationNamespace)
-        else conn.destination_namespace,
+        "destination namespace": (
+            conn.destination_namespace.name
+            if isinstance(conn.destination_namespace, AirbyteDestinationNamespace)
+            else conn.destination_namespace
+        ),
         "prefix": conn.prefix,
     }
 

{dagster-airbyte-0.20.4 → dagster-airbyte-0.20.6}/dagster_airbyte/managed/types.py

@@ -330,9 +330,11 @@ class InitializedAirbyteConnection:
                 destination=dest,
                 stream_config=streams,
                 normalize_data=len(api_dict["operationIds"]) > 0,
-                destination_namespace=api_dict["namespaceFormat"]
-                if api_dict["namespaceDefinition"] == "customformat"
-                else AirbyteDestinationNamespace(api_dict["namespaceDefinition"]),
+                destination_namespace=(
+                    api_dict["namespaceFormat"]
+                    if api_dict["namespaceDefinition"] == "customformat"
+                    else AirbyteDestinationNamespace(api_dict["namespaceDefinition"])
+                ),
                 prefix=api_dict["prefix"] if api_dict.get("prefix") else None,
             ),
             api_dict["connectionId"],

{dagster-airbyte-0.20.4 → dagster-airbyte-0.20.6}/dagster_airbyte/resources.py

@@ -438,9 +438,11 @@ class AirbyteResource(BaseAirbyteResource):
                             headers=headers,
                             json=data,
                             timeout=self.request_timeout,
-                            auth=(self.username, self.password)
-                            if self.username and self.password
-                            else None,
+                            auth=(
+                                (self.username, self.password)
+                                if self.username and self.password
+                                else None
+                            ),
                         ),
                         self.request_additional_params,
                     ),

{dagster-airbyte-0.20.4 → dagster-airbyte-0.20.6}/dagster_airbyte/types.py

@@ -9,8 +9,7 @@ class AirbyteTableMetadata:
         schema: TableSchema,
         normalization_tables: Optional[Mapping[str, "AirbyteTableMetadata"]] = None,
     ):
-        """Contains metadata about an Airbyte table, including its schema and any created normalization tables.
-        """
+        """Contains metadata about an Airbyte table, including its schema and any created normalization tables."""
         self.schema = schema
         self.normalization_tables = normalization_tables or dict()
 

{dagster-airbyte-0.20.4 → dagster-airbyte-0.20.6}/dagster_airbyte/utils.py

@@ -52,9 +52,9 @@ def generate_materializations(
     # all the streams that are set to be sync'd by this connection
     all_stream_props = {
         prefix
-        + stream["stream"]["name"]: stream.get("stream", {})
-        .get("jsonSchema", {})
-        .get("properties", {})
+        + stream["stream"]["name"]: (
+            stream.get("stream", {}).get("jsonSchema", {}).get("properties", {})
+        )
         for stream in output.connection_details.get("syncCatalog", {}).get("streams", [])
         if stream.get("config", {}).get("selected")
     }

dagster-airbyte-0.20.6/dagster_airbyte/version.py

@@ -0,0 +1 @@
+__version__ = "0.20.6"

{dagster-airbyte-0.20.4 → dagster-airbyte-0.20.6/dagster_airbyte.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dagster-airbyte
-Version: 0.20.4
+Version: 0.20.6
 Summary: Package for integrating Airbyte with Dagster.
 Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-airbyte
 Author: Elementl

dagster-airbyte-0.20.6/dagster_airbyte.egg-info/requires.txt

@@ -0,0 +1,8 @@
+dagster==1.4.6
+requests
+
+[managed]
+dagster-managed-elements==0.20.6
+
+[test]
+requests-mock

{dagster-airbyte-0.20.4 → dagster-airbyte-0.20.6}/setup.py

@@ -35,7 +35,7 @@ setup(
     ],
     packages=find_packages(exclude=["dagster_airbyte_tests*"]),
     install_requires=[
-        "dagster==1.4.4",
+        "dagster==1.4.6",
         "requests",
     ],
     zip_safe=False,
@@ -49,7 +49,7 @@ setup(
             "requests-mock",
         ],
         "managed": [
-            "dagster-managed-elements==0.20.4",
+            "dagster-managed-elements==0.20.6",
         ],
     },
 )

dagster-airbyte-0.20.4/dagster_airbyte/version.py

@@ -1 +0,0 @@
-__version__ = "0.20.4"

dagster-airbyte-0.20.4/dagster_airbyte.egg-info/requires.txt

@@ -1,8 +0,0 @@
-dagster==1.4.4
-requests
-
-[managed]
-dagster-managed-elements==0.20.4
-
-[test]
-requests-mock