dagster-airbyte 0.25.1__py3-none-any.whl → 0.25.10__py3-none-any.whl

dagster_airbyte/translator.py

@@ -0,0 +1,236 @@
+from collections.abc import Mapping, Sequence
+from enum import Enum
+from typing import Any, Optional
+
+from dagster._annotations import deprecated, experimental
+from dagster._core.definitions.asset_key import AssetKey
+from dagster._core.definitions.asset_spec import AssetSpec
+from dagster._core.definitions.metadata.metadata_set import NamespacedMetadataSet, TableMetadataSet
+from dagster._record import record
+from dagster._serdes.serdes import whitelist_for_serdes
+from dagster._utils.cached_method import cached_method
+
+from dagster_airbyte.utils import generate_table_schema, get_airbyte_connection_table_name
+
+
+class AirbyteJobStatusType(str, Enum):
+    RUNNING = "running"
+    SUCCEEDED = "succeeded"
+    CANCELLED = "cancelled"
+    PENDING = "pending"
+    FAILED = "failed"
+    ERROR = "error"
+    INCOMPLETE = "incomplete"
+
+
+@deprecated(breaking_version="1.10", additional_warn_text="Use `AirbyteJobStatusType` instead.")
+class AirbyteState:
+    RUNNING = AirbyteJobStatusType.RUNNING
+    SUCCEEDED = AirbyteJobStatusType.SUCCEEDED
+    CANCELLED = AirbyteJobStatusType.CANCELLED
+    PENDING = AirbyteJobStatusType.PENDING
+    FAILED = AirbyteJobStatusType.FAILED
+    ERROR = AirbyteJobStatusType.ERROR
+    INCOMPLETE = AirbyteJobStatusType.INCOMPLETE
+
+
+@record
+class AirbyteConnectionTableProps:
+    table_name: str
+    stream_prefix: Optional[str]
+    stream_name: str
+    json_schema: Mapping[str, Any]
+    connection_id: str
+    connection_name: str
+    destination_type: Optional[str]
+    database: Optional[str]
+    schema: Optional[str]
+
+    @property
+    def fully_qualified_table_name(self) -> Optional[str]:
+        return (
+            f"{self.database}.{self.schema}.{self.stream_name}"
+            if self.database and self.schema
+            else None
+        )
+
+
+@whitelist_for_serdes
+@record
+class AirbyteConnection:
+    """Represents an Airbyte connection, based on data as returned from the API."""
+
+    id: str
+    name: str
+    stream_prefix: Optional[str]
+    streams: Mapping[str, "AirbyteStream"]
+    destination_id: str
+
+    @classmethod
+    def from_connection_details(
+        cls,
+        connection_details: Mapping[str, Any],
+    ) -> "AirbyteConnection":
+        return cls(
+            id=connection_details["connectionId"],
+            name=connection_details["name"],
+            stream_prefix=connection_details.get("prefix"),
+            streams={
+                stream_details["stream"]["name"]: AirbyteStream.from_stream_details(
+                    stream_details=stream_details
+                )
+                for stream_details in connection_details.get("syncCatalog", {}).get("streams", [])
+            },
+            destination_id=connection_details["destinationId"],
+        )
+
+
+@whitelist_for_serdes
+@record
+class AirbyteDestination:
+    """Represents an Airbyte destination, based on data as returned from the API."""
+
+    id: str
+    type: str
+    database: Optional[str]
+    schema: Optional[str]
+
+    @classmethod
+    def from_destination_details(
+        cls,
+        destination_details: Mapping[str, Any],
+    ) -> "AirbyteDestination":
+        return cls(
+            id=destination_details["destinationId"],
+            type=destination_details["destinationType"],
+            database=destination_details["configuration"].get("database"),
+            schema=destination_details["configuration"].get("schema"),
+        )
+
+
+@whitelist_for_serdes
+@record
+class AirbyteStream:
+    """Represents an Airbyte stream, based on data as returned from the API.
+    A stream in Airbyte corresponds to a table.
+    """
+
+    name: str
+    selected: bool
+    json_schema: Mapping[str, Any]
+
+    @classmethod
+    def from_stream_details(
+        cls,
+        stream_details: Mapping[str, Any],
+    ) -> "AirbyteStream":
+        return cls(
+            name=stream_details["stream"]["name"],
+            selected=stream_details["config"].get("selected", False),
+            json_schema=stream_details["stream"].get("jsonSchema", {}),
+        )
+
+
+@whitelist_for_serdes
+@record
+class AirbyteJob:
+    """Represents an Airbyte job, based on data as returned from the API."""
+
+    id: int
+    status: str
+
+    @classmethod
+    def from_job_details(
+        cls,
+        job_details: Mapping[str, Any],
+    ) -> "AirbyteJob":
+        return cls(
+            id=job_details["jobId"],
+            status=job_details["status"],
+        )
+
+
+@whitelist_for_serdes
+@record
+class AirbyteWorkspaceData:
+    """A record representing all content in an Airbyte workspace.
+    This applies to both Airbyte OSS and Cloud.
+    """
+
+    connections_by_id: Mapping[str, AirbyteConnection]
+    destinations_by_id: Mapping[str, AirbyteDestination]
+
+    @cached_method
+    def to_airbyte_connection_table_props_data(self) -> Sequence[AirbyteConnectionTableProps]:
+        """Method that converts a `AirbyteWorkspaceData` object
+        to a collection of `AirbyteConnectionTableProps` objects.
+        """
+        data: list[AirbyteConnectionTableProps] = []
+
+        for connection in self.connections_by_id.values():
+            destination = self.destinations_by_id[connection.destination_id]
+
+            for stream in connection.streams.values():
+                if stream.selected:
+                    data.append(
+                        AirbyteConnectionTableProps(
+                            table_name=get_airbyte_connection_table_name(
+                                stream_prefix=connection.stream_prefix,
+                                stream_name=stream.name,
+                            ),
+                            stream_prefix=connection.stream_prefix,
+                            stream_name=stream.name,
+                            json_schema=stream.json_schema,
+                            connection_id=connection.id,
+                            connection_name=connection.name,
+                            destination_type=destination.type,
+                            database=destination.database,
+                            schema=destination.schema,
+                        )
+                    )
+
+        return data
+
+
+class AirbyteMetadataSet(NamespacedMetadataSet):
+    connection_id: str
+    connection_name: str
+    stream_prefix: Optional[str] = None
+
+    @classmethod
+    def namespace(cls) -> str:
+        return "dagster-airbyte"
+
+
+@experimental
+class DagsterAirbyteTranslator:
+    """Translator class which converts a `AirbyteConnectionTableProps` object into AssetSpecs.
+    Subclass this class to implement custom logic how to translate Airbyte content into asset spec.
+    """
+
+    def get_asset_spec(self, props: AirbyteConnectionTableProps) -> AssetSpec:
+        """Get the AssetSpec for a table synced by an Airbyte connection."""
+        table_schema_props = (
+            props.json_schema.get("properties")
+            or props.json_schema.get("items", {}).get("properties")
+            or {}
+        )
+        column_schema = generate_table_schema(table_schema_props)
+
+        metadata = {
+            **TableMetadataSet(
+                column_schema=column_schema,
+                table_name=props.fully_qualified_table_name,
+            ),
+            **AirbyteMetadataSet(
+                connection_id=props.connection_id,
+                connection_name=props.connection_name,
+                stream_prefix=props.stream_prefix,
+            ),
+        }
+
+        return AssetSpec(
+            key=AssetKey(props.table_name),
+            metadata=metadata,
+            kinds={"airbyte", *({props.destination_type} if props.destination_type else set())},
+        )

dagster_airbyte/types.py

@@ -1,4 +1,5 @@
-from typing import Any, Mapping, NamedTuple, Optional
+from collections.abc import Mapping
+from typing import Any, NamedTuple, Optional
 
 from dagster._core.definitions.metadata.table import TableSchema
 

dagster_airbyte/utils.py

@@ -1,10 +1,31 @@
-from typing import Any, Iterator, Mapping, Optional, Sequence
+import re
+from collections.abc import Iterator, Mapping, Sequence
+from typing import TYPE_CHECKING, Any, Optional
 
-from dagster import AssetMaterialization, MetadataValue
+from dagster import (
+    AssetMaterialization,
+    AssetsDefinition,
+    DagsterInvariantViolationError,
+    MetadataValue,
+)
 from dagster._core.definitions.metadata.table import TableColumn, TableSchema
 
 from dagster_airbyte.types import AirbyteOutput
 
+if TYPE_CHECKING:
+    from dagster_airbyte import DagsterAirbyteTranslator
+
+DAGSTER_AIRBYTE_TRANSLATOR_METADATA_KEY = "dagster-airbyte/dagster_airbyte_translator"
+
+
+def clean_name(name: str) -> str:
+    """Cleans an input to be a valid Dagster asset name."""
+    return re.sub(r"[^a-z0-9]+", "_", name.lower())
+
+
+def get_airbyte_connection_table_name(stream_prefix: Optional[str], stream_name: str) -> str:
+    return f"{stream_prefix if stream_prefix else ''}{stream_name}"
+
 
 def generate_table_schema(stream_schema_props: Mapping[str, Any]) -> TableSchema:
     return TableSchema(
@@ -74,3 +95,18 @@ def generate_materializations(
             all_stream_stats.get(stream_name, {}),
             asset_key_prefix=asset_key_prefix,
         )
+
+
+def get_translator_from_airbyte_assets(
+    airbyte_assets: AssetsDefinition,
+) -> "DagsterAirbyteTranslator":
+    metadata_by_key = airbyte_assets.metadata_by_key or {}
+    first_asset_key = next(iter(airbyte_assets.metadata_by_key.keys()))
+    first_metadata = metadata_by_key.get(first_asset_key, {})
+    dagster_airbyte_translator = first_metadata.get(DAGSTER_AIRBYTE_TRANSLATOR_METADATA_KEY)
+    if dagster_airbyte_translator is None:
+        raise DagsterInvariantViolationError(
+            f"Expected to find airbyte translator metadata on asset {first_asset_key.to_user_string()},"
+            " but did not. Did you pass in assets that weren't generated by @airbyte_assets?"
+        )
+    return dagster_airbyte_translator

dagster_airbyte/version.py

@@ -1 +1 @@
-__version__ = "0.25.1"
+__version__ = "0.25.10"

{dagster_airbyte-0.25.1.dist-info → dagster_airbyte-0.25.10.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dagster-airbyte
-Version: 0.25.1
+Version: 0.25.10
 Summary: Package for integrating Airbyte with Dagster.
 Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-airbyte
 Author: Dagster Labs
@@ -14,10 +14,11 @@ Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Operating System :: OS Independent
 Requires-Python: >=3.9,<3.13
 License-File: LICENSE
-Requires-Dist: dagster ==1.9.1
+Requires-Dist: dagster ==1.9.10
 Requires-Dist: requests
 Provides-Extra: managed
-Requires-Dist: dagster-managed-elements ==0.25.1 ; extra == 'managed'
+Requires-Dist: dagster-managed-elements ==0.25.10 ; extra == 'managed'
 Provides-Extra: test
 Requires-Dist: requests-mock ; extra == 'test'
+Requires-Dist: flaky ; extra == 'test'
 

dagster_airbyte-0.25.10.dist-info/RECORD

@@ -0,0 +1,23 @@
+dagster_airbyte/__init__.py,sha256=T3mFUhqESk-F0FISreWfuLzULKYCO67WtGzbxGVUxdY,1719
+dagster_airbyte/asset_decorator.py,sha256=s0wFVFqe8kOGSiwyZx9n96C-_tnYfGuxsf_qefeFjJU,4209
+dagster_airbyte/asset_defs.py,sha256=039JPL5Jy0WP34K3JUO3hvQpbBLVLMOqY0bgyh73VHA,50059
+dagster_airbyte/cli.py,sha256=HErteP1MjfHozKKSrznh0yAreKETbXp5NDHzXGsdvvE,425
+dagster_airbyte/ops.py,sha256=oOEczVYpqVRTc1-0osfpR5FZbPjNJDYihgRjk_qtOQA,4229
+dagster_airbyte/py.typed,sha256=la67KBlbjXN-_-DfGNcdOcjYumVpKG_Tkw-8n5dnGB4,8
+dagster_airbyte/resources.py,sha256=PhRaLAiYtN83vk3sqhzebhw605DYF1ZY6Q87XvG0KRs,53552
+dagster_airbyte/translator.py,sha256=jOdBA5cDb1dRIgJZlqQv_SzqPRzvnm9N-UmA0yK_tsI,7546
+dagster_airbyte/types.py,sha256=Pc3X0BGYo2dabRGUJf0TinSNoW_QHFOEioTyAN0NCyc,1571
+dagster_airbyte/utils.py,sha256=wJtzx5cWq8Bp1f3T4XjLPaxjXZGstHlL4kRYP2c49bs,4157
+dagster_airbyte/version.py,sha256=Nw3Sk1aJKD__b1pX4zBOVEcqxnFwhwsJ1cNd5rN8Cgs,24
+dagster_airbyte/managed/__init__.py,sha256=6SBtyNOMJ9Cu2UIwFExJHpL_ZVFo3rPMvyIxVOsKvWE,469
+dagster_airbyte/managed/reconciliation.py,sha256=K-n2_nPwLQrsZSXSo7a0AcAjglMDsvhRErb5zmxlu7E,34814
+dagster_airbyte/managed/types.py,sha256=isPfX8L9YwtZAf9Vk4hhxBePLR00AEldsdK2TsM1H2o,14611
+dagster_airbyte/managed/generated/__init__.py,sha256=eYq-yfXEeffuKAVFXY8plD0se1wHjFNVqklpbu9gljw,108
+dagster_airbyte/managed/generated/destinations.py,sha256=x1wmWlXvOJHtfaZva3ErdKuVS--sDvfidSXR5ji9G5w,119692
+dagster_airbyte/managed/generated/sources.py,sha256=y0TPNvcRd8c9mhje-NoXsHeKRPt1nXcpww8mNAtqCps,282685
+dagster_airbyte-0.25.10.dist-info/LICENSE,sha256=TMatHW4_G9ldRdodEAp-l2Xa2WvsdeOh60E3v1R2jis,11349
+dagster_airbyte-0.25.10.dist-info/METADATA,sha256=K8P5rqqTVw-_l9_WtHXht4DHtvitORG5g_U2V3O2mso,918
+dagster_airbyte-0.25.10.dist-info/WHEEL,sha256=yQN5g4mg4AybRjkgi-9yy4iQEFibGQmlz78Pik5Or-A,92
+dagster_airbyte-0.25.10.dist-info/entry_points.txt,sha256=XrbLOz3LpgPV5fdwMmgdP6Rp1AfSG07KeWIddLqh7Lw,61
+dagster_airbyte-0.25.10.dist-info/top_level.txt,sha256=HLwIRQCzqItn88_KbPP8DNTKKQEBUVKk6NCn4PrCtqY,16
+dagster_airbyte-0.25.10.dist-info/RECORD,,

dagster_airbyte-0.25.1.dist-info/RECORD

@@ -1,21 +0,0 @@
-dagster_airbyte/__init__.py,sha256=DpJXhv73w1oKpWnXRRfItCTb61oEZR3FJzyMZzBgRfM,1231
-dagster_airbyte/asset_defs.py,sha256=UFzHdNRgFPNU8xqDGcD-ce-J9I82HIj3N2GN5tg8G_Y,46039
-dagster_airbyte/cli.py,sha256=HErteP1MjfHozKKSrznh0yAreKETbXp5NDHzXGsdvvE,425
-dagster_airbyte/ops.py,sha256=pq6mp7vN2wXgo3gJMuWaAcxTmfkZ7d1zWzPyL_auSEY,4208
-dagster_airbyte/py.typed,sha256=la67KBlbjXN-_-DfGNcdOcjYumVpKG_Tkw-8n5dnGB4,8
-dagster_airbyte/resources.py,sha256=FoHI_Eiw2F6tnI1KZMfkgXhThdU6DplHp6INsbjUFsg,29802
-dagster_airbyte/types.py,sha256=w1DyTcXyuzrG3wfkOPYFtwj7snHcgqf-dC7_pRjiE1Q,1544
-dagster_airbyte/utils.py,sha256=cFKCkGFAvwr17KFTeqpVtQRDsNo4zpqw9yr2-1YSJeI,2823
-dagster_airbyte/version.py,sha256=ACu2Z3Q3TFgYpAno_eu9ssJ1QULjNXvjGvyqDSHrQ_o,23
-dagster_airbyte/managed/__init__.py,sha256=6SBtyNOMJ9Cu2UIwFExJHpL_ZVFo3rPMvyIxVOsKvWE,469
-dagster_airbyte/managed/reconciliation.py,sha256=HgrLT-Xs8vWY9SfbdBXuorMf60KCn5Qz7bPITW5MxJo,34862
-dagster_airbyte/managed/types.py,sha256=ja056Wm7_ZFw1XGSNmdxmBy2TcOxbnylJCpRA2ng2TE,14596
-dagster_airbyte/managed/generated/__init__.py,sha256=eYq-yfXEeffuKAVFXY8plD0se1wHjFNVqklpbu9gljw,108
-dagster_airbyte/managed/generated/destinations.py,sha256=x1wmWlXvOJHtfaZva3ErdKuVS--sDvfidSXR5ji9G5w,119692
-dagster_airbyte/managed/generated/sources.py,sha256=wyNoGJiNvW8mjRRs6b-_lWFs0Fgy-MZlRaxiN6bP-4s,282691
-dagster_airbyte-0.25.1.dist-info/LICENSE,sha256=TMatHW4_G9ldRdodEAp-l2Xa2WvsdeOh60E3v1R2jis,11349
-dagster_airbyte-0.25.1.dist-info/METADATA,sha256=8sqSbzWGG-CW3k3psUXmZj955VZWzyMp-hZaneVeiro,876
-dagster_airbyte-0.25.1.dist-info/WHEEL,sha256=yQN5g4mg4AybRjkgi-9yy4iQEFibGQmlz78Pik5Or-A,92
-dagster_airbyte-0.25.1.dist-info/entry_points.txt,sha256=XrbLOz3LpgPV5fdwMmgdP6Rp1AfSG07KeWIddLqh7Lw,61
-dagster_airbyte-0.25.1.dist-info/top_level.txt,sha256=HLwIRQCzqItn88_KbPP8DNTKKQEBUVKk6NCn4PrCtqY,16
-dagster_airbyte-0.25.1.dist-info/RECORD,,