dagster-airbyte 0.24.3__py3-none-any.whl → 0.28.3__py3-none-any.whl

dagster_airbyte/translator.py

@@ -0,0 +1,255 @@
+from collections.abc import Mapping, Sequence
+from datetime import datetime
+from enum import Enum
+from typing import Any, Optional
+
+from dagster._annotations import beta, deprecated
+from dagster._core.definitions.asset_key import AssetKey
+from dagster._core.definitions.assets.definition.asset_spec import AssetSpec
+from dagster._core.definitions.metadata.metadata_set import NamespacedMetadataSet, TableMetadataSet
+from dagster._record import record
+from dagster._utils.cached_method import cached_method
+from dagster_shared.serdes import whitelist_for_serdes
+
+from dagster_airbyte.utils import generate_table_schema, get_airbyte_connection_table_name
+
+
+class AirbyteJobStatusType(str, Enum):
+    RUNNING = "running"
+    SUCCEEDED = "succeeded"
+    CANCELLED = "cancelled"
+    PENDING = "pending"
+    FAILED = "failed"
+    ERROR = "error"
+    INCOMPLETE = "incomplete"
+
+
+@deprecated(breaking_version="1.10", additional_warn_text="Use `AirbyteJobStatusType` instead.")
+class AirbyteState:
+    RUNNING = AirbyteJobStatusType.RUNNING
+    SUCCEEDED = AirbyteJobStatusType.SUCCEEDED
+    CANCELLED = AirbyteJobStatusType.CANCELLED
+    PENDING = AirbyteJobStatusType.PENDING
+    FAILED = AirbyteJobStatusType.FAILED
+    ERROR = AirbyteJobStatusType.ERROR
+    INCOMPLETE = AirbyteJobStatusType.INCOMPLETE
+
+
+@record
+class AirbyteConnectionTableProps:
+    table_name: str
+    stream_prefix: Optional[str]
+    stream_name: str
+    json_schema: Mapping[str, Any]
+    connection_id: str
+    connection_name: str
+    destination_type: Optional[str]
+    database: Optional[str]
+    schema: Optional[str]
+
+    @property
+    def fully_qualified_table_name(self) -> Optional[str]:
+        return (
+            f"{self.database}.{self.schema}.{self.stream_name}"
+            if self.database and self.schema
+            else None
+        )
+
+
+@whitelist_for_serdes
+@record
+class AirbyteConnection:
+    """Represents an Airbyte connection, based on data as returned from the API."""
+
+    id: str
+    name: str
+    stream_prefix: Optional[str]
+    streams: Mapping[str, "AirbyteStream"]
+    destination_id: str
+
+    @classmethod
+    def from_connection_details(
+        cls,
+        connection_details: Mapping[str, Any],
+    ) -> "AirbyteConnection":
+        return cls(
+            id=connection_details["connectionId"],
+            name=connection_details["name"],
+            stream_prefix=connection_details.get("prefix"),
+            streams={
+                stream_details["stream"]["name"]: AirbyteStream.from_stream_details(
+                    stream_details=stream_details
+                )
+                for stream_details in connection_details.get("syncCatalog", {}).get("streams", [])
+            },
+            destination_id=connection_details["destinationId"],
+        )
+
+
+@whitelist_for_serdes
+@record
+class AirbyteDestination:
+    """Represents an Airbyte destination, based on data as returned from the API."""
+
+    id: str
+    type: str
+    database: Optional[str]
+    schema: Optional[str]
+
+    @classmethod
+    def from_destination_details(
+        cls,
+        destination_details: Mapping[str, Any],
+    ) -> "AirbyteDestination":
+        return cls(
+            id=destination_details["destinationId"],
+            type=destination_details["destinationType"],
+            database=destination_details["configuration"].get("database"),
+            schema=destination_details["configuration"].get("schema"),
+        )
+
+
+@whitelist_for_serdes
+@record
+class AirbyteStream:
+    """Represents an Airbyte stream, based on data as returned from the API.
+    A stream in Airbyte corresponds to a table.
+    """
+
+    name: str
+    selected: bool
+    json_schema: Mapping[str, Any]
+
+    @classmethod
+    def from_stream_details(
+        cls,
+        stream_details: Mapping[str, Any],
+    ) -> "AirbyteStream":
+        return cls(
+            name=stream_details["stream"]["name"],
+            selected=stream_details["config"].get("selected", False),
+            json_schema=stream_details["stream"].get("jsonSchema", {}),
+        )
+
+
+@whitelist_for_serdes
+@record
+class AirbyteJob:
+    """Represents an Airbyte job, based on data as returned from the API."""
+
+    id: int
+    status: str
+    type: str
+    connection_id: str | None = None
+    start_time: datetime | None = None
+    last_updated_at: datetime | None = None
+    duration: str | None = None
+    bytes_synced: int | None = None
+    rows_synced: int | None = None
+
+    @classmethod
+    def from_job_details(
+        cls,
+        job_details: Mapping[str, Any],
+    ) -> "AirbyteJob":
+        return cls(
+            id=job_details["jobId"],
+            status=job_details["status"],
+            type=job_details["jobType"],
+            connection_id=job_details.get("connectionId"),
+            start_time=datetime.fromisoformat(job_details["startTime"])
+            if "startTime" in job_details
+            else None,
+            last_updated_at=datetime.fromisoformat(job_details["lastUpdatedAt"])
+            if "lastUpdatedAt" in job_details
+            else None,
+            duration=job_details.get("duration"),
+            bytes_synced=job_details.get("bytesScanned"),
+            rows_synced=job_details.get("rowsSynced"),
+        )
+
+
+@whitelist_for_serdes
+@record
+class AirbyteWorkspaceData:
+    """A record representing all content in an Airbyte workspace.
+    This applies to both Airbyte OSS and Cloud.
+    """
+
+    connections_by_id: Mapping[str, AirbyteConnection]
+    destinations_by_id: Mapping[str, AirbyteDestination]
+
+    @cached_method
+    def to_airbyte_connection_table_props_data(self) -> Sequence[AirbyteConnectionTableProps]:
+        """Method that converts a `AirbyteWorkspaceData` object
+        to a collection of `AirbyteConnectionTableProps` objects.
+        """
+        data: list[AirbyteConnectionTableProps] = []
+
+        for connection in self.connections_by_id.values():
+            destination = self.destinations_by_id[connection.destination_id]
+
+            for stream in connection.streams.values():
+                if stream.selected:
+                    data.append(
+                        AirbyteConnectionTableProps(
+                            table_name=get_airbyte_connection_table_name(
+                                stream_prefix=connection.stream_prefix,
+                                stream_name=stream.name,
+                            ),
+                            stream_prefix=connection.stream_prefix,
+                            stream_name=stream.name,
+                            json_schema=stream.json_schema,
+                            connection_id=connection.id,
+                            connection_name=connection.name,
+                            destination_type=destination.type,
+                            database=destination.database,
+                            schema=destination.schema,
+                        )
+                    )
+
+        return data
+
+
+class AirbyteMetadataSet(NamespacedMetadataSet):
+    connection_id: str
+    connection_name: str
+    stream_prefix: Optional[str] = None
+
+    @classmethod
+    def namespace(cls) -> str:
+        return "dagster-airbyte"
+
+
+@beta
+class DagsterAirbyteTranslator:
+    """Translator class which converts a `AirbyteConnectionTableProps` object into AssetSpecs.
+    Subclass this class to implement custom logic how to translate Airbyte content into asset spec.
+    """
+
+    def get_asset_spec(self, props: AirbyteConnectionTableProps) -> AssetSpec:
+        """Get the AssetSpec for a table synced by an Airbyte connection."""
+        table_schema_props = (
+            props.json_schema.get("properties")
+            or props.json_schema.get("items", {}).get("properties")
+            or {}
+        )
+        column_schema = generate_table_schema(table_schema_props)
+
+        metadata = {
+            **TableMetadataSet(
+                column_schema=column_schema,
+                table_name=props.fully_qualified_table_name,
+            ),
+            **AirbyteMetadataSet(
+                connection_id=props.connection_id,
+                connection_name=props.connection_name,
+                stream_prefix=props.stream_prefix,
+            ),
+        }
+
+        return AssetSpec(
+            key=AssetKey(props.table_name),
+            metadata=metadata,
+            kinds={"airbyte", *({props.destination_type} if props.destination_type else set())},
+        )

dagster_airbyte/types.py

@@ -1,4 +1,5 @@
-from typing import Any, Mapping, NamedTuple, Optional
+from collections.abc import Mapping
+from typing import Any, NamedTuple, Optional
 
 from dagster._core.definitions.metadata.table import TableSchema
 
@@ -6,10 +7,14 @@ from dagster._core.definitions.metadata.table import TableSchema
 class AirbyteTableMetadata:
     def __init__(
         self,
+        raw_table_name: str,
         schema: TableSchema,
         normalization_tables: Optional[Mapping[str, "AirbyteTableMetadata"]] = None,
     ):
-        """Contains metadata about an Airbyte table, including its schema and any created normalization tables."""
+        """Contains metadata about an Airbyte table, including its destination raw table name,
+        schema and any created normalization tables.
+        """
+        self.raw_table_name = raw_table_name
         self.schema = schema
         self.normalization_tables = normalization_tables or dict()
 
@@ -25,7 +30,7 @@ class AirbyteOutput(
 ):
     """Contains recorded information about the state of a Airbyte connection job after a sync completes.
 
-    Attributes:
+    Args:
         job_details (Dict[str, Any]):
             The raw Airbyte API response containing the details of the initiated job. For info
             on the schema of this dictionary, see: https://airbyte-public-api-docs.s3.us-east-2.amazonaws.com/rapidoc-api-docs.html#post-/v1/jobs/get

dagster_airbyte/utils.py

@@ -1,10 +1,29 @@
-from typing import Any, Iterator, Mapping, Optional, Sequence
+from collections.abc import Iterator, Mapping, Sequence
+from typing import TYPE_CHECKING, Any, Optional
 
-from dagster import AssetMaterialization, MetadataValue
+from dagster import (
+    AssetMaterialization,
+    AssetsDefinition,
+    DagsterInvariantViolationError,
+    MetadataValue,
+)
 from dagster._core.definitions.metadata.table import TableColumn, TableSchema
+from dagster._utils.names import clean_name_lower
 
 from dagster_airbyte.types import AirbyteOutput
 
+if TYPE_CHECKING:
+    from dagster_airbyte.translator import DagsterAirbyteTranslator
+
+DAGSTER_AIRBYTE_TRANSLATOR_METADATA_KEY = "dagster-airbyte/dagster_airbyte_translator"
+
+
+clean_name = clean_name_lower
+
+
+def get_airbyte_connection_table_name(stream_prefix: Optional[str], stream_name: str) -> str:
+    return f"{stream_prefix if stream_prefix else ''}{stream_name}"
+
 
 def generate_table_schema(stream_schema_props: Mapping[str, Any]) -> TableSchema:
     return TableSchema(
@@ -74,3 +93,18 @@ def generate_materializations(
             all_stream_stats.get(stream_name, {}),
             asset_key_prefix=asset_key_prefix,
         )
+
+
+def get_translator_from_airbyte_assets(
+    airbyte_assets: AssetsDefinition,
+) -> "DagsterAirbyteTranslator":
+    metadata_by_key = airbyte_assets.metadata_by_key or {}
+    first_asset_key = next(iter(airbyte_assets.metadata_by_key.keys()))
+    first_metadata = metadata_by_key.get(first_asset_key, {})
+    dagster_airbyte_translator = first_metadata.get(DAGSTER_AIRBYTE_TRANSLATOR_METADATA_KEY)
+    if dagster_airbyte_translator is None:
+        raise DagsterInvariantViolationError(
+            f"Expected to find airbyte translator metadata on asset {first_asset_key.to_user_string()},"
+            " but did not. Did you pass in assets that weren't generated by @airbyte_assets?"
+        )
+    return dagster_airbyte_translator

dagster_airbyte/version.py

@@ -1 +1 @@
-__version__ = "0.24.3"
+__version__ = "0.28.3"

{dagster_airbyte-0.24.3.dist-info → dagster_airbyte-0.28.3.dist-info}/METADATA

@@ -1,24 +1,33 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: dagster-airbyte
-Version: 0.24.3
+Version: 0.28.3
 Summary: Package for integrating Airbyte with Dagster.
 Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-airbyte
 Author: Dagster Labs
 Author-email: hello@dagsterlabs.com
 License: Apache-2.0
-Classifier: Programming Language :: Python :: 3.8
-Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Operating System :: OS Independent
-Requires-Python: >=3.8,<3.13
+Requires-Python: >=3.10,<3.14
 License-File: LICENSE
-Requires-Dist: dagster ==1.8.3
+Requires-Dist: dagster==1.12.3
 Requires-Dist: requests
-Provides-Extra: managed
-Requires-Dist: dagster-managed-elements ==0.24.3 ; extra == 'managed'
 Provides-Extra: test
-Requires-Dist: requests-mock ; extra == 'test'
-
+Requires-Dist: requests-mock; extra == "test"
+Requires-Dist: flaky; extra == "test"
+Provides-Extra: managed
+Requires-Dist: dagster-managed-elements==0.28.3; extra == "managed"
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: home-page
+Dynamic: license
+Dynamic: license-file
+Dynamic: provides-extra
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary

dagster_airbyte-0.28.3.dist-info/RECORD

@@ -0,0 +1,28 @@
+dagster_airbyte/__init__.py,sha256=UO5Gf6M6q1c7bbL5uW6NH9HuNkaH2y_xVfqMY9tN7xk,2073
+dagster_airbyte/asset_decorator.py,sha256=bCA22GG8ch5eGmm6BkztIfTY6ROXWJq1AInzO7fwJ2M,4667
+dagster_airbyte/asset_defs.py,sha256=8yDuE2r1_4IAl9bY7BkbJpINWgJjtPY1t_WZeedGZho,51909
+dagster_airbyte/cli.py,sha256=HErteP1MjfHozKKSrznh0yAreKETbXp5NDHzXGsdvvE,425
+dagster_airbyte/legacy_resources.py,sha256=ks5doF0YZr7nXeU6ibQh1qiIMPAd_c2jusOkqdZKysc,31125
+dagster_airbyte/ops.py,sha256=zDuROSIpqMhiioycTpLRgGnrp71lZNd4ts6-yybPg_s,4273
+dagster_airbyte/py.typed,sha256=la67KBlbjXN-_-DfGNcdOcjYumVpKG_Tkw-8n5dnGB4,8
+dagster_airbyte/resources.py,sha256=Nb-cPHYtkzYjmIb7c0onA9hYLpe5JLcEjbXQf9_3Cf4,41654
+dagster_airbyte/translator.py,sha256=-aT-Au1VqaqHnRvivOaHslK2C6db5eNKB3mAJbQwvw8,8364
+dagster_airbyte/types.py,sha256=TYUjI3skjLYeANjesgJ-IAJNu8bAnL1ymsUfz5LsRTE,1565
+dagster_airbyte/utils.py,sha256=wG9119kXi87JgcOjK7iNozr-svZocJBQYoHBMmnXZcE,4092
+dagster_airbyte/version.py,sha256=PFj1PlFYtLhN7k4TLEHMneuQ4Ep8o7DCrl69TL8I0SY,23
+dagster_airbyte/components/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+dagster_airbyte/components/workspace_component/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+dagster_airbyte/components/workspace_component/component.py,sha256=yTc_uzU5--kbjfDzE0TbVHWqqrRu3kh1C-YzN-xWg34,16208
+dagster_airbyte/components/workspace_component/scaffolder.py,sha256=zk5HDJ6C8zcfUWKslJVyqyBB0LV3JzNAocY2ptT517s,1046
+dagster_airbyte/managed/__init__.py,sha256=6SBtyNOMJ9Cu2UIwFExJHpL_ZVFo3rPMvyIxVOsKvWE,469
+dagster_airbyte/managed/reconciliation.py,sha256=V9bNMwNRvyIXF0O3cSaPlw3CmpbVaWQVEQarhrXf7n0,34866
+dagster_airbyte/managed/types.py,sha256=isPfX8L9YwtZAf9Vk4hhxBePLR00AEldsdK2TsM1H2o,14611
+dagster_airbyte/managed/generated/__init__.py,sha256=eYq-yfXEeffuKAVFXY8plD0se1wHjFNVqklpbu9gljw,108
+dagster_airbyte/managed/generated/destinations.py,sha256=x1wmWlXvOJHtfaZva3ErdKuVS--sDvfidSXR5ji9G5w,119692
+dagster_airbyte/managed/generated/sources.py,sha256=y0TPNvcRd8c9mhje-NoXsHeKRPt1nXcpww8mNAtqCps,282685
+dagster_airbyte-0.28.3.dist-info/licenses/LICENSE,sha256=4lsMW-RCvfVD4_F57wrmpe3vX1xwUk_OAKKmV_XT7Z0,11348
+dagster_airbyte-0.28.3.dist-info/METADATA,sha256=92qHM2fBXVeMMibGM-YiIvuDqe43_ZwxmMrAuKJByUI,1117
+dagster_airbyte-0.28.3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+dagster_airbyte-0.28.3.dist-info/entry_points.txt,sha256=096yvfMP-gNsCgDg9vDQtinis5QGpD-e_kHEhcHaML8,120
+dagster_airbyte-0.28.3.dist-info/top_level.txt,sha256=HLwIRQCzqItn88_KbPP8DNTKKQEBUVKk6NCn4PrCtqY,16
+dagster_airbyte-0.28.3.dist-info/RECORD,,

{dagster_airbyte-0.24.3.dist-info → dagster_airbyte-0.28.3.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.41.2)
+Generator: setuptools (80.9.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

{dagster_airbyte-0.24.3.dist-info → dagster_airbyte-0.28.3.dist-info}/entry_points.txt

@@ -1,2 +1,5 @@
 [console_scripts]
 dagster-airbyte = dagster_airbyte.cli:main
+
+[dagster_dg_cli.plugin]
+dagster_airbyte = dagster_airbyte

{dagster_airbyte-0.24.3.dist-info → dagster_airbyte-0.28.3.dist-info/licenses}/LICENSE

@@ -186,7 +186,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright 2023 Dagster Labs, Inc".
+   Copyright 2025 Dagster Labs, Inc.
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

dagster_airbyte-0.24.3.dist-info/RECORD

@@ -1,21 +0,0 @@
-dagster_airbyte/__init__.py,sha256=KRUcHzOxDEmo9oO7qRwJNrRnxK28PeHTeUemMS4z2Gc,1215
-dagster_airbyte/asset_defs.py,sha256=7PziNgIbN9xQwRjapygGvdWXI27p-ig6cdq6sRZ40W0,48511
-dagster_airbyte/cli.py,sha256=HErteP1MjfHozKKSrznh0yAreKETbXp5NDHzXGsdvvE,425
-dagster_airbyte/ops.py,sha256=fmZ7u223streZfx5Cwo0a-HhkWJVhBWiGsmwY9XXso8,4194
-dagster_airbyte/py.typed,sha256=la67KBlbjXN-_-DfGNcdOcjYumVpKG_Tkw-8n5dnGB4,8
-dagster_airbyte/resources.py,sha256=FoHI_Eiw2F6tnI1KZMfkgXhThdU6DplHp6INsbjUFsg,29802
-dagster_airbyte/types.py,sha256=fwqUv_MZCegwHhSELgUqm1H1JVUb-m83CyXFkd0r-ko,1425
-dagster_airbyte/utils.py,sha256=cFKCkGFAvwr17KFTeqpVtQRDsNo4zpqw9yr2-1YSJeI,2823
-dagster_airbyte/version.py,sha256=FVT3zgMnGxhctio1D7Bj2hvIqrqQQ-a9tvDQYKSSekk,23
-dagster_airbyte/managed/__init__.py,sha256=azuKhCI8eUsmEGBL2M2d6qkqQ0vhsV9H5IioP83kbm8,423
-dagster_airbyte/managed/reconciliation.py,sha256=HgrLT-Xs8vWY9SfbdBXuorMf60KCn5Qz7bPITW5MxJo,34862
-dagster_airbyte/managed/types.py,sha256=ja056Wm7_ZFw1XGSNmdxmBy2TcOxbnylJCpRA2ng2TE,14596
-dagster_airbyte/managed/generated/__init__.py,sha256=1ChWhXU1uyX_9rClMD6XHzkriKOc0fjIQntsoI3G99g,76
-dagster_airbyte/managed/generated/destinations.py,sha256=x1wmWlXvOJHtfaZva3ErdKuVS--sDvfidSXR5ji9G5w,119692
-dagster_airbyte/managed/generated/sources.py,sha256=e360ITeAECzqEvHw04onfBWciMntjR2IhoxOd5ibno4,282709
-dagster_airbyte-0.24.3.dist-info/LICENSE,sha256=TMatHW4_G9ldRdodEAp-l2Xa2WvsdeOh60E3v1R2jis,11349
-dagster_airbyte-0.24.3.dist-info/METADATA,sha256=5cqL9WoNn1ldQGtXnAJ7apgw-bffD_jAE6X9cGTsqeY,926
-dagster_airbyte-0.24.3.dist-info/WHEEL,sha256=yQN5g4mg4AybRjkgi-9yy4iQEFibGQmlz78Pik5Or-A,92
-dagster_airbyte-0.24.3.dist-info/entry_points.txt,sha256=XrbLOz3LpgPV5fdwMmgdP6Rp1AfSG07KeWIddLqh7Lw,61
-dagster_airbyte-0.24.3.dist-info/top_level.txt,sha256=HLwIRQCzqItn88_KbPP8DNTKKQEBUVKk6NCn4PrCtqY,16
-dagster_airbyte-0.24.3.dist-info/RECORD,,