dagster-airbyte 0.23.16__py3-none-any.whl → 0.24.1__py3-none-any.whl

dagster_airbyte/py.typed

@@ -0,0 +1 @@
+partial

dagster_airbyte/resources.py

@@ -5,12 +5,14 @@ import sys
 import time
 from abc import abstractmethod
 from contextlib import contextmanager
+from datetime import datetime, timedelta
 from typing import Any, Dict, List, Mapping, Optional, cast
 
 import requests
 from dagster import (
     ConfigurableResource,
     Failure,
+    InitResourceContext,
     _check as check,
     get_dagster_logger,
     resource,
@@ -19,13 +21,17 @@ from dagster._config.pythonic_config import infer_schema_from_config_class
 from dagster._core.definitions.resource_definition import dagster_maintained_resource
 from dagster._utils.cached_method import cached_method
 from dagster._utils.merger import deep_merge_dicts
-from pydantic import Field
+from pydantic import Field, PrivateAttr
 from requests.exceptions import RequestException
 
 from dagster_airbyte.types import AirbyteOutput
 
 DEFAULT_POLL_INTERVAL_SECONDS = 10
 
+# The access token expire every 3 minutes in Airbyte Cloud.
+# Refresh is needed after 2.5 minutes to avoid the "token expired" error message.
+AIRBYTE_CLOUD_REFRESH_TIMEDELTA_SECONDS = 150
+
 
 class AirbyteState:
     RUNNING = "running"
@@ -94,7 +100,11 @@ class BaseAirbyteResource(ConfigurableResource):
         raise NotImplementedError()
 
     def make_request(
-        self, endpoint: str, data: Optional[Mapping[str, object]] = None, method: str = "POST"
+        self,
+        endpoint: str,
+        data: Optional[Mapping[str, object]] = None,
+        method: str = "POST",
+        include_additional_request_params: bool = True,
     ) -> Optional[Mapping[str, object]]:
         """Creates and sends a request to the desired Airbyte REST API endpoint.
 
@@ -120,10 +130,11 @@ class BaseAirbyteResource(ConfigurableResource):
                 if data:
                     request_args["json"] = data
 
-                request_args = deep_merge_dicts(
-                    request_args,
-                    self.all_additional_request_params,
-                )
+                if include_additional_request_params:
+                    request_args = deep_merge_dicts(
+                        request_args,
+                        self.all_additional_request_params,
+                    )
 
                 response = requests.request(
                     **request_args,
@@ -244,7 +255,7 @@ class BaseAirbyteResource(ConfigurableResource):
 
 
 class AirbyteCloudResource(BaseAirbyteResource):
-    """This resource allows users to programatically interface with the Airbyte Cloud API to launch
+    """This resource allows users to programmatically interface with the Airbyte Cloud API to launch
     syncs and monitor their progress.
 
     **Examples:**
@@ -255,7 +266,8 @@ class AirbyteCloudResource(BaseAirbyteResource):
         from dagster_airbyte import AirbyteResource
 
         my_airbyte_resource = AirbyteCloudResource(
-            api_key=EnvVar("AIRBYTE_API_KEY"),
+            client_id=EnvVar("AIRBYTE_CLIENT_ID"),
+            client_secret=EnvVar("AIRBYTE_CLIENT_SECRET"),
         )
 
         airbyte_assets = build_airbyte_assets(
@@ -269,7 +281,15 @@ class AirbyteCloudResource(BaseAirbyteResource):
         )
     """
 
-    api_key: str = Field(..., description="The Airbyte Cloud API key.")
+    client_id: str = Field(..., description="The Airbyte Cloud client ID.")
+    client_secret: str = Field(..., description="The Airbyte Cloud client secret.")
+
+    _access_token_value: Optional[str] = PrivateAttr(default=None)
+    _access_token_timestamp: Optional[float] = PrivateAttr(default=None)
+
+    def setup_for_execution(self, context: InitResourceContext) -> None:
+        # Refresh access token when the resource is initialized
+        self._refresh_access_token()
 
     @property
     def api_base_url(self) -> str:
@@ -277,7 +297,32 @@ class AirbyteCloudResource(BaseAirbyteResource):
 
     @property
     def all_additional_request_params(self) -> Mapping[str, Any]:
-        return {"headers": {"Authorization": f"Bearer {self.api_key}", "User-Agent": "dagster"}}
+        # Make sure the access token is refreshed before using it when calling the API.
+        if self._needs_refreshed_access_token():
+            self._refresh_access_token()
+        return {
+            "headers": {
+                "Authorization": f"Bearer {self._access_token_value}",
+                "User-Agent": "dagster",
+            }
+        }
+
+    def make_request(
+        self,
+        endpoint: str,
+        data: Optional[Mapping[str, object]] = None,
+        method: str = "POST",
+        include_additional_request_params: bool = True,
+    ) -> Optional[Mapping[str, object]]:
+        # Make sure the access token is refreshed before using it when calling the API.
+        if include_additional_request_params and self._needs_refreshed_access_token():
+            self._refresh_access_token()
+        return super().make_request(
+            endpoint=endpoint,
+            data=data,
+            method=method,
+            include_additional_request_params=include_additional_request_params,
+        )
 
     def start_sync(self, connection_id: str) -> Mapping[str, object]:
         job_sync = check.not_none(
@@ -306,6 +351,31 @@ class AirbyteCloudResource(BaseAirbyteResource):
         # Airbyte Cloud does not support streaming logs yet
         return False
 
+    def _refresh_access_token(self) -> None:
+        response = check.not_none(
+            self.make_request(
+                endpoint="/applications/token",
+                data={
+                    "client_id": self.client_id,
+                    "client_secret": self.client_secret,
+                },
+                # Must not pass the bearer access token when refreshing it.
+                include_additional_request_params=False,
+            )
+        )
+        self._access_token_value = str(response["access_token"])
+        self._access_token_timestamp = datetime.now().timestamp()
+
+    def _needs_refreshed_access_token(self) -> bool:
+        return (
+            not self._access_token_value
+            or not self._access_token_timestamp
+            or self._access_token_timestamp
+            <= datetime.timestamp(
+                datetime.now() - timedelta(seconds=AIRBYTE_CLOUD_REFRESH_TIMEDELTA_SECONDS)
+            )
+        )
+
 
 class AirbyteResource(BaseAirbyteResource):
     """This resource allows users to programatically interface with the Airbyte REST API to launch

dagster_airbyte/version.py

@@ -1 +1 @@
-__version__ = "0.23.16"
+__version__ = "0.24.1"

{dagster_airbyte-0.23.16.dist-info → dagster_airbyte-0.24.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dagster-airbyte
-Version: 0.23.16
+Version: 0.24.1
 Summary: Package for integrating Airbyte with Dagster.
 Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-airbyte
 Author: Dagster Labs
@@ -15,10 +15,10 @@ Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Operating System :: OS Independent
 Requires-Python: >=3.8,<3.13
 License-File: LICENSE
-Requires-Dist: dagster ==1.7.16
+Requires-Dist: dagster ==1.8.1
 Requires-Dist: requests
 Provides-Extra: managed
-Requires-Dist: dagster-managed-elements ==0.23.16 ; extra == 'managed'
+Requires-Dist: dagster-managed-elements ==0.24.1 ; extra == 'managed'
 Provides-Extra: test
 Requires-Dist: requests-mock ; extra == 'test'
 

{dagster_airbyte-0.23.16.dist-info → dagster_airbyte-0.24.1.dist-info}/RECORD

@@ -2,19 +2,20 @@ dagster_airbyte/__init__.py,sha256=KRUcHzOxDEmo9oO7qRwJNrRnxK28PeHTeUemMS4z2Gc,1
 dagster_airbyte/asset_defs.py,sha256=7PziNgIbN9xQwRjapygGvdWXI27p-ig6cdq6sRZ40W0,48511
 dagster_airbyte/cli.py,sha256=HErteP1MjfHozKKSrznh0yAreKETbXp5NDHzXGsdvvE,425
 dagster_airbyte/ops.py,sha256=fmZ7u223streZfx5Cwo0a-HhkWJVhBWiGsmwY9XXso8,4194
-dagster_airbyte/resources.py,sha256=ZRRKc1DwKluK6exEKkJ_FWIG5TKyTDOg1zt83TEM3Bg,27040
+dagster_airbyte/py.typed,sha256=la67KBlbjXN-_-DfGNcdOcjYumVpKG_Tkw-8n5dnGB4,8
+dagster_airbyte/resources.py,sha256=FoHI_Eiw2F6tnI1KZMfkgXhThdU6DplHp6INsbjUFsg,29802
 dagster_airbyte/types.py,sha256=fwqUv_MZCegwHhSELgUqm1H1JVUb-m83CyXFkd0r-ko,1425
 dagster_airbyte/utils.py,sha256=cFKCkGFAvwr17KFTeqpVtQRDsNo4zpqw9yr2-1YSJeI,2823
-dagster_airbyte/version.py,sha256=M0iweVcoG5aVc8DYn-pqd24UZQKlVsdhqQB4VFCL5Es,24
+dagster_airbyte/version.py,sha256=23Q5wqrB_jTfEuw8bupXFhwqZO0_-oiISvjn1gCVUsA,23
 dagster_airbyte/managed/__init__.py,sha256=azuKhCI8eUsmEGBL2M2d6qkqQ0vhsV9H5IioP83kbm8,423
 dagster_airbyte/managed/reconciliation.py,sha256=HgrLT-Xs8vWY9SfbdBXuorMf60KCn5Qz7bPITW5MxJo,34862
 dagster_airbyte/managed/types.py,sha256=ja056Wm7_ZFw1XGSNmdxmBy2TcOxbnylJCpRA2ng2TE,14596
 dagster_airbyte/managed/generated/__init__.py,sha256=1ChWhXU1uyX_9rClMD6XHzkriKOc0fjIQntsoI3G99g,76
 dagster_airbyte/managed/generated/destinations.py,sha256=x1wmWlXvOJHtfaZva3ErdKuVS--sDvfidSXR5ji9G5w,119692
 dagster_airbyte/managed/generated/sources.py,sha256=e360ITeAECzqEvHw04onfBWciMntjR2IhoxOd5ibno4,282709
-dagster_airbyte-0.23.16.dist-info/LICENSE,sha256=TMatHW4_G9ldRdodEAp-l2Xa2WvsdeOh60E3v1R2jis,11349
-dagster_airbyte-0.23.16.dist-info/METADATA,sha256=xtbyfw3LT0aB-qENO_ioqaQxabpMOi1ZJPiV2As-Pdg,929
-dagster_airbyte-0.23.16.dist-info/WHEEL,sha256=yQN5g4mg4AybRjkgi-9yy4iQEFibGQmlz78Pik5Or-A,92
-dagster_airbyte-0.23.16.dist-info/entry_points.txt,sha256=XrbLOz3LpgPV5fdwMmgdP6Rp1AfSG07KeWIddLqh7Lw,61
-dagster_airbyte-0.23.16.dist-info/top_level.txt,sha256=HLwIRQCzqItn88_KbPP8DNTKKQEBUVKk6NCn4PrCtqY,16
-dagster_airbyte-0.23.16.dist-info/RECORD,,
+dagster_airbyte-0.24.1.dist-info/LICENSE,sha256=TMatHW4_G9ldRdodEAp-l2Xa2WvsdeOh60E3v1R2jis,11349
+dagster_airbyte-0.24.1.dist-info/METADATA,sha256=B7cPCw8Oeh20-PnZq8RdbbxbdUodGAOtmh4JdBRqbIM,926
+dagster_airbyte-0.24.1.dist-info/WHEEL,sha256=yQN5g4mg4AybRjkgi-9yy4iQEFibGQmlz78Pik5Or-A,92
+dagster_airbyte-0.24.1.dist-info/entry_points.txt,sha256=XrbLOz3LpgPV5fdwMmgdP6Rp1AfSG07KeWIddLqh7Lw,61
+dagster_airbyte-0.24.1.dist-info/top_level.txt,sha256=HLwIRQCzqItn88_KbPP8DNTKKQEBUVKk6NCn4PrCtqY,16
+dagster_airbyte-0.24.1.dist-info/RECORD,,