airbyte-cdk 0.62.0__py3-none-any.whl → 0.62.2__py3-none-any.whl

airbyte_cdk/sources/connector_state_manager.py

@@ -38,7 +38,7 @@ class ConnectorStateManager:
 
     def __init__(
         self,
-        stream_instance_map: Mapping[str, AirbyteStream],
+        stream_instance_map: Mapping[str, Union[Stream, AirbyteStream]],
         state: Optional[Union[List[AirbyteStateMessage], MutableMapping[str, Any]]] = None,
     ):
         shared_state, per_stream_states = self._extract_from_state_message(state, stream_instance_map)
@@ -107,7 +107,9 @@ class ConnectorStateManager:
 
     @classmethod
     def _extract_from_state_message(
-        cls, state: Optional[Union[List[AirbyteStateMessage], MutableMapping[str, Any]]], stream_instance_map: Mapping[str, AirbyteStream]
+        cls,
+        state: Optional[Union[List[AirbyteStateMessage], MutableMapping[str, Any]]],
+        stream_instance_map: Mapping[str, Union[Stream, AirbyteStream]],
     ) -> Tuple[Optional[AirbyteStateBlob], MutableMapping[HashableStreamDescriptor, Optional[AirbyteStateBlob]]]:
         """
         Takes an incoming list of state messages or the legacy state format and extracts state attributes according to type
@@ -159,7 +161,7 @@ class ConnectorStateManager:
 
     @staticmethod
     def _create_descriptor_to_stream_state_mapping(
-        state: MutableMapping[str, Any], stream_to_instance_map: Mapping[str, Stream]
+        state: MutableMapping[str, Any], stream_to_instance_map: Mapping[str, Union[Stream, AirbyteStream]]
     ) -> MutableMapping[HashableStreamDescriptor, Optional[AirbyteStateBlob]]:
         """
         Takes incoming state received in the legacy format and transforms it into a mapping of StreamDescriptor to AirbyteStreamState

airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py

@@ -14,6 +14,7 @@ from airbyte_cdk.models import FailureType, Level
 from airbyte_cdk.sources.http_logger import format_http_message
 from airbyte_cdk.sources.message import MessageRepository, NoopMessageRepository
 from airbyte_cdk.utils import AirbyteTracedException
+from airbyte_cdk.utils.airbyte_secrets_utils import add_to_secrets
 from requests.auth import AuthBase
 
 from ..exceptions import DefaultBackoffException
@@ -115,9 +116,20 @@ class AbstractOauth2Authenticator(AuthBase):
     def _get_refresh_access_token_response(self) -> Any:
         try:
             response = requests.request(method="POST", url=self.get_token_refresh_endpoint(), data=self.build_refresh_request_body())
-            self._log_response(response)
-            response.raise_for_status()
-            return response.json()
+            if response.ok:
+                response_json = response.json()
+                # Add the access token to the list of secrets so it is replaced before logging the response
+                # An argument could be made to remove the prevous access key from the list of secrets, but unmasking values seems like a security incident waiting to happen...
+                access_key = response_json.get(self.get_access_token_name())
+                if not access_key:
+                    raise Exception("Token refresh API response was missing access token {self.get_access_token_name()}")
+                add_to_secrets(access_key)
+                self._log_response(response)
+                return response_json
+            else:
+                # log the response even if the request failed for troubleshooting purposes
+                self._log_response(response)
+                response.raise_for_status()
         except requests.exceptions.RequestException as e:
             if e.response is not None:
                 if e.response.status_code == 429 or e.response.status_code >= 500:

airbyte_cdk/test/mock_http/mocker.py

@@ -60,7 +60,9 @@ class HttpMocker(contextlib.ContextDecorator):
         getattr(self._mocker, method)(
             requests_mock.ANY,
             additional_matcher=self._matches_wrapper(matcher),
-            response_list=[{"text": response.body, "status_code": response.status_code} for response in responses],
+            response_list=[
+                {"text": response.body, "status_code": response.status_code, "headers": response.headers} for response in responses
+            ],
         )
 
     def get(self, request: HttpRequest, responses: Union[HttpResponse, List[HttpResponse]]) -> None:

airbyte_cdk/test/mock_http/response.py

@@ -1,10 +1,14 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 
+from types import MappingProxyType
+from typing import Mapping
+
 
 class HttpResponse:
-    def __init__(self, body: str, status_code: int = 200):
+    def __init__(self, body: str, status_code: int = 200, headers: Mapping[str, str] = MappingProxyType({})):
         self._body = body
         self._status_code = status_code
+        self._headers = headers
 
     @property
     def body(self) -> str:
@@ -13,3 +17,7 @@ class HttpResponse:
     @property
     def status_code(self) -> int:
         return self._status_code
+
+    @property
+    def headers(self) -> Mapping[str, str]:
+        return self._headers

airbyte_cdk/utils/airbyte_secrets_utils.py

@@ -10,7 +10,7 @@ import dpath.util
 def get_secret_paths(spec: Mapping[str, Any]) -> List[List[str]]:
     paths = []
 
-    def traverse_schema(schema_item: Any, path: List[str]):
+    def traverse_schema(schema_item: Any, path: List[str]) -> None:
         """
         schema_item can be any property or value in the originally input jsonschema, depending on how far down the recursion stack we go
         path is the path to that schema item in the original input
@@ -56,12 +56,18 @@ def get_secrets(connection_specification: Mapping[str, Any], config: Mapping[str
 __SECRETS_FROM_CONFIG: List[str] = []
 
 
-def update_secrets(secrets: List[str]):
+def update_secrets(secrets: List[str]) -> None:
     """Update the list of secrets to be replaced"""
     global __SECRETS_FROM_CONFIG
     __SECRETS_FROM_CONFIG = secrets
 
 
+def add_to_secrets(secret: str) -> None:
+    """Add to the list of secrets to be replaced"""
+    global __SECRETS_FROM_CONFIG
+    __SECRETS_FROM_CONFIG.append(secret)
+
+
 def filter_secrets(string: str) -> str:
     """Filter secrets from a string by replacing them with ****"""
     # TODO this should perform a maximal match for each secret. if "x" and "xk" are both secret values, and this method is called twice on

{airbyte_cdk-0.62.0.dist-info → airbyte_cdk-0.62.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 0.62.0
+Version: 0.62.2
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://github.com/airbytehq/airbyte
 Author: Airbyte

{airbyte_cdk-0.62.0.dist-info → airbyte_cdk-0.62.2.dist-info}/RECORD

@@ -26,7 +26,7 @@ airbyte_cdk/models/well_known_types.py,sha256=KKfNbow2gdLoC1Z4hcXy_JR8m_acsB2ol7
 airbyte_cdk/sources/__init__.py,sha256=Ov7Uf03KPSZUmMZqZfUAK3tQwsdKjDQUDvTb-H0JyfA,1141
 airbyte_cdk/sources/abstract_source.py,sha256=GSpNwbwJ0v-KvxWa0u_nWeC0r6G2fZNkpKUhXzf6YlI,14399
 airbyte_cdk/sources/config.py,sha256=PYsY7y2u3EUwxLiEb96JnuKwH_E8CuxKggsRO2ZPSRc,856
-airbyte_cdk/sources/connector_state_manager.py,sha256=5E0O5yOj-4XRr7pSZF-SD1RrSvkKN1V-833HDCVBzaw,10983
+airbyte_cdk/sources/connector_state_manager.py,sha256=p9iwWbb5uqRbsrHsdZBMXKmyHgLVbsOcV3QQexBFnPE,11052
 airbyte_cdk/sources/http_config.py,sha256=OBZeuyFilm6NlDlBhFQvHhTWabEvZww6OHDIlZujIS0,730
 airbyte_cdk/sources/http_logger.py,sha256=v0kkpDtA0GUOgj6_3AayrYaBrSHBqG4t3MGbrtxaNmU,1437
 airbyte_cdk/sources/source.py,sha256=dk50z8Roc28MJ8FxWe652B-GwItO__bTZqFm7WOtHnw,4412
@@ -232,7 +232,7 @@ airbyte_cdk/sources/streams/http/auth/core.py,sha256=_s9wewvvIcOgYjhHGDj_YHApnF5
 airbyte_cdk/sources/streams/http/auth/oauth.py,sha256=zchPWN1utNg02F93f5b4UFI5OXYo8-QhocbsXhLdG4U,4135
 airbyte_cdk/sources/streams/http/auth/token.py,sha256=oU1ul0LsGsPGN_vOJOKw1xX2y_XWULRxjqXu7Rivcr8,1940
 airbyte_cdk/sources/streams/http/requests_native_auth/__init__.py,sha256=RN0D3nOX1xLgwEwKWu6pkGy3XqBFzKSNZ8Lf6umU2eY,413
-airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py,sha256=sknTSDgwPX3IhHRqEnxp9tuRE5Iu8jU_dyIKsFad1wY,9257
+airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py,sha256=VfI81zpWYMrzhU0nzN7J1lYz-aZCyHzvv-cLqzm8in0,10125
 airbyte_cdk/sources/streams/http/requests_native_auth/abstract_token.py,sha256=T0hVF2cBXGgIfrCslvTC1uNm9rNbYjENNl2Cb3mXuSY,961
 airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py,sha256=HrnA76k4i-YOp7ygDwyMg0Hf80gjJaNSU3GbXiTzxMc,13381
 airbyte_cdk/sources/streams/http/requests_native_auth/token.py,sha256=hDti8DlF_R5YYX95hg9BPogYtG-KUYtOifrFDv_L3Hk,2456
@@ -253,12 +253,12 @@ airbyte_cdk/test/entrypoint_wrapper.py,sha256=uTOEYoWkYnbkooPJ4a4gZ-NEll5j1tTCAz
 airbyte_cdk/test/state_builder.py,sha256=SlKadhKVi38ZSKMeceVAxjowxsDDT9vJoG6gU4zDrQE,705
 airbyte_cdk/test/mock_http/__init__.py,sha256=uil6k-0NbUyDFZXtWw88HaS7r13i43VzA9H7hOHzZx8,322
 airbyte_cdk/test/mock_http/matcher.py,sha256=J4C8g8PkdKo4OwHWMJGYJIyrLnQpXI5gXWUtyxsxHpM,1240
-airbyte_cdk/test/mock_http/mocker.py,sha256=R-RvMufoI22TtZno8WpSIy0BVhtwoI9OvEGDL6HJU5w,6143
+airbyte_cdk/test/mock_http/mocker.py,sha256=Sb1Nnf3bVEJfiy5_IliRcyIiIPQL8esSWmm5j9u0E_E,6202
 airbyte_cdk/test/mock_http/request.py,sha256=dd_i47FOGD5iRlU23daotv2gEn5NOVqTBAqykxdG6-0,3687
-airbyte_cdk/test/mock_http/response.py,sha256=XjkouKy3eCsylZRROzl1qZs6LfjnO4C3L1hcJoqTHOs,354
+airbyte_cdk/test/mock_http/response.py,sha256=F09QGG8N3Z8fL_b0rmSKTYoKgku5yZJQCpj0Fwwxu3s,588
 airbyte_cdk/test/mock_http/response_builder.py,sha256=sc0lU_LN3wjBc4mFFV-3Y5IhYeapRdtB_-EDdHfyArA,7804
 airbyte_cdk/utils/__init__.py,sha256=qZoNqzEKhIXdN_ZfvXlIGnmiDDjCFy6BVCzzWjUZcuU,294
-airbyte_cdk/utils/airbyte_secrets_utils.py,sha256=q3aDl8T10ufGbeqnUPqbZLxQcHdkf2kDfQK_upWzBbI,2894
+airbyte_cdk/utils/airbyte_secrets_utils.py,sha256=UIu8jzVGstjrlT8iKxvWieiO57rmxuOtx8QphHEzs9Y,3079
 airbyte_cdk/utils/analytics_message.py,sha256=om0y9U_Y1RNHREi_K8D3mkZgBNfwiOyG71ixOBKZbVs,598
 airbyte_cdk/utils/constants.py,sha256=QzCi7j5SqpI5I06uRvQ8FC73JVJi7rXaRnR3E_gro5c,108
 airbyte_cdk/utils/datetime_format_inferrer.py,sha256=gGKDQ3OdY18R5CVFhq4c7zB_E4Cxe6J6SLA29cz3cJM,3954
@@ -299,7 +299,7 @@ unit_tests/sources/declarative/test_declarative_stream.py,sha256=Tt3PBIAo7DeQgvX
 unit_tests/sources/declarative/test_manifest_declarative_source.py,sha256=HsDeDgtipkciNOnOeaM1H7eUh1Noq_OVDoFEMugm124,63391
 unit_tests/sources/declarative/test_yaml_declarative_source.py,sha256=6HhsUFgB7ueN0yOUHWb4gpPYLng5jasxN_plvz3x37g,5097
 unit_tests/sources/declarative/auth/__init__.py,sha256=4Hw-PX1-VgESLF16cDdvuYCzGJtHntThLF4qIiULWeo,61
-unit_tests/sources/declarative/auth/test_oauth.py,sha256=pNUzscHDR8K7tedF7v_lSMVSOEOeq9bAlxHG2cttdU0,12854
+unit_tests/sources/declarative/auth/test_oauth.py,sha256=NLa7zUhN2pmFsEZFykxpzKVYuw_9luHMkQtFfxP72U4,14039
 unit_tests/sources/declarative/auth/test_selective_authenticator.py,sha256=RAolWBLCLtibul5wlteQzLAdnUF8vh893qAr9fhoYOk,1315
 unit_tests/sources/declarative/auth/test_session_token_auth.py,sha256=nKNBx7yGrrvFW9BUwG2xI472Q2sNXl2j1LhWZuUaWmY,6183
 unit_tests/sources/declarative/auth/test_token_auth.py,sha256=Kg90S04_4WjTUCzwcj62OxnF_TPQcjL_r7-BaeDhxPI,7384
@@ -445,7 +445,7 @@ unit_tests/test/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 unit_tests/test/test_entrypoint_wrapper.py,sha256=m4csYvjO2PzvZZma7K322SBBiL5D33xuv8eUMjitDXE,10839
 unit_tests/test/mock_http/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 unit_tests/test/mock_http/test_matcher.py,sha256=dBndYzqvo3AdHRilLrqruXdPviwi91gWt-ubDsGb-yg,2327
-unit_tests/test/mock_http/test_mocker.py,sha256=Zp7nhuEGWLsysNFMIcNhu5GhWrNd_JQMIXOjZHYRdY4,8409
+unit_tests/test/mock_http/test_mocker.py,sha256=sOoWutnrPDKB99Y3bkEyr3HFELGivwuklVJ_ii8C-ew,8523
 unit_tests/test/mock_http/test_request.py,sha256=O9ihefGNiZKpHqsGtis6BjF8VoaOULNR8zOblVqmsL4,7602
 unit_tests/test/mock_http/test_response_builder.py,sha256=IxAww4gaOxG-9MW8kEZkRzYL2mO6xe4jIsxhi40i2ow,7878
 unit_tests/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -453,11 +453,11 @@ unit_tests/utils/test_datetime_format_inferrer.py,sha256=1EUW1_afccMDrZM6YZyyPqr
 unit_tests/utils/test_mapping_helpers.py,sha256=hqRppuban9hGKviiNFqp2fNdAz77d1_gjvgg8L7-jy8,1408
 unit_tests/utils/test_rate_limiting.py,sha256=ESrPBH61EZSeAf-hYWnd49igCgkUWnT21rUHPQaOLQM,873
 unit_tests/utils/test_schema_inferrer.py,sha256=Z2jHBZ540wnYkylIdV_2xr75Vtwlxuyg4MNPAG-xhpk,7817
-unit_tests/utils/test_secret_utils.py,sha256=XKe0f1RHYii8iwE6ATmBr5JGDI1pzzrnZUGdUSMJQP4,4886
+unit_tests/utils/test_secret_utils.py,sha256=CdKK8A2-5XVxbXVtX22FK9dwwMeP5KNqDH6luWRXSNw,5256
 unit_tests/utils/test_stream_status_utils.py,sha256=Xr8MZ2HWgTVIyMbywDvuYkRaUF4RZLQOT8-JjvcfR24,2970
 unit_tests/utils/test_traced_exception.py,sha256=bDFP5zMBizFenz6V2WvEZTRCKGB5ijh3DBezjbfoYIs,4198
-airbyte_cdk-0.62.0.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
-airbyte_cdk-0.62.0.dist-info/METADATA,sha256=tppN9XjW0XKd-E_5J7xTL6F6RG9u6TI3HmjnYbH-klw,11073
-airbyte_cdk-0.62.0.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-airbyte_cdk-0.62.0.dist-info/top_level.txt,sha256=edvsDKTnE6sD2wfCUaeTfKf5gQIL6CPVMwVL2sWZzqo,51
-airbyte_cdk-0.62.0.dist-info/RECORD,,
+airbyte_cdk-0.62.2.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
+airbyte_cdk-0.62.2.dist-info/METADATA,sha256=mg5FUvzFvSF_W3YQZY6V6fUcSoUy4C03oFt2hF6w0FI,11073
+airbyte_cdk-0.62.2.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+airbyte_cdk-0.62.2.dist-info/top_level.txt,sha256=edvsDKTnE6sD2wfCUaeTfKf5gQIL6CPVMwVL2sWZzqo,51
+airbyte_cdk-0.62.2.dist-info/RECORD,,

unit_tests/sources/declarative/auth/test_oauth.py

@@ -10,6 +10,7 @@ import pendulum
 import pytest
 import requests
 from airbyte_cdk.sources.declarative.auth import DeclarativeOauth2Authenticator
+from airbyte_cdk.utils.airbyte_secrets_utils import filter_secrets
 from requests import Response
 
 LOGGER = logging.getLogger(__name__)
@@ -165,6 +166,32 @@ class TestOauth2Authenticator:
 
         assert ("access_token", 1000) == token
 
+        filtered = filter_secrets("access_token")
+        assert filtered == "****"
+
+    def test_refresh_access_token_missing_access_token(self, mocker):
+        oauth = DeclarativeOauth2Authenticator(
+            token_refresh_endpoint="{{ config['refresh_endpoint'] }}",
+            client_id="{{ config['client_id'] }}",
+            client_secret="{{ config['client_secret'] }}",
+            refresh_token="{{ config['refresh_token'] }}",
+            config=config,
+            scopes=["scope1", "scope2"],
+            token_expiry_date="{{ config['token_expiry_date'] }}",
+            refresh_request_body={
+                "custom_field": "{{ config['custom_field'] }}",
+                "another_field": "{{ config['another_field'] }}",
+                "scopes": ["no_override"],
+            },
+            parameters={},
+        )
+
+        resp.status_code = 200
+        mocker.patch.object(resp, "json", return_value={"expires_in": 1000})
+        mocker.patch.object(requests, "request", side_effect=mock_request, autospec=True)
+        with pytest.raises(Exception):
+            oauth.refresh_access_token()
+
     @pytest.mark.parametrize(
         "timestamp, expected_date",
         [

unit_tests/test/mock_http/test_mocker.py

@@ -15,6 +15,7 @@ _ANOTHER_RESPONSE_BODY = "another body"
 _A_RESPONSE = HttpResponse("any response")
 _SOME_QUERY_PARAMS = {"q1": "query value"}
 _SOME_HEADERS = {"h1": "header value"}
+_OTHER_HEADERS = {"h2": "another header value"}
 _SOME_REQUEST_BODY_MAPPING = {"first_field": "first_value", "second_field": 2}
 _SOME_REQUEST_BODY_STR = "some_request_body"
 
@@ -24,13 +25,14 @@ class HttpMockerTest(TestCase):
     def test_given_get_request_match_when_decorate_then_return_response(self, http_mocker):
         http_mocker.get(
             HttpRequest(_A_URL, _SOME_QUERY_PARAMS, _SOME_HEADERS),
-            HttpResponse(_A_RESPONSE_BODY, 474),
+            HttpResponse(_A_RESPONSE_BODY, 474, _OTHER_HEADERS),
         )
 
         response = requests.get(_A_URL, params=_SOME_QUERY_PARAMS, headers=_SOME_HEADERS)
 
         assert response.text == _A_RESPONSE_BODY
         assert response.status_code == 474
+        assert response.headers == _OTHER_HEADERS
 
     @HttpMocker()
     def test_given_loose_headers_matching_when_decorate_then_match(self, http_mocker):

unit_tests/utils/test_secret_utils.py

@@ -3,7 +3,7 @@
 #
 
 import pytest
-from airbyte_cdk.utils.airbyte_secrets_utils import filter_secrets, get_secret_paths, get_secrets, update_secrets
+from airbyte_cdk.utils.airbyte_secrets_utils import add_to_secrets, filter_secrets, get_secret_paths, get_secrets, update_secrets
 
 SECRET_STRING_KEY = "secret_key1"
 SECRET_STRING_VALUE = "secret_value"
@@ -121,3 +121,15 @@ def test_secret_filtering():
     update_secrets([SECRET_STRING_VALUE, SECRET_STRING_2_VALUE])
     filtered = filter_secrets(sensitive_str)
     assert filtered == f"**** {NOT_SECRET_VALUE} **** ****"
+
+
+def test_secrets_added_are_filtered():
+    ADDED_SECRET = "only_a_secret_if_added"
+    sensitive_str = f"{ADDED_SECRET} {NOT_SECRET_VALUE}"
+
+    filtered = filter_secrets(sensitive_str)
+    assert filtered == sensitive_str
+
+    add_to_secrets(ADDED_SECRET)
+    filtered = filter_secrets(sensitive_str)
+    assert filtered == f"**** {NOT_SECRET_VALUE}"