airbyte-cdk 6.18.2__tar.gz → 6.19.1__tar.gz

{airbyte_cdk-6.18.2 → airbyte_cdk-6.19.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: airbyte-cdk
-Version: 6.18.2
+Version: 6.19.1
 Summary: A framework for writing Airbyte Connectors.
 License: MIT
 Keywords: airbyte,connector-development-kit,cdk

{airbyte_cdk-6.18.2 → airbyte_cdk-6.19.1}/airbyte_cdk/sources/declarative/auth/oauth.py

@@ -39,6 +39,7 @@ class DeclarativeOauth2Authenticator(AbstractOauth2Authenticator, DeclarativeAut
         token_expiry_date_format str: format of the datetime; provide it if expires_in is returned in datetime instead of seconds
         token_expiry_is_time_of_expiration bool: set True it if expires_in is returned as time of expiration instead of the number seconds until expiration
         refresh_request_body (Optional[Mapping[str, Any]]): The request body to send in the refresh request
+        refresh_request_headers (Optional[Mapping[str, Any]]): The request headers to send in the refresh request
         grant_type: The grant_type to request for access_token. If set to refresh_token, the refresh_token parameter has to be provided
         message_repository (MessageRepository): the message repository used to emit logs on HTTP requests
     """
@@ -61,6 +62,7 @@ class DeclarativeOauth2Authenticator(AbstractOauth2Authenticator, DeclarativeAut
     expires_in_name: Union[InterpolatedString, str] = "expires_in"
     refresh_token_name: Union[InterpolatedString, str] = "refresh_token"
     refresh_request_body: Optional[Mapping[str, Any]] = None
+    refresh_request_headers: Optional[Mapping[str, Any]] = None
     grant_type_name: Union[InterpolatedString, str] = "grant_type"
     grant_type: Union[InterpolatedString, str] = "refresh_token"
     message_repository: MessageRepository = NoopMessageRepository()
@@ -101,6 +103,9 @@ class DeclarativeOauth2Authenticator(AbstractOauth2Authenticator, DeclarativeAut
         self._refresh_request_body = InterpolatedMapping(
             self.refresh_request_body or {}, parameters=parameters
         )
+        self._refresh_request_headers = InterpolatedMapping(
+            self.refresh_request_headers or {}, parameters=parameters
+        )
         self._token_expiry_date: pendulum.DateTime = (
             pendulum.parse(
                 InterpolatedString.create(self.token_expiry_date, parameters=parameters).eval(
@@ -178,6 +183,9 @@ class DeclarativeOauth2Authenticator(AbstractOauth2Authenticator, DeclarativeAut
     def get_refresh_request_body(self) -> Mapping[str, Any]:
         return self._refresh_request_body.eval(self.config)
 
+    def get_refresh_request_headers(self) -> Mapping[str, Any]:
+        return self._refresh_request_headers.eval(self.config)
+
     def get_token_expiry_date(self) -> pendulum.DateTime:
         return self._token_expiry_date  # type: ignore # _token_expiry_date is a pendulum.DateTime. It is never None despite what mypy thinks
 

{airbyte_cdk-6.18.2 → airbyte_cdk-6.19.1}/airbyte_cdk/sources/declarative/declarative_component_schema.yaml

@@ -678,7 +678,7 @@ definitions:
     properties:
       type:
         type: string
-        enum: [ CustomSchemaNormalization ]
+        enum: [CustomSchemaNormalization]
       class_name:
         title: Class Name
         description: Fully-qualified name of the class that will be implementing the custom normalization. The format is `source_<name>.<package>.<class_name>`.
@@ -1139,6 +1139,14 @@ definitions:
           - applicationId: "{{ config['application_id'] }}"
             applicationSecret: "{{ config['application_secret'] }}"
             token: "{{ config['token'] }}"
+      refresh_request_headers:
+        title: Refresh Request Headers
+        description: Headers of the request sent to get a new access token.
+        type: object
+        additionalProperties: true
+        examples:
+          - Authorization: "<AUTH_TOKEN>"
+            Content-Type: "application/x-www-form-urlencoded"
       scopes:
         title: Scopes
         description: List of scopes that should be granted to the access token.
@@ -2886,6 +2894,7 @@ definitions:
       parser:
         anyOf:
           - "$ref": "#/definitions/GzipParser"
+          - "$ref": "#/definitions/JsonParser"
           - "$ref": "#/definitions/JsonLineParser"
           - "$ref": "#/definitions/CsvParser"
   #    PARSERS
@@ -2902,6 +2911,20 @@ definitions:
         anyOf:
           - "$ref": "#/definitions/JsonLineParser"
           - "$ref": "#/definitions/CsvParser"
+          - "$ref": "#/definitions/JsonParser"
+  JsonParser:
+    title: JsonParser
+    description: Parser used for parsing str, bytes, or bytearray data and returning data in a dictionary format.
+    type: object
+    required:
+      - type
+    properties:
+      type:
+        type: string
+        enum: [JsonParser]
+      encoding:
+        type: string
+        default: utf-8
   JsonLineParser:
     type: object
     required:

{airbyte_cdk-6.18.2 → airbyte_cdk-6.19.1}/airbyte_cdk/sources/declarative/decoders/composite_raw_decoder.py

@@ -7,9 +7,12 @@ from dataclasses import dataclass
 from io import BufferedIOBase, TextIOWrapper
 from typing import Any, Generator, MutableMapping, Optional
 
+import orjson
 import requests
 
+from airbyte_cdk.models import FailureType
 from airbyte_cdk.sources.declarative.decoders.decoder import Decoder
+from airbyte_cdk.utils import AirbyteTracedException
 
 logger = logging.getLogger("airbyte")
 
@@ -42,6 +45,46 @@ class GzipParser(Parser):
             yield from self.inner_parser.parse(gzipobj)
 
 
+@dataclass
+class JsonParser(Parser):
+    encoding: str = "utf-8"
+
+    def parse(self, data: BufferedIOBase) -> Generator[MutableMapping[str, Any], None, None]:
+        """
+        Attempts to deserialize data using orjson library. As an extra layer of safety we fallback on the json library to deserialize the data.
+        """
+        raw_data = data.read()
+        body_json = self._parse_orjson(raw_data) or self._parse_json(raw_data)
+
+        if body_json is None:
+            raise AirbyteTracedException(
+                message="Response JSON data failed to be parsed. See logs for more information.",
+                internal_message=f"Response JSON data failed to be parsed.",
+                failure_type=FailureType.system_error,
+            )
+
+        if isinstance(body_json, list):
+            yield from body_json
+        else:
+            yield from [body_json]
+
+    def _parse_orjson(self, raw_data: bytes) -> Optional[Any]:
+        try:
+            return orjson.loads(raw_data.decode(self.encoding))
+        except Exception as exc:
+            logger.debug(
+                f"Failed to parse JSON data using orjson library. Falling back to json library. {exc}"
+            )
+            return None
+
+    def _parse_json(self, raw_data: bytes) -> Optional[Any]:
+        try:
+            return json.loads(raw_data.decode(self.encoding))
+        except Exception as exc:
+            logger.error(f"Failed to parse JSON data using json library. {exc}")
+            return None
+
+
 @dataclass
 class JsonLineParser(Parser):
     encoding: Optional[str] = "utf-8"

{airbyte_cdk-6.18.2 → airbyte_cdk-6.19.1}/airbyte_cdk/sources/declarative/models/declarative_component_schema.py

@@ -571,6 +571,17 @@ class OAuthAuthenticator(BaseModel):
         ],
         title="Refresh Request Body",
     )
+    refresh_request_headers: Optional[Dict[str, Any]] = Field(
+        None,
+        description="Headers of the request sent to get a new access token.",
+        examples=[
+            {
+                "Authorization": "<AUTH_TOKEN>",
+                "Content-Type": "application/x-www-form-urlencoded",
+            }
+        ],
+        title="Refresh Request Headers",
+    )
     scopes: Optional[List[str]] = Field(
         None,
         description="List of scopes that should be granted to the access token.",
@@ -1201,6 +1212,14 @@ class LegacySessionTokenAuthenticator(BaseModel):
     parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
 
 
+class JsonParser(BaseModel):
+    class Config:
+        extra = Extra.allow
+
+    type: Literal["JsonParser"]
+    encoding: Optional[str] = "utf-8"
+
+
 class JsonLineParser(BaseModel):
     type: Literal["JsonLineParser"]
     encoding: Optional[str] = "utf-8"
@@ -1599,7 +1618,7 @@ class RecordSelector(BaseModel):
 
 class GzipParser(BaseModel):
     type: Literal["GzipParser"]
-    inner_parser: Union[JsonLineParser, CsvParser]
+    inner_parser: Union[JsonLineParser, CsvParser, JsonParser]
 
 
 class Spec(BaseModel):
@@ -1634,7 +1653,7 @@ class CompositeErrorHandler(BaseModel):
 
 class CompositeRawDecoder(BaseModel):
     type: Literal["CompositeRawDecoder"]
-    parser: Union[GzipParser, JsonLineParser, CsvParser]
+    parser: Union[GzipParser, JsonParser, JsonLineParser, CsvParser]
 
 
 class DeclarativeSource1(BaseModel):

{airbyte_cdk-6.18.2 → airbyte_cdk-6.19.1}/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py

@@ -72,6 +72,8 @@ from airbyte_cdk.sources.declarative.decoders.composite_raw_decoder import (
     CsvParser,
     GzipParser,
     JsonLineParser,
+    JsonParser,
+    Parser,
 )
 from airbyte_cdk.sources.declarative.extractors import (
     DpathExtractor,
@@ -247,6 +249,9 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     JsonLineParser as JsonLineParserModel,
 )
+from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
+    JsonParser as JsonParserModel,
+)
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     JwtAuthenticator as JwtAuthenticatorModel,
 )
@@ -522,6 +527,7 @@ class ModelToComponentFactory:
             JsonDecoderModel: self.create_json_decoder,
             JsonlDecoderModel: self.create_jsonl_decoder,
             JsonLineParserModel: self.create_json_line_parser,
+            JsonParserModel: self.create_json_parser,
             GzipJsonDecoderModel: self.create_gzipjson_decoder,
             GzipParserModel: self.create_gzip_parser,
             KeysToLowerModel: self.create_keys_to_lower_transformation,
@@ -1032,17 +1038,17 @@ class ModelToComponentFactory:
         self, model: CursorPaginationModel, config: Config, decoder: Decoder, **kwargs: Any
     ) -> CursorPaginationStrategy:
         if isinstance(decoder, PaginationDecoderDecorator):
-            if not isinstance(decoder.decoder, (JsonDecoder, XmlDecoder)):
-                raise ValueError(
-                    f"Provided decoder of {type(decoder.decoder)=} is not supported. Please set JsonDecoder or XmlDecoder instead."
-                )
+            inner_decoder = decoder.decoder
+        else:
+            inner_decoder = decoder
+            decoder = PaginationDecoderDecorator(decoder=decoder)
+
+        if self._is_supported_decoder_for_pagination(inner_decoder):
             decoder_to_use = decoder
         else:
-            if not isinstance(decoder, (JsonDecoder, XmlDecoder)):
-                raise ValueError(
-                    f"Provided decoder of {type(decoder)=} is not supported. Please set JsonDecoder or XmlDecoder instead."
-                )
-            decoder_to_use = PaginationDecoderDecorator(decoder=decoder)
+            raise ValueError(
+                self._UNSUPPORTED_DECODER_ERROR.format(decoder_type=type(inner_decoder))
+            )
 
         return CursorPaginationStrategy(
             cursor_value=model.cursor_value,
@@ -1515,11 +1521,10 @@ class ModelToComponentFactory:
         cursor_used_for_stop_condition: Optional[DeclarativeCursor] = None,
     ) -> Union[DefaultPaginator, PaginatorTestReadDecorator]:
         if decoder:
-            if not isinstance(decoder, (JsonDecoder, XmlDecoder)):
-                raise ValueError(
-                    f"Provided decoder of {type(decoder)=} is not supported. Please set JsonDecoder or XmlDecoder instead."
-                )
-            decoder_to_use = PaginationDecoderDecorator(decoder=decoder)
+            if self._is_supported_decoder_for_pagination(decoder):
+                decoder_to_use = PaginationDecoderDecorator(decoder=decoder)
+            else:
+                raise ValueError(self._UNSUPPORTED_DECODER_ERROR.format(decoder_type=type(decoder)))
         else:
             decoder_to_use = PaginationDecoderDecorator(decoder=JsonDecoder(parameters={}))
         page_size_option = (
@@ -1748,6 +1753,11 @@ class ModelToComponentFactory:
     def create_json_decoder(model: JsonDecoderModel, config: Config, **kwargs: Any) -> JsonDecoder:
         return JsonDecoder(parameters={})
 
+    @staticmethod
+    def create_json_parser(model: JsonParserModel, config: Config, **kwargs: Any) -> JsonParser:
+        encoding = model.encoding if model.encoding else "utf-8"
+        return JsonParser(encoding=encoding)
+
     @staticmethod
     def create_jsonl_decoder(
         model: JsonlDecoderModel, config: Config, **kwargs: Any
@@ -1909,6 +1919,9 @@ class ModelToComponentFactory:
                 refresh_request_body=InterpolatedMapping(
                     model.refresh_request_body or {}, parameters=model.parameters or {}
                 ).eval(config),
+                refresh_request_headers=InterpolatedMapping(
+                    model.refresh_request_headers or {}, parameters=model.parameters or {}
+                ).eval(config),
                 scopes=model.scopes,
                 token_expiry_date_format=model.token_expiry_date_format,
                 message_repository=self._message_repository,
@@ -1928,6 +1941,7 @@ class ModelToComponentFactory:
             grant_type_name=model.grant_type_name or "grant_type",
             grant_type=model.grant_type or "refresh_token",
             refresh_request_body=model.refresh_request_body,
+            refresh_request_headers=model.refresh_request_headers,
             refresh_token_name=model.refresh_token_name or "refresh_token",
             refresh_token=model.refresh_token,
             scopes=model.scopes,
@@ -1940,22 +1954,22 @@ class ModelToComponentFactory:
             message_repository=self._message_repository,
         )
 
-    @staticmethod
     def create_offset_increment(
-        model: OffsetIncrementModel, config: Config, decoder: Decoder, **kwargs: Any
+        self, model: OffsetIncrementModel, config: Config, decoder: Decoder, **kwargs: Any
     ) -> OffsetIncrement:
         if isinstance(decoder, PaginationDecoderDecorator):
-            if not isinstance(decoder.decoder, (JsonDecoder, XmlDecoder)):
-                raise ValueError(
-                    f"Provided decoder of {type(decoder.decoder)=} is not supported. Please set JsonDecoder or XmlDecoder instead."
-                )
+            inner_decoder = decoder.decoder
+        else:
+            inner_decoder = decoder
+            decoder = PaginationDecoderDecorator(decoder=decoder)
+
+        if self._is_supported_decoder_for_pagination(inner_decoder):
             decoder_to_use = decoder
         else:
-            if not isinstance(decoder, (JsonDecoder, XmlDecoder)):
-                raise ValueError(
-                    f"Provided decoder of {type(decoder)=} is not supported. Please set JsonDecoder or XmlDecoder instead."
-                )
-            decoder_to_use = PaginationDecoderDecorator(decoder=decoder)
+            raise ValueError(
+                self._UNSUPPORTED_DECODER_ERROR.format(decoder_type=type(inner_decoder))
+            )
+
         return OffsetIncrement(
             page_size=model.page_size,
             config=config,
@@ -2555,3 +2569,25 @@ class ModelToComponentFactory:
             components_mapping=components_mapping,
             parameters=model.parameters or {},
         )
+
+    _UNSUPPORTED_DECODER_ERROR = (
+        "Specified decoder of {decoder_type} is not supported for pagination."
+        "Please set as `JsonDecoder`, `XmlDecoder`, or a `CompositeRawDecoder` with an inner_parser of `JsonParser` or `GzipParser` instead."
+        "If using `GzipParser`, please ensure that the lowest level inner_parser is a `JsonParser`."
+    )
+
+    def _is_supported_decoder_for_pagination(self, decoder: Decoder) -> bool:
+        if isinstance(decoder, (JsonDecoder, XmlDecoder)):
+            return True
+        elif isinstance(decoder, CompositeRawDecoder):
+            return self._is_supported_parser_for_pagination(decoder.parser)
+        else:
+            return False
+
+    def _is_supported_parser_for_pagination(self, parser: Parser) -> bool:
+        if isinstance(parser, JsonParser):
+            return True
+        elif isinstance(parser, GzipParser):
+            return isinstance(parser.inner_parser, JsonParser)
+        else:
+            return False

{airbyte_cdk-6.18.2 → airbyte_cdk-6.19.1}/airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py

@@ -98,6 +98,14 @@ class AbstractOauth2Authenticator(AuthBase):
 
         return payload
 
+    def build_refresh_request_headers(self) -> Mapping[str, Any] | None:
+        """
+        Returns the request headers to set on the refresh request
+
+        """
+        headers = self.get_refresh_request_headers()
+        return headers if headers else None
+
     def _wrap_refresh_token_exception(
         self, exception: requests.exceptions.RequestException
     ) -> bool:
@@ -128,6 +136,7 @@ class AbstractOauth2Authenticator(AuthBase):
                 method="POST",
                 url=self.get_token_refresh_endpoint(),  # type: ignore # returns None, if not provided, but str | bytes is expected.
                 data=self.build_refresh_request_body(),
+                headers=self.build_refresh_request_headers(),
             )
             if response.ok:
                 response_json = response.json()
@@ -254,6 +263,10 @@ class AbstractOauth2Authenticator(AuthBase):
     def get_refresh_request_body(self) -> Mapping[str, Any]:
         """Returns the request body to set on the refresh request"""
 
+    @abstractmethod
+    def get_refresh_request_headers(self) -> Mapping[str, Any]:
+        """Returns the request headers to set on the refresh request"""
+
     @abstractmethod
     def get_grant_type(self) -> str:
         """Returns grant_type specified for requesting access_token"""

{airbyte_cdk-6.18.2 → airbyte_cdk-6.19.1}/airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py

@@ -39,6 +39,7 @@ class Oauth2Authenticator(AbstractOauth2Authenticator):
         access_token_name: str = "access_token",
         expires_in_name: str = "expires_in",
         refresh_request_body: Mapping[str, Any] | None = None,
+        refresh_request_headers: Mapping[str, Any] | None = None,
         grant_type_name: str = "grant_type",
         grant_type: str = "refresh_token",
         token_expiry_is_time_of_expiration: bool = False,
@@ -57,6 +58,7 @@ class Oauth2Authenticator(AbstractOauth2Authenticator):
         self._access_token_name = access_token_name
         self._expires_in_name = expires_in_name
         self._refresh_request_body = refresh_request_body
+        self._refresh_request_headers = refresh_request_headers
         self._grant_type_name = grant_type_name
         self._grant_type = grant_type
 
@@ -101,6 +103,9 @@ class Oauth2Authenticator(AbstractOauth2Authenticator):
     def get_refresh_request_body(self) -> Mapping[str, Any]:
         return self._refresh_request_body  # type: ignore [return-value]
 
+    def get_refresh_request_headers(self) -> Mapping[str, Any]:
+        return self._refresh_request_headers  # type: ignore [return-value]
+
     def get_grant_type_name(self) -> str:
         return self._grant_type_name
 
@@ -149,6 +154,7 @@ class SingleUseRefreshTokenOauth2Authenticator(Oauth2Authenticator):
         expires_in_name: str = "expires_in",
         refresh_token_name: str = "refresh_token",
         refresh_request_body: Mapping[str, Any] | None = None,
+        refresh_request_headers: Mapping[str, Any] | None = None,
         grant_type_name: str = "grant_type",
         grant_type: str = "refresh_token",
         client_id_name: str = "client_id",
@@ -174,6 +180,7 @@ class SingleUseRefreshTokenOauth2Authenticator(Oauth2Authenticator):
             expires_in_name (str, optional): Name of the name of the field that characterizes when the current access token will expire, used to parse the refresh token response. Defaults to "expires_in".
             refresh_token_name (str, optional): Name of the name of the refresh token field, used to parse the refresh token response. Defaults to "refresh_token".
             refresh_request_body (Mapping[str, Any], optional): Custom key value pair that will be added to the refresh token request body. Defaults to None.
+            refresh_request_headers (Mapping[str, Any], optional): Custom key value pair that will be added to the refresh token request headers. Defaults to None.
             grant_type (str, optional): OAuth grant type. Defaults to "refresh_token".
             client_id (Optional[str]): The client id to authenticate. If not specified, defaults to credentials.client_id in the config object.
             client_secret (Optional[str]): The client secret to authenticate. If not specified, defaults to credentials.client_secret in the config object.
@@ -220,6 +227,7 @@ class SingleUseRefreshTokenOauth2Authenticator(Oauth2Authenticator):
             access_token_name=access_token_name,
             expires_in_name=expires_in_name,
             refresh_request_body=refresh_request_body,
+            refresh_request_headers=refresh_request_headers,
             grant_type_name=self._grant_type_name,
             grant_type=grant_type,
             token_expiry_date_format=token_expiry_date_format,

{airbyte_cdk-6.18.2 → airbyte_cdk-6.19.1}/pyproject.toml

@@ -23,7 +23,7 @@ classifiers = [
 keywords = ["airbyte", "connector-development-kit", "cdk"]
 
 # Python CDK uses dynamic versioning: https://github.com/mtkennerly/poetry-dynamic-versioning
-version = "6.18.2"  # Version will be calculated dynamically.
+version = "6.19.1"  # Version will be calculated dynamically.
 
 [tool.poetry-dynamic-versioning]
 enable = false