airbyte-cdk 7.2.3__py3-none-any.whl → 7.3.1__py3-none-any.whl

airbyte_cdk/sources/declarative/auth/jwt.py

@@ -6,7 +6,7 @@ import base64
 import json
 from dataclasses import InitVar, dataclass
 from datetime import datetime
-from typing import Any, Mapping, Optional, Union, cast
+from typing import Any, Mapping, MutableMapping, Optional, Union, cast
 
 import jwt
 from cryptography.hazmat.primitives import serialization
@@ -14,12 +14,15 @@ from cryptography.hazmat.primitives.asymmetric.ec import EllipticCurvePrivateKey
 from cryptography.hazmat.primitives.asymmetric.ed448 import Ed448PrivateKey
 from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey
 from cryptography.hazmat.primitives.asymmetric.rsa import RSAPrivateKey
-from cryptography.hazmat.primitives.asymmetric.types import PrivateKeyTypes
 
 from airbyte_cdk.sources.declarative.auth.declarative_authenticator import DeclarativeAuthenticator
 from airbyte_cdk.sources.declarative.interpolation.interpolated_boolean import InterpolatedBoolean
 from airbyte_cdk.sources.declarative.interpolation.interpolated_mapping import InterpolatedMapping
 from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
+from airbyte_cdk.sources.declarative.requesters.request_option import (
+    RequestOption,
+    RequestOptionType,
+)
 
 # Type alias for keys that JWT library accepts
 JwtKeyTypes = Union[
@@ -86,6 +89,7 @@ class JwtAuthenticator(DeclarativeAuthenticator):
     additional_jwt_headers: Optional[Mapping[str, Any]] = None
     additional_jwt_payload: Optional[Mapping[str, Any]] = None
     passphrase: Optional[Union[InterpolatedString, str]] = None
+    request_option: Optional[RequestOption] = None
 
     def __post_init__(self, parameters: Mapping[str, Any]) -> None:
         self._secret_key = InterpolatedString.create(self.secret_key, parameters=parameters)
@@ -121,6 +125,13 @@ class JwtAuthenticator(DeclarativeAuthenticator):
             else None
         )
 
+        # When we first implemented the JWT authenticator, we assumed that the signed token was always supposed
+        # to be loaded into the request headers under the `Authorization` key. This is not always the case, but
+        # this default option allows for backwards compatibility to be retained for existing connectors
+        self._request_option = self.request_option or RequestOption(
+            inject_into=RequestOptionType.header, field_name="Authorization", parameters=parameters
+        )
+
     def _get_jwt_headers(self) -> dict[str, Any]:
         """
         Builds and returns the headers used when signing the JWT.
@@ -213,7 +224,8 @@ class JwtAuthenticator(DeclarativeAuthenticator):
 
     @property
     def auth_header(self) -> str:
-        return "Authorization"
+        options = self._get_request_options(RequestOptionType.header)
+        return next(iter(options.keys()), "")
 
     @property
     def token(self) -> str:
@@ -222,3 +234,18 @@ class JwtAuthenticator(DeclarativeAuthenticator):
             if self._get_header_prefix()
             else self._get_signed_token()
         )
+
+    def get_request_params(self) -> Mapping[str, Any]:
+        return self._get_request_options(RequestOptionType.request_parameter)
+
+    def get_request_body_data(self) -> Union[Mapping[str, Any], str]:
+        return self._get_request_options(RequestOptionType.body_data)
+
+    def get_request_body_json(self) -> Mapping[str, Any]:
+        return self._get_request_options(RequestOptionType.body_json)
+
+    def _get_request_options(self, option_type: RequestOptionType) -> Mapping[str, Any]:
+        options: MutableMapping[str, Any] = {}
+        if self._request_option.inject_into == option_type:
+            self._request_option.inject_into_request(options, self.token, self.config)
+        return options

airbyte_cdk/sources/declarative/declarative_component_schema.yaml

@@ -1276,6 +1276,10 @@ definitions:
         type: string
         examples:
           - "{{ config['passphrase'] }}"
+      request_option:
+        title: Request Option
+        description: A request option describing where the signed JWT token that is generated should be injected into the outbound API request.
+        "$ref": "#/definitions/RequestOption"
       $parameters:
         type: object
         additionalProperties: true

airbyte_cdk/sources/declarative/models/declarative_component_schema.py

@@ -1,5 +1,3 @@
-# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
-
 # generated by datamodel-codegen:
 #   filename:  declarative_component_schema.yaml
 
@@ -397,66 +395,6 @@ class JwtPayload(BaseModel):
     )
 
 
-class JwtAuthenticator(BaseModel):
-    type: Literal["JwtAuthenticator"]
-    secret_key: str = Field(
-        ...,
-        description="Secret used to sign the JSON web token.",
-        examples=["{{ config['secret_key'] }}"],
-        title="Secret Key",
-    )
-    base64_encode_secret_key: Optional[bool] = Field(
-        False,
-        description='When set to true, the secret key will be base64 encoded prior to being encoded as part of the JWT. Only set to "true" when required by the API.',
-        title="Base64-encode Secret Key",
-    )
-    algorithm: Algorithm = Field(
-        ...,
-        description="Algorithm used to sign the JSON web token.",
-        examples=["ES256", "HS256", "RS256", "{{ config['algorithm'] }}"],
-        title="Algorithm",
-    )
-    token_duration: Optional[int] = Field(
-        1200,
-        description="The amount of time in seconds a JWT token can be valid after being issued.",
-        examples=[1200, 3600],
-        title="Token Duration",
-    )
-    header_prefix: Optional[str] = Field(
-        None,
-        description="The prefix to be used within the Authentication header.",
-        examples=["Bearer", "Basic"],
-        title="Header Prefix",
-    )
-    jwt_headers: Optional[JwtHeaders] = Field(
-        None,
-        description="JWT headers used when signing JSON web token.",
-        title="JWT Headers",
-    )
-    additional_jwt_headers: Optional[Dict[str, Any]] = Field(
-        None,
-        description="Additional headers to be included with the JWT headers object.",
-        title="Additional JWT Headers",
-    )
-    jwt_payload: Optional[JwtPayload] = Field(
-        None,
-        description="JWT Payload used when signing JSON web token.",
-        title="JWT Payload",
-    )
-    additional_jwt_payload: Optional[Dict[str, Any]] = Field(
-        None,
-        description="Additional properties to be added to the JWT payload.",
-        title="Additional JWT Payload Properties",
-    )
-    passphrase: Optional[str] = Field(
-        None,
-        description="A passphrase/password used to encrypt the private key. Only provide a passphrase if required by the API for JWT authentication. The API will typically provide the passphrase when generating the public/private key pair.",
-        examples=["{{ config['passphrase'] }}"],
-        title="Passphrase",
-    )
-    parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
-
-
 class RefreshTokenUpdater(BaseModel):
     refresh_token_name: Optional[str] = Field(
         "refresh_token",
@@ -502,148 +440,6 @@ class RefreshTokenUpdater(BaseModel):
     )
 
 
-class OAuthAuthenticator(BaseModel):
-    type: Literal["OAuthAuthenticator"]
-    client_id_name: Optional[str] = Field(
-        "client_id",
-        description="The name of the property to use to refresh the `access_token`.",
-        examples=["custom_app_id"],
-        title="Client ID Property Name",
-    )
-    client_id: Optional[str] = Field(
-        None,
-        description="The OAuth client ID. Fill it in the user inputs.",
-        examples=[
-            "{{ config['client_id'] }}",
-            "{{ config['credentials']['client_id }}",
-        ],
-        title="Client ID",
-    )
-    client_secret_name: Optional[str] = Field(
-        "client_secret",
-        description="The name of the property to use to refresh the `access_token`.",
-        examples=["custom_app_secret"],
-        title="Client Secret Property Name",
-    )
-    client_secret: Optional[str] = Field(
-        None,
-        description="The OAuth client secret. Fill it in the user inputs.",
-        examples=[
-            "{{ config['client_secret'] }}",
-            "{{ config['credentials']['client_secret }}",
-        ],
-        title="Client Secret",
-    )
-    refresh_token_name: Optional[str] = Field(
-        "refresh_token",
-        description="The name of the property to use to refresh the `access_token`.",
-        examples=["custom_app_refresh_value"],
-        title="Refresh Token Property Name",
-    )
-    refresh_token: Optional[str] = Field(
-        None,
-        description="Credential artifact used to get a new access token.",
-        examples=[
-            "{{ config['refresh_token'] }}",
-            "{{ config['credentials]['refresh_token'] }}",
-        ],
-        title="Refresh Token",
-    )
-    token_refresh_endpoint: Optional[str] = Field(
-        None,
-        description="The full URL to call to obtain a new access token.",
-        examples=["https://connect.squareup.com/oauth2/token"],
-        title="Token Refresh Endpoint",
-    )
-    access_token_name: Optional[str] = Field(
-        "access_token",
-        description="The name of the property which contains the access token in the response from the token refresh endpoint.",
-        examples=["access_token"],
-        title="Access Token Property Name",
-    )
-    access_token_value: Optional[str] = Field(
-        None,
-        description="The value of the access_token to bypass the token refreshing using `refresh_token`.",
-        examples=["secret_access_token_value"],
-        title="Access Token Value",
-    )
-    expires_in_name: Optional[str] = Field(
-        "expires_in",
-        description="The name of the property which contains the expiry date in the response from the token refresh endpoint.",
-        examples=["expires_in"],
-        title="Token Expiry Property Name",
-    )
-    grant_type_name: Optional[str] = Field(
-        "grant_type",
-        description="The name of the property to use to refresh the `access_token`.",
-        examples=["custom_grant_type"],
-        title="Grant Type Property Name",
-    )
-    grant_type: Optional[str] = Field(
-        "refresh_token",
-        description="Specifies the OAuth2 grant type. If set to refresh_token, the refresh_token needs to be provided as well. For client_credentials, only client id and secret are required. Other grant types are not officially supported.",
-        examples=["refresh_token", "client_credentials"],
-        title="Grant Type",
-    )
-    refresh_request_body: Optional[Dict[str, Any]] = Field(
-        None,
-        description="Body of the request sent to get a new access token.",
-        examples=[
-            {
-                "applicationId": "{{ config['application_id'] }}",
-                "applicationSecret": "{{ config['application_secret'] }}",
-                "token": "{{ config['token'] }}",
-            }
-        ],
-        title="Refresh Request Body",
-    )
-    refresh_request_headers: Optional[Dict[str, Any]] = Field(
-        None,
-        description="Headers of the request sent to get a new access token.",
-        examples=[
-            {
-                "Authorization": "<AUTH_TOKEN>",
-                "Content-Type": "application/x-www-form-urlencoded",
-            }
-        ],
-        title="Refresh Request Headers",
-    )
-    scopes: Optional[List[str]] = Field(
-        None,
-        description="List of scopes that should be granted to the access token.",
-        examples=[["crm.list.read", "crm.objects.contacts.read", "crm.schema.contacts.read"]],
-        title="Scopes",
-    )
-    token_expiry_date: Optional[str] = Field(
-        None,
-        description="The access token expiry date.",
-        examples=["2023-04-06T07:12:10.421833+00:00", 1680842386],
-        title="Token Expiry Date",
-    )
-    token_expiry_date_format: Optional[str] = Field(
-        None,
-        description="The format of the time to expiration datetime. Provide it if the time is returned as a date-time string instead of seconds.",
-        examples=["%Y-%m-%d %H:%M:%S.%f+00:00"],
-        title="Token Expiry Date Format",
-    )
-    refresh_token_updater: Optional[RefreshTokenUpdater] = Field(
-        None,
-        description="When the refresh token updater is defined, new refresh tokens, access tokens and the access token expiry date are written back from the authentication response to the config object. This is important if the refresh token can only used once.",
-        title="Refresh Token Updater",
-    )
-    profile_assertion: Optional[JwtAuthenticator] = Field(
-        None,
-        description="The authenticator being used to authenticate the client authenticator.",
-        title="Profile Assertion",
-    )
-    use_profile_assertion: Optional[bool] = Field(
-        False,
-        description="Enable using profile assertion as a flow for OAuth authorization.",
-        title="Use Profile Assertion",
-    )
-    parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
-
-
 class Rate(BaseModel):
     class Config:
         extra = Extra.allow
@@ -1895,6 +1691,213 @@ class DatetimeBasedCursor(BaseModel):
     parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
 
 
+class JwtAuthenticator(BaseModel):
+    type: Literal["JwtAuthenticator"]
+    secret_key: str = Field(
+        ...,
+        description="Secret used to sign the JSON web token.",
+        examples=["{{ config['secret_key'] }}"],
+        title="Secret Key",
+    )
+    base64_encode_secret_key: Optional[bool] = Field(
+        False,
+        description='When set to true, the secret key will be base64 encoded prior to being encoded as part of the JWT. Only set to "true" when required by the API.',
+        title="Base64-encode Secret Key",
+    )
+    algorithm: Algorithm = Field(
+        ...,
+        description="Algorithm used to sign the JSON web token.",
+        examples=["ES256", "HS256", "RS256", "{{ config['algorithm'] }}"],
+        title="Algorithm",
+    )
+    token_duration: Optional[int] = Field(
+        1200,
+        description="The amount of time in seconds a JWT token can be valid after being issued.",
+        examples=[1200, 3600],
+        title="Token Duration",
+    )
+    header_prefix: Optional[str] = Field(
+        None,
+        description="The prefix to be used within the Authentication header.",
+        examples=["Bearer", "Basic"],
+        title="Header Prefix",
+    )
+    jwt_headers: Optional[JwtHeaders] = Field(
+        None,
+        description="JWT headers used when signing JSON web token.",
+        title="JWT Headers",
+    )
+    additional_jwt_headers: Optional[Dict[str, Any]] = Field(
+        None,
+        description="Additional headers to be included with the JWT headers object.",
+        title="Additional JWT Headers",
+    )
+    jwt_payload: Optional[JwtPayload] = Field(
+        None,
+        description="JWT Payload used when signing JSON web token.",
+        title="JWT Payload",
+    )
+    additional_jwt_payload: Optional[Dict[str, Any]] = Field(
+        None,
+        description="Additional properties to be added to the JWT payload.",
+        title="Additional JWT Payload Properties",
+    )
+    passphrase: Optional[str] = Field(
+        None,
+        description="A passphrase/password used to encrypt the private key. Only provide a passphrase if required by the API for JWT authentication. The API will typically provide the passphrase when generating the public/private key pair.",
+        examples=["{{ config['passphrase'] }}"],
+        title="Passphrase",
+    )
+    request_option: Optional[RequestOption] = Field(
+        None,
+        description="A request option describing where the signed JWT token that is generated should be injected into the outbound API request.",
+        title="Request Option",
+    )
+    parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
+
+
+class OAuthAuthenticator(BaseModel):
+    type: Literal["OAuthAuthenticator"]
+    client_id_name: Optional[str] = Field(
+        "client_id",
+        description="The name of the property to use to refresh the `access_token`.",
+        examples=["custom_app_id"],
+        title="Client ID Property Name",
+    )
+    client_id: Optional[str] = Field(
+        None,
+        description="The OAuth client ID. Fill it in the user inputs.",
+        examples=[
+            "{{ config['client_id'] }}",
+            "{{ config['credentials']['client_id }}",
+        ],
+        title="Client ID",
+    )
+    client_secret_name: Optional[str] = Field(
+        "client_secret",
+        description="The name of the property to use to refresh the `access_token`.",
+        examples=["custom_app_secret"],
+        title="Client Secret Property Name",
+    )
+    client_secret: Optional[str] = Field(
+        None,
+        description="The OAuth client secret. Fill it in the user inputs.",
+        examples=[
+            "{{ config['client_secret'] }}",
+            "{{ config['credentials']['client_secret }}",
+        ],
+        title="Client Secret",
+    )
+    refresh_token_name: Optional[str] = Field(
+        "refresh_token",
+        description="The name of the property to use to refresh the `access_token`.",
+        examples=["custom_app_refresh_value"],
+        title="Refresh Token Property Name",
+    )
+    refresh_token: Optional[str] = Field(
+        None,
+        description="Credential artifact used to get a new access token.",
+        examples=[
+            "{{ config['refresh_token'] }}",
+            "{{ config['credentials]['refresh_token'] }}",
+        ],
+        title="Refresh Token",
+    )
+    token_refresh_endpoint: Optional[str] = Field(
+        None,
+        description="The full URL to call to obtain a new access token.",
+        examples=["https://connect.squareup.com/oauth2/token"],
+        title="Token Refresh Endpoint",
+    )
+    access_token_name: Optional[str] = Field(
+        "access_token",
+        description="The name of the property which contains the access token in the response from the token refresh endpoint.",
+        examples=["access_token"],
+        title="Access Token Property Name",
+    )
+    access_token_value: Optional[str] = Field(
+        None,
+        description="The value of the access_token to bypass the token refreshing using `refresh_token`.",
+        examples=["secret_access_token_value"],
+        title="Access Token Value",
+    )
+    expires_in_name: Optional[str] = Field(
+        "expires_in",
+        description="The name of the property which contains the expiry date in the response from the token refresh endpoint.",
+        examples=["expires_in"],
+        title="Token Expiry Property Name",
+    )
+    grant_type_name: Optional[str] = Field(
+        "grant_type",
+        description="The name of the property to use to refresh the `access_token`.",
+        examples=["custom_grant_type"],
+        title="Grant Type Property Name",
+    )
+    grant_type: Optional[str] = Field(
+        "refresh_token",
+        description="Specifies the OAuth2 grant type. If set to refresh_token, the refresh_token needs to be provided as well. For client_credentials, only client id and secret are required. Other grant types are not officially supported.",
+        examples=["refresh_token", "client_credentials"],
+        title="Grant Type",
+    )
+    refresh_request_body: Optional[Dict[str, Any]] = Field(
+        None,
+        description="Body of the request sent to get a new access token.",
+        examples=[
+            {
+                "applicationId": "{{ config['application_id'] }}",
+                "applicationSecret": "{{ config['application_secret'] }}",
+                "token": "{{ config['token'] }}",
+            }
+        ],
+        title="Refresh Request Body",
+    )
+    refresh_request_headers: Optional[Dict[str, Any]] = Field(
+        None,
+        description="Headers of the request sent to get a new access token.",
+        examples=[
+            {
+                "Authorization": "<AUTH_TOKEN>",
+                "Content-Type": "application/x-www-form-urlencoded",
+            }
+        ],
+        title="Refresh Request Headers",
+    )
+    scopes: Optional[List[str]] = Field(
+        None,
+        description="List of scopes that should be granted to the access token.",
+        examples=[["crm.list.read", "crm.objects.contacts.read", "crm.schema.contacts.read"]],
+        title="Scopes",
+    )
+    token_expiry_date: Optional[str] = Field(
+        None,
+        description="The access token expiry date.",
+        examples=["2023-04-06T07:12:10.421833+00:00", 1680842386],
+        title="Token Expiry Date",
+    )
+    token_expiry_date_format: Optional[str] = Field(
+        None,
+        description="The format of the time to expiration datetime. Provide it if the time is returned as a date-time string instead of seconds.",
+        examples=["%Y-%m-%d %H:%M:%S.%f+00:00"],
+        title="Token Expiry Date Format",
+    )
+    refresh_token_updater: Optional[RefreshTokenUpdater] = Field(
+        None,
+        description="When the refresh token updater is defined, new refresh tokens, access tokens and the access token expiry date are written back from the authentication response to the config object. This is important if the refresh token can only used once.",
+        title="Refresh Token Updater",
+    )
+    profile_assertion: Optional[JwtAuthenticator] = Field(
+        None,
+        description="The authenticator being used to authenticate the client authenticator.",
+        title="Profile Assertion",
+    )
+    use_profile_assertion: Optional[bool] = Field(
+        False,
+        description="Enable using profile assertion as a flow for OAuth authorization.",
+        title="Use Profile Assertion",
+    )
+    parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
+
+
 class FixedWindowCallRatePolicy(BaseModel):
     class Config:
         extra = Extra.allow

airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py

@@ -2683,12 +2683,16 @@ class ModelToComponentFactory:
             file_path=model.file_path or "", config=config, parameters=model.parameters or {}
         )
 
-    @staticmethod
     def create_jwt_authenticator(
-        model: JwtAuthenticatorModel, config: Config, **kwargs: Any
+        self, model: JwtAuthenticatorModel, config: Config, **kwargs: Any
     ) -> JwtAuthenticator:
         jwt_headers = model.jwt_headers or JwtHeadersModel(kid=None, typ="JWT", cty=None)
         jwt_payload = model.jwt_payload or JwtPayloadModel(iss=None, sub=None, aud=None)
+        request_option = (
+            self._create_component_from_model(model.request_option, config)
+            if model.request_option
+            else None
+        )
         return JwtAuthenticator(
             config=config,
             parameters=model.parameters or {},
@@ -2706,6 +2710,7 @@ class ModelToComponentFactory:
             additional_jwt_headers=model.additional_jwt_headers,
             additional_jwt_payload=model.additional_jwt_payload,
             passphrase=model.passphrase,
+            request_option=request_option,
         )
 
     def create_list_partition_router(

airbyte_cdk/sources/file_based/file_based_stream_reader.py

@@ -3,13 +3,15 @@
 #
 
 import logging
+import time
 from abc import ABC, abstractmethod
 from datetime import datetime
 from enum import Enum
 from io import IOBase
 from os import makedirs, path
-from typing import Any, Callable, Iterable, List, MutableMapping, Optional, Set, Tuple
+from typing import Any, Iterable, List, MutableMapping, Optional, Set, Tuple
 
+from airbyte_protocol_dataclasses.models import FailureType
 from wcmatch.glob import GLOBSTAR, globmatch
 
 from airbyte_cdk.models import AirbyteRecordMessageFileReference
@@ -19,8 +21,9 @@ from airbyte_cdk.sources.file_based.config.validate_config_transfer_modes import
     preserve_directory_structure,
     use_file_transfer,
 )
+from airbyte_cdk.sources.file_based.exceptions import FileSizeLimitError
 from airbyte_cdk.sources.file_based.file_record_data import FileRecordData
-from airbyte_cdk.sources.file_based.remote_file import RemoteFile
+from airbyte_cdk.sources.file_based.remote_file import RemoteFile, UploadableRemoteFile
 
 
 class FileReadMode(Enum):
@@ -34,6 +37,7 @@ class AbstractFileBasedStreamReader(ABC):
     FILE_NAME = "file_name"
     LOCAL_FILE_PATH = "local_file_path"
     FILE_FOLDER = "file_folder"
+    FILE_SIZE_LIMIT = 1_500_000_000
 
     def __init__(self) -> None:
         self._config = None
@@ -113,16 +117,6 @@ class AbstractFileBasedStreamReader(ABC):
                     seen.add(file.uri)
                     yield file
 
-    @abstractmethod
-    def file_size(self, file: RemoteFile) -> int:
-        """Utility method to get size of the remote file.
-
-        This is required for connectors that will support writing to
-        files. If the connector does not support writing files, then the
-        subclass can simply `return 0`.
-        """
-        ...
-
     @staticmethod
     def file_matches_globs(file: RemoteFile, globs: List[str]) -> bool:
         # Use the GLOBSTAR flag to enable recursive ** matching
@@ -153,9 +147,8 @@ class AbstractFileBasedStreamReader(ABC):
             return include_identities_stream(self.config)
         return False
 
-    @abstractmethod
     def upload(
-        self, file: RemoteFile, local_directory: str, logger: logging.Logger
+        self, file: UploadableRemoteFile, local_directory: str, logger: logging.Logger
     ) -> Tuple[FileRecordData, AirbyteRecordMessageFileReference]:
         """
         This is required for connectors that will support writing to
@@ -173,7 +166,53 @@ class AbstractFileBasedStreamReader(ABC):
                    - file_size_bytes (int): The size of the referenced file in bytes.
                    - source_file_relative_path (str): The relative path to the referenced file in source.
         """
-        ...
+        if not isinstance(file, UploadableRemoteFile):
+            raise TypeError(f"Expected UploadableRemoteFile, got {type(file)}")
+
+        file_size = file.size
+
+        if file_size > self.FILE_SIZE_LIMIT:
+            message = f"File size exceeds the {self.FILE_SIZE_LIMIT / 1e9} GB limit."
+            raise FileSizeLimitError(
+                message=message, internal_message=message, failure_type=FailureType.config_error
+            )
+
+        file_paths = self._get_file_transfer_paths(
+            source_file_relative_path=file.source_file_relative_path,
+            staging_directory=local_directory,
+        )
+        local_file_path = file_paths[self.LOCAL_FILE_PATH]
+        file_relative_path = file_paths[self.FILE_RELATIVE_PATH]
+        file_name = file_paths[self.FILE_NAME]
+
+        logger.info(
+            f"Starting to download the file {file.file_uri_for_logging} with size: {file_size / (1024 * 1024):,.2f} MB ({file_size / (1024 * 1024 * 1024):.2f} GB)"
+        )
+        start_download_time = time.time()
+
+        file.download_to_local_directory(local_file_path)
+
+        write_duration = time.time() - start_download_time
+        logger.info(
+            f"Finished downloading the file {file.file_uri_for_logging} and saved to {local_file_path} in {write_duration:,.2f} seconds."
+        )
+
+        file_record_data = FileRecordData(
+            folder=file_paths[self.FILE_FOLDER],
+            file_name=file_name,
+            bytes=file_size,
+            id=file.id,
+            mime_type=file.mime_type,
+            created_at=file.created_at,
+            updated_at=file.updated_at,
+            source_uri=file.uri,
+        )
+        file_reference = AirbyteRecordMessageFileReference(
+            staging_file_url=local_file_path,
+            source_file_relative_path=file_relative_path,
+            file_size_bytes=file_size,
+        )
+        return file_record_data, file_reference
 
     def _get_file_transfer_paths(
         self, source_file_relative_path: str, staging_directory: str

airbyte_cdk/sources/file_based/file_types/file_transfer.py

@@ -7,7 +7,7 @@ from typing import Iterable, Tuple
 from airbyte_cdk.models import AirbyteRecordMessageFileReference
 from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader
 from airbyte_cdk.sources.file_based.file_record_data import FileRecordData
-from airbyte_cdk.sources.file_based.remote_file import RemoteFile
+from airbyte_cdk.sources.file_based.remote_file import UploadableRemoteFile
 from airbyte_cdk.sources.utils.files_directory import get_files_directory
 
 
@@ -17,7 +17,7 @@ class FileTransfer:
 
     def upload(
         self,
-        file: RemoteFile,
+        file: UploadableRemoteFile,
         stream_reader: AbstractFileBasedStreamReader,
         logger: logging.Logger,
     ) -> Iterable[Tuple[FileRecordData, AirbyteRecordMessageFileReference]]:

airbyte_cdk/sources/file_based/remote_file.py

@@ -1,7 +1,7 @@
 #
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
-
+from abc import ABC, abstractmethod
 from datetime import datetime
 from typing import Optional
 
@@ -16,3 +16,42 @@ class RemoteFile(BaseModel):
     uri: str
     last_modified: datetime
     mime_type: Optional[str] = None
+
+
+class UploadableRemoteFile(RemoteFile, ABC):
+    """
+    A file in a file-based stream that supports uploading(file transferring).
+    """
+
+    id: Optional[str] = None
+    created_at: Optional[str] = None
+    updated_at: Optional[str] = None
+
+    @property
+    @abstractmethod
+    def size(self) -> int:
+        """
+        Returns the file size in bytes.
+        """
+        ...
+
+    @abstractmethod
+    def download_to_local_directory(self, local_file_path: str) -> None:
+        """
+        Download the file from remote source to local storage.
+        """
+        ...
+
+    @property
+    def source_file_relative_path(self) -> str:
+        """
+        Returns the relative path of the source file.
+        """
+        return self.uri
+
+    @property
+    def file_uri_for_logging(self) -> str:
+        """
+        Returns the URI for the file being logged.
+        """
+        return self.uri

{airbyte_cdk-7.2.3.dist-info → airbyte_cdk-7.3.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 7.2.3
+Version: 7.3.1
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://airbyte.com
 License: MIT

{airbyte_cdk-7.2.3.dist-info → airbyte_cdk-7.3.1.dist-info}/RECORD

@@ -115,7 +115,7 @@ airbyte_cdk/sources/declarative/async_job/status.py,sha256=mkExR-uOAO1ckUnclaUOa
 airbyte_cdk/sources/declarative/async_job/timer.py,sha256=Fb8P72CQ7jIzJyzMSSNuBf2vt8bmrg9SrfmNxKwph2A,1242
 airbyte_cdk/sources/declarative/auth/__init__.py,sha256=e2CRrcBWGhz3sQu3Oh34d1riEIwXipGS8hrSB1pu0Oo,284
 airbyte_cdk/sources/declarative/auth/declarative_authenticator.py,sha256=nf-OmRUHYG4ORBwyb5CANzuHEssE-oNmL-Lccn41Td8,1099
-airbyte_cdk/sources/declarative/auth/jwt.py,sha256=zZANSwaq-LkO22VbcdZloRrv5u7zTaC770xvWRtSKrE,9779
+airbyte_cdk/sources/declarative/auth/jwt.py,sha256=Zf2U17HZAzzXGAnneAuq4zynrE7OalasVeMYSnvPdZc,11190
 airbyte_cdk/sources/declarative/auth/oauth.py,sha256=bCwf3f3Td_CA8DZ6CXMVPNiImM9QEGDxkcLKzSo3-f0,14339
 airbyte_cdk/sources/declarative/auth/selective_authenticator.py,sha256=qGwC6YsCldr1bIeKG6Qo-A9a5cTdHw-vcOn3OtQrS4c,1540
 airbyte_cdk/sources/declarative/auth/token.py,sha256=2EnE78EhBOY9hbeZnQJ9AuFaM-G7dccU-oKo_LThRQk,11070
@@ -130,7 +130,7 @@ airbyte_cdk/sources/declarative/concurrent_declarative_source.py,sha256=z0AgJ6AZ
 airbyte_cdk/sources/declarative/datetime/__init__.py,sha256=4Hw-PX1-VgESLF16cDdvuYCzGJtHntThLF4qIiULWeo,61
 airbyte_cdk/sources/declarative/datetime/datetime_parser.py,sha256=_zGNGq31RNy_0QBLt_EcTvgPyhj7urPdx6oA3M5-r3o,3150
 airbyte_cdk/sources/declarative/datetime/min_max_datetime.py,sha256=0BHBtDNQZfvwM45-tY5pNlTcKAFSGGNxemoi0Jic-0E,5785
-airbyte_cdk/sources/declarative/declarative_component_schema.yaml,sha256=dWfbQUSNdW1HuY1NZslMNL8sTF-I3mKFXa9Fo3747-0,187551
+airbyte_cdk/sources/declarative/declarative_component_schema.yaml,sha256=D5WeK1Iw_T0ZxLKCmKLowyO7GwxnwfhYom-sd1W14uQ,187793
 airbyte_cdk/sources/declarative/decoders/__init__.py,sha256=JHb_0d3SE6kNY10mxA5YBEKPeSbsWYjByq1gUQxepoE,953
 airbyte_cdk/sources/declarative/decoders/composite_raw_decoder.py,sha256=qB4lRUrCXLTE-a3VlpOLaazHiC7RIF_FIVJesuz7ebw,8078
 airbyte_cdk/sources/declarative/decoders/decoder.py,sha256=1PeKwuMK8x9dsA2zqUjSVinEWVSEgYcUS6npiW3aC2c,855
@@ -165,14 +165,14 @@ airbyte_cdk/sources/declarative/migrations/legacy_to_per_partition_state_migrati
 airbyte_cdk/sources/declarative/migrations/state_migration.py,sha256=KWPjealMLKSMtajXgkdGgKg7EmTLR-CqqD7UIh0-eDU,794
 airbyte_cdk/sources/declarative/models/__init__.py,sha256=nUFxNCiKeYRVXuZEKA7GD-lTHxsiKcQ8FitZjKhPIvE,100
 airbyte_cdk/sources/declarative/models/base_model_with_deprecations.py,sha256=Imnj3yef0aqRdLfaUxkIYISUb8YkiPrRH_wBd-x8HjM,5999
-airbyte_cdk/sources/declarative/models/declarative_component_schema.py,sha256=THYAUpV9dDO3n7nTshHrq9iMulGDy60QKM1_jy5t8Ak,131930
+airbyte_cdk/sources/declarative/models/declarative_component_schema.py,sha256=t2td-7swgXY3RJc1VDBFUYI2Blc55j5TDFdg90aHwlU,132123
 airbyte_cdk/sources/declarative/parsers/__init__.py,sha256=ZnqYNxHsKCgO38IwB34RQyRMXTs4GTvlRi3ImKnIioo,61
 airbyte_cdk/sources/declarative/parsers/custom_code_compiler.py,sha256=nlVvHC511NUyDEEIRBkoeDTAvLqKNp-hRy8D19z8tdk,5941
 airbyte_cdk/sources/declarative/parsers/custom_exceptions.py,sha256=wnRUP0Xeru9Rbu5OexXSDN9QWDo8YU4tT9M2LDVOgGA,802
 airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py,sha256=la9Ulpc0lQewiBLKJ0FpsWxyU5XISv-ulmFRHJLJ1Pc,11292
 airbyte_cdk/sources/declarative/parsers/manifest_normalizer.py,sha256=EtKjS9c94yNp3AwQC8KUCQaAYW5T3zvFYxoWYjc_buI,19729
 airbyte_cdk/sources/declarative/parsers/manifest_reference_resolver.py,sha256=pJmg78vqE5VfUrF_KJnWjucQ4k9IWFULeAxHCowrHXE,6806
-airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py,sha256=1U2IMabGxODG9-sh2zNNHUBk3bkcWFssPisFL76vIVk,183420
+airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py,sha256=lMjcNJKqU3HcOjNZyR4_Ln1v32HK-7iNcFq4nmFjxSE,183622
 airbyte_cdk/sources/declarative/partition_routers/__init__.py,sha256=TBC9AkGaUqHm2IKHMPN6punBIcY5tWGULowcLoAVkfw,1109
 airbyte_cdk/sources/declarative/partition_routers/async_job_partition_router.py,sha256=VelO7zKqKtzMJ35jyFeg0ypJLQC0plqqIBNXoBW1G2E,3001
 airbyte_cdk/sources/declarative/partition_routers/cartesian_product_stream_slicer.py,sha256=ocm4hZ4k-tEGs5HLrtI8ecWSK0hGqNH0Rvz2byx_HZk,6927
@@ -298,18 +298,18 @@ airbyte_cdk/sources/file_based/discovery_policy/default_discovery_policy.py,sha2
 airbyte_cdk/sources/file_based/exceptions.py,sha256=WP0qkG6fpWoBpOyyicgp5YNE393VWyegq5qSy0v4QtM,7362
 airbyte_cdk/sources/file_based/file_based_source.py,sha256=Xg8OYWnGc-OcVBglvS08uwAWGWHBhEqsBnyODIkOK-4,20051
 airbyte_cdk/sources/file_based/file_based_stream_permissions_reader.py,sha256=4e7FXqQ9hueacexC0SyrZyjF8oREYHza8pKF9CgKbD8,5050
-airbyte_cdk/sources/file_based/file_based_stream_reader.py,sha256=rwz8AhEIqYB9gBF7uW9eR--eUiHOntzuwLH8jFHNacE,7854
+airbyte_cdk/sources/file_based/file_based_stream_reader.py,sha256=Yg9KRXpyAtElBrUOO8oX4WHQH6k6Lk7keklrZmB5Klg,9614
 airbyte_cdk/sources/file_based/file_record_data.py,sha256=Vkr5AyZzlsOezjVCLhFrm_WpymlQdolWCnFAwqLJ9Iw,453
 airbyte_cdk/sources/file_based/file_types/__init__.py,sha256=blCLn0-2LC-ZdgcNyDEhqM2RiUvEjEBh-G4-t32ZtuM,1268
 airbyte_cdk/sources/file_based/file_types/avro_parser.py,sha256=USEYqiICXBWpDV443VtNOCmUA-GINzY_Zah74_5w3qQ,10860
 airbyte_cdk/sources/file_based/file_types/csv_parser.py,sha256=QlCXB-ry3np67Q_VerQEPoWDOTcPTB6Go4ydZxY9ae4,20445
 airbyte_cdk/sources/file_based/file_types/excel_parser.py,sha256=BeplCq0hmojELU6bZCvvpRLpQ9us81TqbGYwrhd3INo,7188
-airbyte_cdk/sources/file_based/file_types/file_transfer.py,sha256=5l2Jo6bp6neDmgM427PrZMZeqU0hCIZVWnzUZ_7BT10,1100
+airbyte_cdk/sources/file_based/file_types/file_transfer.py,sha256=rFxWaqItBux9tPf4xU03LT6b-wDZf1QolM92mP8Diuk,1120
 airbyte_cdk/sources/file_based/file_types/file_type_parser.py,sha256=JgpH21PrbRqwK92BJklZWvh2TndA6xZ-eP1LPMo44oQ,2832
 airbyte_cdk/sources/file_based/file_types/jsonl_parser.py,sha256=GwyNyxmST4RX-XpXy7xVH0D-znYWWBmGv_pVAu95oHQ,5886
 airbyte_cdk/sources/file_based/file_types/parquet_parser.py,sha256=XenFg5sJ-UBnIkSmsiNJRou11NO0zZXx-RXgPHMT2NA,10487
 airbyte_cdk/sources/file_based/file_types/unstructured_parser.py,sha256=2TYOQl62FQPCa8otLbkDIk_j01EP3oWaKSfXGhCjCHg,19492
-airbyte_cdk/sources/file_based/remote_file.py,sha256=yqRz93vPe8PBXLIMJ5W5u2JRlZRhg6sBrAjn3pPjJ8A,315
+airbyte_cdk/sources/file_based/remote_file.py,sha256=1Afzr2WFWwjiUz8R2vNFepeI192UNeHOZAXIGTWOzOM,1248
 airbyte_cdk/sources/file_based/schema_helpers.py,sha256=dKXAOTmMI3YmC5u7PeHC9AaZmlL6ft7CYSFQKCg0sXw,9911
 airbyte_cdk/sources/file_based/schema_validation_policies/__init__.py,sha256=FkByIyEy56x2_awYnxGPqGaOp7zAzpAoRkPZHKySI9M,536
 airbyte_cdk/sources/file_based/schema_validation_policies/abstract_schema_validation_policy.py,sha256=kjvX7nOmUALYd7HuZHilUzgJPZ-MnZ08mtvuBnt2tQ0,618
@@ -457,9 +457,9 @@ airbyte_cdk/utils/slice_hasher.py,sha256=EDxgROHDbfG-QKQb59m7h_7crN1tRiawdf5uU7G
 airbyte_cdk/utils/spec_schema_transformations.py,sha256=9YDJmnIGFsT51CVQf2tSSvTapGimITjEFGbUTSZAGTI,963
 airbyte_cdk/utils/stream_status_utils.py,sha256=ZmBoiy5HVbUEHAMrUONxZvxnvfV9CesmQJLDTAIWnWw,1171
 airbyte_cdk/utils/traced_exception.py,sha256=C8uIBuCL_E4WnBAOPSxBicD06JAldoN9fGsQDp463OY,6292
-airbyte_cdk-7.2.3.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
-airbyte_cdk-7.2.3.dist-info/LICENSE_SHORT,sha256=aqF6D1NcESmpn-cqsxBtszTEnHKnlsp8L4x9wAh3Nxg,55
-airbyte_cdk-7.2.3.dist-info/METADATA,sha256=gZXCUx6Kq31vc4g4TcmcOfaNbnKlW4af5Ka_WqVjvSg,6798
-airbyte_cdk-7.2.3.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-airbyte_cdk-7.2.3.dist-info/entry_points.txt,sha256=eLZ2UYvJZGm1s07Pplcs--1Gim60YhZWTb53j_dghwU,195
-airbyte_cdk-7.2.3.dist-info/RECORD,,
+airbyte_cdk-7.3.1.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
+airbyte_cdk-7.3.1.dist-info/LICENSE_SHORT,sha256=aqF6D1NcESmpn-cqsxBtszTEnHKnlsp8L4x9wAh3Nxg,55
+airbyte_cdk-7.3.1.dist-info/METADATA,sha256=_n29oKSyO6A6mUMN1c6YqHvrJRFQrXOIhhAh1E0PuXo,6798
+airbyte_cdk-7.3.1.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+airbyte_cdk-7.3.1.dist-info/entry_points.txt,sha256=eLZ2UYvJZGm1s07Pplcs--1Gim60YhZWTb53j_dghwU,195
+airbyte_cdk-7.3.1.dist-info/RECORD,,