airbyte-cdk 6.7.1rc3__py3-none-any.whl → 6.7.2__py3-none-any.whl

airbyte_cdk/sources/declarative/extractors/record_filter.py

@@ -1,7 +1,6 @@
 #
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
-import datetime
 from dataclasses import InitVar, dataclass
 from typing import Any, Iterable, Mapping, Optional, Union
 
@@ -11,7 +10,7 @@ from airbyte_cdk.sources.declarative.incremental import (
     PerPartitionWithGlobalCursor,
 )
 from airbyte_cdk.sources.declarative.interpolation.interpolated_boolean import InterpolatedBoolean
-from airbyte_cdk.sources.types import Config, StreamSlice, StreamState
+from airbyte_cdk.sources.types import Config, Record, StreamSlice, StreamState
 
 
 @dataclass
@@ -68,20 +67,6 @@ class ClientSideIncrementalRecordFilterDecorator(RecordFilter):
         self._date_time_based_cursor = date_time_based_cursor
         self._substream_cursor = substream_cursor
 
-    @property
-    def _cursor_field(self) -> str:
-        return self._date_time_based_cursor.cursor_field.eval(self._date_time_based_cursor.config)  # type: ignore # eval returns a string in this context
-
-    @property
-    def _start_date_from_config(self) -> datetime.datetime:
-        return self._date_time_based_cursor._start_datetime.get_datetime(
-            self._date_time_based_cursor.config
-        )
-
-    @property
-    def _end_datetime(self) -> datetime.datetime:
-        return self._date_time_based_cursor.select_best_end_datetime()
-
     def filter_records(
         self,
         records: Iterable[Mapping[str, Any]],
@@ -89,16 +74,14 @@ class ClientSideIncrementalRecordFilterDecorator(RecordFilter):
         stream_slice: Optional[StreamSlice] = None,
         next_page_token: Optional[Mapping[str, Any]] = None,
     ) -> Iterable[Mapping[str, Any]]:
-        state_value = self._get_state_value(
-            stream_state, stream_slice or StreamSlice(partition={}, cursor_slice={})
-        )
-        filter_date: datetime.datetime = self._get_filter_date(state_value)
         records = (
             record
             for record in records
-            if self._end_datetime
-            >= self._date_time_based_cursor.parse_date(record[self._cursor_field])
-            >= filter_date
+            if (self._substream_cursor or self._date_time_based_cursor).should_be_synced(
+                # Record is created on the fly to align with cursors interface; stream name is ignored as we don't need it here
+                # Record stream name is empty cause it is not used durig the filtering
+                Record(data=record, associated_slice=stream_slice, stream_name="")
+            )
         )
         if self.condition:
             records = super().filter_records(
@@ -108,28 +91,3 @@ class ClientSideIncrementalRecordFilterDecorator(RecordFilter):
                 next_page_token=next_page_token,
             )
         yield from records
-
-    def _get_state_value(
-        self, stream_state: StreamState, stream_slice: StreamSlice
-    ) -> Optional[str]:
-        """
-        Return cursor_value or None in case it was not found.
-        Cursor_value may be empty if:
-            1. It is an initial sync => no stream_state exist at all.
-            2. In Parent-child stream, and we already make initial sync, so stream_state is present.
-               During the second read, we receive one extra record from parent and therefore no stream_state for this record will be found.
-
-        :param StreamState stream_state: State
-        :param StreamSlice stream_slice: Current Stream slice
-        :return Optional[str]: cursor_value in case it was found, otherwise None.
-        """
-        state = (self._substream_cursor or self._date_time_based_cursor).select_state(stream_slice)
-
-        return state.get(self._cursor_field) if state else None
-
-    def _get_filter_date(self, state_value: Optional[str]) -> datetime.datetime:
-        start_date_parsed = self._start_date_from_config
-        if state_value:
-            return max(start_date_parsed, self._date_time_based_cursor.parse_date(state_value))
-        else:
-            return start_date_parsed

airbyte_cdk/sources/declarative/extractors/record_selector.py

@@ -3,12 +3,14 @@
 #
 
 from dataclasses import InitVar, dataclass, field
-from typing import Any, Iterable, List, Mapping, Optional
+from typing import Any, Iterable, List, Mapping, Optional, Union
 
 import requests
+
 from airbyte_cdk.sources.declarative.extractors.http_selector import HttpSelector
 from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor
 from airbyte_cdk.sources.declarative.extractors.record_filter import RecordFilter
+from airbyte_cdk.sources.declarative.interpolation import InterpolatedString
 from airbyte_cdk.sources.declarative.models import SchemaNormalization
 from airbyte_cdk.sources.declarative.transformations import RecordTransformation
 from airbyte_cdk.sources.types import Config, Record, StreamSlice, StreamState
@@ -37,11 +39,34 @@ class RecordSelector(HttpSelector):
     config: Config
     parameters: InitVar[Mapping[str, Any]]
     schema_normalization: TypeTransformer
+    name: str
+    _name: Union[InterpolatedString, str] = field(init=False, repr=False, default="")
     record_filter: Optional[RecordFilter] = None
     transformations: List[RecordTransformation] = field(default_factory=lambda: [])
 
     def __post_init__(self, parameters: Mapping[str, Any]) -> None:
         self._parameters = parameters
+        self._name = (
+            InterpolatedString(self._name, parameters=parameters)
+            if isinstance(self._name, str)
+            else self._name
+        )
+
+    @property  # type: ignore
+    def name(self) -> str:
+        """
+        :return: Stream name
+        """
+        return (
+            str(self._name.eval(self.config))
+            if isinstance(self._name, InterpolatedString)
+            else self._name
+        )
+
+    @name.setter
+    def name(self, value: str) -> None:
+        if not isinstance(value, property):
+            self._name = value
 
     def select_records(
         self,
@@ -85,7 +110,7 @@ class RecordSelector(HttpSelector):
         transformed_data = self._transform(filtered_data, stream_state, stream_slice)
         normalized_data = self._normalize_by_schema(transformed_data, schema=records_schema)
         for data in normalized_data:
-            yield Record(data, stream_slice)
+            yield Record(data=data, stream_name=self.name, associated_slice=stream_slice)
 
     def _normalize_by_schema(
         self, records: Iterable[Mapping[str, Any]], schema: Optional[Mapping[str, Any]]
@@ -125,6 +150,9 @@ class RecordSelector(HttpSelector):
         for record in records:
             for transformation in self.transformations:
                 transformation.transform(
-                    record, config=self.config, stream_state=stream_state, stream_slice=stream_slice
-                )  # type: ignore  # record has type Mapping[str, Any], but Dict[str, Any] expected
+                    record,  # type: ignore  # record has type Mapping[str, Any], but Dict[str, Any] expected
+                    config=self.config,
+                    stream_state=stream_state,
+                    stream_slice=stream_slice,
+                )
             yield record

airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py

@@ -6,18 +6,21 @@ import os
 import uuid
 import zlib
 from contextlib import closing
+from dataclasses import InitVar, dataclass
 from typing import Any, Dict, Iterable, Mapping, Optional, Tuple
 
 import pandas as pd
 import requests
-from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor
 from numpy import nan
 
+from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor
+
 EMPTY_STR: str = ""
 DEFAULT_ENCODING: str = "utf-8"
 DOWNLOAD_CHUNK_SIZE: int = 1024 * 10
 
 
+@dataclass
 class ResponseToFileExtractor(RecordExtractor):
     """
     This class is used when having very big HTTP responses (usually streamed) which would require too much memory so we use disk space as
@@ -27,7 +30,9 @@ class ResponseToFileExtractor(RecordExtractor):
     a first iteration so we will only support CSV parsing using pandas as salesforce and sendgrid were doing.
     """
 
-    def __init__(self) -> None:
+    parameters: InitVar[Mapping[str, Any]]
+
+    def __post_init__(self, parameters: Mapping[str, Any]) -> None:
         self.logger = logging.getLogger("airbyte")
 
     def _get_response_encoding(self, headers: Dict[str, Any]) -> str:

airbyte_cdk/sources/declarative/incremental/datetime_based_cursor.py

@@ -7,6 +7,8 @@ from dataclasses import InitVar, dataclass, field
 from datetime import timedelta
 from typing import Any, Callable, Iterable, List, Mapping, MutableMapping, Optional, Union
 
+from isodate import Duration, duration_isoformat, parse_duration
+
 from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, Level, Type
 from airbyte_cdk.sources.declarative.datetime.datetime_parser import DatetimeParser
 from airbyte_cdk.sources.declarative.datetime.min_max_datetime import MinMaxDatetime
@@ -19,7 +21,6 @@ from airbyte_cdk.sources.declarative.requesters.request_option import (
 )
 from airbyte_cdk.sources.message import MessageRepository
 from airbyte_cdk.sources.types import Config, Record, StreamSlice, StreamState
-from isodate import Duration, duration_isoformat, parse_duration
 
 
 @dataclass

airbyte_cdk/sources/declarative/incremental/global_substream_cursor.py

@@ -340,8 +340,11 @@ class GlobalSubstreamCursor(DeclarativeCursor):
     @staticmethod
     def _convert_record_to_cursor_record(record: Record) -> Record:
         return Record(
-            record.data,
-            StreamSlice(partition={}, cursor_slice=record.associated_slice.cursor_slice)
+            data=record.data,
+            stream_name=record.stream_name,
+            associated_slice=StreamSlice(
+                partition={}, cursor_slice=record.associated_slice.cursor_slice
+            )
             if record.associated_slice
             else None,
         )

airbyte_cdk/sources/declarative/incremental/per_partition_cursor.py

@@ -325,8 +325,11 @@ class PerPartitionCursor(DeclarativeCursor):
     @staticmethod
     def _convert_record_to_cursor_record(record: Record) -> Record:
         return Record(
-            record.data,
-            StreamSlice(partition={}, cursor_slice=record.associated_slice.cursor_slice)
+            data=record.data,
+            stream_name=record.stream_name,
+            associated_slice=StreamSlice(
+                partition={}, cursor_slice=record.associated_slice.cursor_slice
+            )
             if record.associated_slice
             else None,
         )

airbyte_cdk/sources/declarative/incremental/per_partition_with_global.py

@@ -194,9 +194,7 @@ class PerPartitionWithGlobalCursor(DeclarativeCursor):
         )
 
     def should_be_synced(self, record: Record) -> bool:
-        return self._global_cursor.should_be_synced(
-            record
-        ) or self._per_partition_cursor.should_be_synced(record)
+        return self._get_active_cursor().should_be_synced(record)
 
     def is_greater_than_or_equal(self, first: Record, second: Record) -> bool:
         return self._global_cursor.is_greater_than_or_equal(first, second)

airbyte_cdk/sources/declarative/interpolation/jinja.py

@@ -6,15 +6,16 @@ import ast
 from functools import cache
 from typing import Any, Mapping, Optional, Tuple, Type
 
-from airbyte_cdk.sources.declarative.interpolation.filters import filters
-from airbyte_cdk.sources.declarative.interpolation.interpolation import Interpolation
-from airbyte_cdk.sources.declarative.interpolation.macros import macros
-from airbyte_cdk.sources.types import Config
 from jinja2 import meta
 from jinja2.environment import Template
 from jinja2.exceptions import UndefinedError
 from jinja2.sandbox import SandboxedEnvironment
 
+from airbyte_cdk.sources.declarative.interpolation.filters import filters
+from airbyte_cdk.sources.declarative.interpolation.interpolation import Interpolation
+from airbyte_cdk.sources.declarative.interpolation.macros import macros
+from airbyte_cdk.sources.types import Config
+
 
 class StreamPartitionAccessEnvironment(SandboxedEnvironment):
     """

airbyte_cdk/sources/declarative/manifest_declarative_source.py

@@ -8,9 +8,12 @@ import pkgutil
 from copy import deepcopy
 from importlib import metadata
 from typing import Any, Dict, Iterator, List, Mapping, Optional
-from packaging.version import Version, InvalidVersion
 
 import yaml
+from jsonschema.exceptions import ValidationError
+from jsonschema.validators import validate
+from packaging.version import InvalidVersion, Version
+
 from airbyte_cdk.models import (
     AirbyteConnectionStatus,
     AirbyteMessage,
@@ -44,8 +47,6 @@ from airbyte_cdk.sources.utils.slice_logger import (
     DebugSliceLogger,
     SliceLogger,
 )
-from jsonschema.exceptions import ValidationError
-from jsonschema.validators import validate
 
 
 class ManifestDeclarativeSource(DeclarativeSource):

airbyte_cdk/sources/declarative/migrations/legacy_to_per_partition_state_migration.py

@@ -5,9 +5,9 @@ from typing import Any, Mapping
 from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
 from airbyte_cdk.sources.declarative.migrations.state_migration import StateMigration
 from airbyte_cdk.sources.declarative.models import (
+    CustomIncrementalSync,
     DatetimeBasedCursor,
     SubstreamPartitionRouter,
-    CustomIncrementalSync,
 )
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import ParentStreamConfig
 

airbyte_cdk/sources/declarative/models/declarative_component_schema.py

@@ -567,6 +567,11 @@ class DpathExtractor(BaseModel):
     parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
 
 
+class ResponseToFileExtractor(BaseModel):
+    type: Literal["ResponseToFileExtractor"]
+    parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
+
+
 class ExponentialBackoffStrategy(BaseModel):
     type: Literal["ExponentialBackoffStrategy"]
     factor: Optional[Union[float, str]] = Field(
@@ -748,6 +753,123 @@ class NoPagination(BaseModel):
     type: Literal["NoPagination"]
 
 
+class State(BaseModel):
+    class Config:
+        extra = Extra.allow
+
+    min: int
+    max: int
+
+
+class OauthConnectorInputSpecification(BaseModel):
+    class Config:
+        extra = Extra.allow
+
+    consent_url: str = Field(
+        ...,
+        description="The DeclarativeOAuth Specific string URL string template to initiate the authentication.\nThe placeholders are replaced during the processing to provide neccessary values.",
+        examples=[
+            {
+                "consent_url": "https://domain.host.com/marketing_api/auth?{client_id_key}={{client_id_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}&{state_key}={{state_key}}"
+            },
+            {
+                "consent_url": "https://endpoint.host.com/oauth2/authorize?{client_id_key}={{client_id_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}&{scope_key}={urlEncoder:{{scope_key}}}&{state_key}={{state_key}}&subdomain={subdomain}"
+            },
+        ],
+        title="DeclarativeOAuth Consent URL",
+    )
+    scope: Optional[str] = Field(
+        None,
+        description="The DeclarativeOAuth Specific string of the scopes needed to be grant for authenticated user.",
+        examples=[{"scope": "user:read user:read_orders workspaces:read"}],
+        title="(Optional) DeclarativeOAuth Scope",
+    )
+    access_token_url: str = Field(
+        ...,
+        description="The DeclarativeOAuth Specific URL templated string to obtain the `access_token`, `refresh_token` etc.\nThe placeholders are replaced during the processing to provide neccessary values.",
+        examples=[
+            {
+                "access_token_url": "https://auth.host.com/oauth2/token?{client_id_key}={{client_id_key}}&{client_secret_key}={{client_secret_key}}&{auth_code_key}={{auth_code_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}"
+            }
+        ],
+        title="DeclarativeOAuth Access Token URL",
+    )
+    access_token_headers: Optional[Dict[str, Any]] = Field(
+        None,
+        description="The DeclarativeOAuth Specific optional headers to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.",
+        examples=[
+            {
+                "access_token_headers": {
+                    "Authorization": "Basic {base64Encoder:{client_id}:{client_secret}}"
+                }
+            }
+        ],
+        title="(Optional) DeclarativeOAuth Access Token Headers",
+    )
+    access_token_params: Optional[Dict[str, Any]] = Field(
+        None,
+        description="The DeclarativeOAuth Specific optional query parameters to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.\nWhen this property is provided, the query params will be encoded as `Json` and included in the outgoing API request.",
+        examples=[
+            {
+                "access_token_params": {
+                    "{auth_code_key}": "{{auth_code_key}}",
+                    "{client_id_key}": "{{client_id_key}}",
+                    "{client_secret_key}": "{{client_secret_key}}",
+                }
+            }
+        ],
+        title="(Optional) DeclarativeOAuth Access Token Query Params (Json Encoded)",
+    )
+    extract_output: List[str] = Field(
+        ...,
+        description="The DeclarativeOAuth Specific list of strings to indicate which keys should be extracted and returned back to the input config.                ",
+        examples=[{"extract_output": ["access_token", "refresh_token", "other_field"]}],
+        title="DeclarativeOAuth Extract Output",
+    )
+    state: Optional[State] = Field(
+        None,
+        description="The DeclarativeOAuth Specific object to provide the criteria of how the `state` query param should be constructed,\nincluding length and complexity.                ",
+        examples=[{"state": {"min": 7, "max": 128}}],
+        title="(Optional) DeclarativeOAuth Configurable State Query Param",
+    )
+    client_id_key: Optional[str] = Field(
+        None,
+        description="The DeclarativeOAuth Specific optional override to provide the custom `client_id` key name, if required by data-provider.",
+        examples=[{"client_id_key": "my_custom_client_id_key_name"}],
+        title="(Optional) DeclarativeOAuth Client ID Key Override",
+    )
+    client_secret_key: Optional[str] = Field(
+        None,
+        description="The DeclarativeOAuth Specific optional override to provide the custom `client_secret` key name, if required by data-provider.",
+        examples=[{"client_secret_key": "my_custom_client_secret_key_name"}],
+        title="(Optional) DeclarativeOAuth Client Secret Key Override",
+    )
+    scope_key: Optional[str] = Field(
+        None,
+        description="The DeclarativeOAuth Specific optional override to provide the custom `scope` key name, if required by data-provider.",
+        examples=[{"scope_key": "my_custom_scope_key_key_name"}],
+        title="(Optional) DeclarativeOAuth Scope Key Override",
+    )
+    state_key: Optional[str] = Field(
+        None,
+        description="The DeclarativeOAuth Specific optional override to provide the custom `state` key name, if required by data-provider.                ",
+        examples=[{"state_key": "my_custom_state_key_key_name"}],
+        title="(Optional) DeclarativeOAuth State Key Override",
+    )
+    auth_code_key: Optional[str] = Field(
+        None,
+        description="The DeclarativeOAuth Specific optional override to provide the custom `code` key name to something like `auth_code` or `custom_auth_code`, if required by data-provider.                ",
+        examples=[{"auth_code_key": "my_custom_auth_code_key_name"}],
+        title="(Optional) DeclarativeOAuth Auth Code Key Override",
+    )
+    redirect_uri_key: Optional[str] = Field(
+        None,
+        description="The DeclarativeOAuth Specific optional override to provide the custom `redirect_uri` key name to something like `callback_uri`, if required by data-provider.",
+        examples=[{"redirect_uri_key": "my_custom_redirect_uri_key_name"}],
+        title="(Optional) DeclarativeOAuth Redirect URI Key Override",
+    )
+
+
 class OAuthConfigSpecification(BaseModel):
     class Config:
         extra = Extra.allow
@@ -766,6 +888,11 @@ class OAuthConfigSpecification(BaseModel):
         ],
         title="OAuth user input",
     )
+    oauth_connector_input_specification: Optional[OauthConnectorInputSpecification] = Field(
+        None,
+        description='The DeclarativeOAuth specific blob.\nPertains to the fields defined by the connector relating to the OAuth flow.\n\nInterpolation capabilities:\n- The variables placeholders are declared as `{my_var}`.\n- The nested resolution variables like `{{my_nested_var}}` is allowed as well.\n\n- The allowed interpolation context is:\n  + base64Encoder - encode to `base64`, {base64Encoder:{my_var_a}:{my_var_b}}\n  + base64Decorer - decode from `base64` encoded string, {base64Decoder:{my_string_variable_or_string_value}}\n  + urlEncoder - encode the input string to URL-like format, {urlEncoder:https://test.host.com/endpoint}\n  + urlDecorer - decode the input url-encoded string into text format, {urlDecoder:https%3A%2F%2Fairbyte.io}\n  + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {codeChallengeS256:{state_value}}\n\nExamples:\n  - The TikTok Marketing DeclarativeOAuth spec:\n  {\n    "oauth_connector_input_specification": {\n      "type": "object",\n      "additionalProperties": false,\n      "properties": {\n          "consent_url": "https://ads.tiktok.com/marketing_api/auth?{client_id_key}={{client_id_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}&{state_key}={{state_key}}",\n          "access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",\n          "access_token_params": {\n              "{auth_code_key}": "{{auth_code_key}}",\n              "{client_id_key}": "{{client_id_key}}",\n              "{client_secret_key}": "{{client_secret_key}}"\n          },\n          "access_token_headers": {\n              "Content-Type": "application/json",\n              "Accept": "application/json"\n          },\n          "extract_output": ["data.access_token"],\n          "client_id_key": "app_id",\n          "client_secret_key": "secret",\n          "auth_code_key": "auth_code"\n      }\n    }\n  }',
+        title="DeclarativeOAuth Connector Specification",
+    )
     complete_oauth_output_specification: Optional[Dict[str, Any]] = Field(
         None,
         description="OAuth specific blob. This is a Json Schema used to validate Json configurations produced by the OAuth flows as they are\nreturned by the distant OAuth APIs.\nMust be a valid JSON describing the fields to merge back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n    complete_oauth_output_specification={\n      refresh_token: {\n        type: string,\n        path_in_connector_config: ['credentials', 'refresh_token']\n      }\n    }",
@@ -1676,6 +1803,9 @@ class AsyncRetriever(BaseModel):
         ...,
         description="Responsible for fetching the final result `urls` provided by the completed / finished / ready async job.",
     )
+    download_extractor: Optional[
+        Union[CustomRecordExtractor, DpathExtractor, ResponseToFileExtractor]
+    ] = Field(None, description="Responsible for fetching the records from provided urls.")
     creation_requester: Union[CustomRequester, HttpRequester] = Field(
         ...,
         description="Requester component that describes how to prepare HTTP requests to send to the source API to create the async server-side job.",
@@ -1726,6 +1856,20 @@ class AsyncRetriever(BaseModel):
         description="Component decoding the response so records can be extracted.",
         title="Decoder",
     )
+    download_decoder: Optional[
+        Union[
+            CustomDecoder,
+            JsonDecoder,
+            JsonlDecoder,
+            IterableDecoder,
+            XmlDecoder,
+            GzipJsonDecoder,
+        ]
+    ] = Field(
+        None,
+        description="Component decoding the download response so records can be extracted.",
+        title="Download Decoder",
+    )
     parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
 
 

airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py

@@ -25,6 +25,9 @@ from typing import (
     get_type_hints,
 )
 
+from isodate import parse_duration
+from pydantic.v1 import BaseModel
+
 from airbyte_cdk.models import FailureType, Level
 from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager
 from airbyte_cdk.sources.declarative.async_job.job_orchestrator import AsyncJobOrchestrator
@@ -267,6 +270,9 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     RequestPath as RequestPathModel,
 )
+from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
+    ResponseToFileExtractor as ResponseToFileExtractorModel,
+)
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     SelectiveAuthenticator as SelectiveAuthenticatorModel,
 )
@@ -367,8 +373,6 @@ from airbyte_cdk.sources.streams.concurrent.state_converters.datetime_stream_sta
 from airbyte_cdk.sources.streams.http.error_handlers.response_models import ResponseAction
 from airbyte_cdk.sources.types import Config
 from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer
-from isodate import parse_duration
-from pydantic.v1 import BaseModel
 
 ComponentDefinition = Mapping[str, Any]
 
@@ -426,6 +430,7 @@ class ModelToComponentFactory:
             DefaultErrorHandlerModel: self.create_default_error_handler,
             DefaultPaginatorModel: self.create_default_paginator,
             DpathExtractorModel: self.create_dpath_extractor,
+            ResponseToFileExtractorModel: self.create_response_to_file_extractor,
             ExponentialBackoffStrategyModel: self.create_exponential_backoff_strategy,
             SessionTokenAuthenticatorModel: self.create_session_token_authenticator,
             HttpRequesterModel: self.create_http_requester,
@@ -1446,6 +1451,13 @@ class ModelToComponentFactory:
             parameters=model.parameters or {},
         )
 
+    def create_response_to_file_extractor(
+        self,
+        model: ResponseToFileExtractorModel,
+        **kwargs: Any,
+    ) -> ResponseToFileExtractor:
+        return ResponseToFileExtractor(parameters=model.parameters or {})
+
     @staticmethod
     def create_exponential_backoff_strategy(
         model: ExponentialBackoffStrategyModel, config: Config
@@ -1780,6 +1792,7 @@ class ModelToComponentFactory:
         self,
         model: RecordSelectorModel,
         config: Config,
+        name: str,
         *,
         transformations: List[RecordTransformation],
         decoder: Optional[Decoder] = None,
@@ -1810,6 +1823,7 @@ class ModelToComponentFactory:
 
         return RecordSelector(
             extractor=extractor,
+            name=name,
             config=config,
             record_filter=record_filter,
             transformations=transformations,
@@ -1880,6 +1894,7 @@ class ModelToComponentFactory:
         )
         record_selector = self._create_component_from_model(
             model=model.record_selector,
+            name=name,
             config=config,
             decoder=decoder,
             transformations=transformations,
@@ -2007,6 +2022,7 @@ class ModelToComponentFactory:
             model=model.record_selector,
             config=config,
             decoder=decoder,
+            name=name,
             transformations=transformations,
             client_side_incremental_sync=client_side_incremental_sync,
         )
@@ -2024,16 +2040,37 @@ class ModelToComponentFactory:
             name=f"job polling - {name}",
         )
         job_download_components_name = f"job download - {name}"
+        download_decoder = (
+            self._create_component_from_model(model=model.download_decoder, config=config)
+            if model.download_decoder
+            else JsonDecoder(parameters={})
+        )
+        download_extractor = (
+            self._create_component_from_model(
+                model=model.download_extractor,
+                config=config,
+                decoder=download_decoder,
+                parameters=model.parameters,
+            )
+            if model.download_extractor
+            else DpathExtractor(
+                [],
+                config=config,
+                decoder=download_decoder,
+                parameters=model.parameters or {},
+            )
+        )
         download_requester = self._create_component_from_model(
             model=model.download_requester,
-            decoder=decoder,
+            decoder=download_decoder,
             config=config,
             name=job_download_components_name,
         )
         download_retriever = SimpleRetriever(
             requester=download_requester,
             record_selector=RecordSelector(
-                extractor=ResponseToFileExtractor(),
+                extractor=download_extractor,
+                name=name,
                 record_filter=None,
                 transformations=[],
                 schema_normalization=TypeTransformer(TransformConfig.NoTransform),

airbyte_cdk/sources/declarative/partition_routers/substream_partition_router.py

@@ -7,6 +7,7 @@ from dataclasses import InitVar, dataclass
 from typing import TYPE_CHECKING, Any, Iterable, List, Mapping, Optional, Union
 
 import dpath
+
 from airbyte_cdk.models import AirbyteMessage
 from airbyte_cdk.models import Type as MessageType
 from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString

airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/constant_backoff_strategy.py

@@ -6,6 +6,7 @@ from dataclasses import InitVar, dataclass
 from typing import Any, Mapping, Optional, Union
 
 import requests
+
 from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
 from airbyte_cdk.sources.streams.http.error_handlers import BackoffStrategy
 from airbyte_cdk.sources.types import Config

airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/exponential_backoff_strategy.py

@@ -6,6 +6,7 @@ from dataclasses import InitVar, dataclass
 from typing import Any, Mapping, Optional, Union
 
 import requests
+
 from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
 from airbyte_cdk.sources.streams.http.error_handlers import BackoffStrategy
 from airbyte_cdk.sources.types import Config

airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_time_from_header_backoff_strategy.py

@@ -7,6 +7,7 @@ from dataclasses import InitVar, dataclass
 from typing import Any, Mapping, Optional, Union
 
 import requests
+
 from airbyte_cdk.models import FailureType
 from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
 from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies.header_helper import (

airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_until_time_from_header_backoff_strategy.py

@@ -9,6 +9,7 @@ from dataclasses import InitVar, dataclass
 from typing import Any, Mapping, Optional, Union
 
 import requests
+
 from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
 from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies.header_helper import (
     get_numeric_value_from_header,

airbyte_cdk/sources/declarative/requesters/error_handlers/composite_error_handler.py

@@ -6,6 +6,7 @@ from dataclasses import InitVar, dataclass
 from typing import Any, List, Mapping, Optional, Union
 
 import requests
+
 from airbyte_cdk.sources.streams.http.error_handlers import ErrorHandler
 from airbyte_cdk.sources.streams.http.error_handlers.response_models import (
     ErrorResolution,