airbyte-cdk 6.13.1.dev4109__py3-none-any.whl → 6.14.0__py3-none-any.whl

airbyte_cdk/sources/declarative/models/declarative_component_schema.py

@@ -268,22 +268,6 @@ class CustomSchemaLoader(BaseModel):
     parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
 
 
-class CustomSchemaNormalization(BaseModel):
-    class Config:
-        extra = Extra.allow
-
-    type: Literal["CustomSchemaNormalization"]
-    class_name: str = Field(
-        ...,
-        description="Fully-qualified name of the class that will be implementing the custom normalization. The format is `source_<name>.<package>.<class_name>`.",
-        examples=[
-            "source_amazon_seller_partner.components.LedgerDetailedViewReportsTypeTransformer"
-        ],
-        title="Class Name",
-    )
-    parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
-
-
 class CustomStateMigration(BaseModel):
     class Config:
         extra = Extra.allow
@@ -481,24 +465,12 @@ class RefreshTokenUpdater(BaseModel):
 
 class OAuthAuthenticator(BaseModel):
     type: Literal["OAuthAuthenticator"]
-    client_id_name: Optional[str] = Field(
-        "client_id",
-        description="The name of the property to use to refresh the `access_token`.",
-        examples=["custom_app_id"],
-        title="Client ID Property Name",
-    )
     client_id: str = Field(
         ...,
         description="The OAuth client ID. Fill it in the user inputs.",
         examples=["{{ config['client_id }}", "{{ config['credentials']['client_id }}"],
         title="Client ID",
     )
-    client_secret_name: Optional[str] = Field(
-        "client_secret",
-        description="The name of the property to use to refresh the `access_token`.",
-        examples=["custom_app_secret"],
-        title="Client Secret Property Name",
-    )
     client_secret: str = Field(
         ...,
         description="The OAuth client secret. Fill it in the user inputs.",
@@ -508,12 +480,6 @@ class OAuthAuthenticator(BaseModel):
         ],
         title="Client Secret",
     )
-    refresh_token_name: Optional[str] = Field(
-        "refresh_token",
-        description="The name of the property to use to refresh the `access_token`.",
-        examples=["custom_app_refresh_value"],
-        title="Refresh Token Property Name",
-    )
     refresh_token: Optional[str] = Field(
         None,
         description="Credential artifact used to get a new access token.",
@@ -547,12 +513,6 @@ class OAuthAuthenticator(BaseModel):
         examples=["expires_in"],
         title="Token Expiry Property Name",
     )
-    grant_type_name: Optional[str] = Field(
-        "grant_type",
-        description="The name of the property to use to refresh the `access_token`.",
-        examples=["custom_grant_type"],
-        title="Grant Type Property Name",
-    )
     grant_type: Optional[str] = Field(
         "refresh_token",
         description="Specifies the OAuth2 grant type. If set to refresh_token, the refresh_token needs to be provided as well. For client_credentials, only client id and secret are required. Other grant types are not officially supported.",
@@ -763,38 +723,6 @@ class KeysToSnakeCase(BaseModel):
 
 class FlattenFields(BaseModel):
     type: Literal["FlattenFields"]
-    flatten_lists: Optional[bool] = Field(
-        True,
-        description="Whether to flatten lists or leave it as is. Default is True.",
-        title="Flatten Lists",
-    )
-    parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
-
-
-class KeysReplace(BaseModel):
-    type: Literal["KeysReplace"]
-    old: str = Field(
-        ...,
-        description="Old value to replace.",
-        examples=[
-            " ",
-            "{{ record.id }}",
-            "{{ config['id'] }}",
-            "{{ stream_slice['id'] }}",
-        ],
-        title="Old value",
-    )
-    new: str = Field(
-        ...,
-        description="New value to set.",
-        examples=[
-            "_",
-            "{{ record.id }}",
-            "{{ config['id'] }}",
-            "{{ stream_slice['id'] }}",
-        ],
-        title="New value",
-    )
     parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
 
 
@@ -883,8 +811,8 @@ class OauthConnectorInputSpecification(BaseModel):
         ...,
         description="The DeclarativeOAuth Specific string URL string template to initiate the authentication.\nThe placeholders are replaced during the processing to provide neccessary values.",
         examples=[
-            "https://domain.host.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}",
-            "https://endpoint.host.com/oauth2/authorize?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{scope_key}}={{{{scope_value}} | urlEncoder}}&{{state_key}}={{state_value}}&subdomain={{subdomain}}",
+            "https://domain.host.com/marketing_api/auth?{client_id_key}={{client_id_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}&{state_key}={{state_key}}",
+            "https://endpoint.host.com/oauth2/authorize?{client_id_key}={{client_id_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}&{scope_key}={urlEncoder:{{scope_key}}}&{state_key}={{state_key}}&subdomain={subdomain}",
         ],
         title="Consent URL",
     )
@@ -898,18 +826,14 @@ class OauthConnectorInputSpecification(BaseModel):
         ...,
         description="The DeclarativeOAuth Specific URL templated string to obtain the `access_token`, `refresh_token` etc.\nThe placeholders are replaced during the processing to provide neccessary values.",
         examples=[
-            "https://auth.host.com/oauth2/token?{{client_id_key}}={{client_id_value}}&{{client_secret_key}}={{client_secret_value}}&{{auth_code_key}}={{auth_code_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}"
+            "https://auth.host.com/oauth2/token?{client_id_key}={{client_id_key}}&{client_secret_key}={{client_secret_key}}&{auth_code_key}={{auth_code_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}"
         ],
         title="Access Token URL",
     )
     access_token_headers: Optional[Dict[str, Any]] = Field(
         None,
         description="The DeclarativeOAuth Specific optional headers to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.",
-        examples=[
-            {
-                "Authorization": "Basic {{ {{ client_id_value }}:{{ client_secret_value }} | base64Encoder }}"
-            }
-        ],
+        examples=[{"Authorization": "Basic {base64Encoder:{client_id}:{client_secret}}"}],
         title="Access Token Headers",
     )
     access_token_params: Optional[Dict[str, Any]] = Field(
@@ -917,15 +841,15 @@ class OauthConnectorInputSpecification(BaseModel):
         description="The DeclarativeOAuth Specific optional query parameters to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.\nWhen this property is provided, the query params will be encoded as `Json` and included in the outgoing API request.",
         examples=[
             {
-                "{{ auth_code_key }}": "{{ auth_code_value }}",
-                "{{ client_id_key }}": "{{ client_id_value }}",
-                "{{ client_secret_key }}": "{{ client_secret_value }}",
+                "{auth_code_key}": "{{auth_code_key}}",
+                "{client_id_key}": "{{client_id_key}}",
+                "{client_secret_key}": "{{client_secret_key}}",
             }
         ],
         title="Access Token Query Params (Json Encoded)",
     )
-    extract_output: Optional[List[str]] = Field(
-        None,
+    extract_output: List[str] = Field(
+        ...,
         description="The DeclarativeOAuth Specific list of strings to indicate which keys should be extracted and returned back to the input config.",
         examples=[["access_token", "refresh_token", "other_field"]],
         title="Extract Output",
@@ -994,7 +918,7 @@ class OAuthConfigSpecification(BaseModel):
     )
     oauth_connector_input_specification: Optional[OauthConnectorInputSpecification] = Field(
         None,
-        description='The DeclarativeOAuth specific blob.\nPertains to the fields defined by the connector relating to the OAuth flow.\n\nInterpolation capabilities:\n- The variables placeholders are declared as `{{my_var}}`.\n- The nested resolution variables like `{{ {{my_nested_var}} }}` is allowed as well.\n\n- The allowed interpolation context is:\n  + base64Encoder - encode to `base64`, {{ {{my_var_a}}:{{my_var_b}} | base64Encoder }}\n  + base64Decorer - decode from `base64` encoded string, {{ {{my_string_variable_or_string_value}} | base64Decoder }}\n  + urlEncoder - encode the input string to URL-like format, {{ https://test.host.com/endpoint | urlEncoder}}\n  + urlDecorer - decode the input url-encoded string into text format, {{ urlDecoder:https%3A%2F%2Fairbyte.io | urlDecoder}}\n  + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {{ {{state_value}} | codeChallengeS256 }}\n\nExamples:\n  - The TikTok Marketing DeclarativeOAuth spec:\n  {\n    "oauth_connector_input_specification": {\n      "type": "object",\n      "additionalProperties": false,\n      "properties": {\n          "consent_url": "https://ads.tiktok.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{ {{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}",\n          "access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",\n          "access_token_params": {\n              "{{ auth_code_key }}": "{{ auth_code_value }}",\n              "{{ client_id_key }}": "{{ client_id_value }}",\n              "{{ client_secret_key }}": "{{ client_secret_value }}"\n          },\n          "access_token_headers": {\n              "Content-Type": "application/json",\n              "Accept": "application/json"\n          },\n          "extract_output": ["data.access_token"],\n          "client_id_key": "app_id",\n          "client_secret_key": "secret",\n          "auth_code_key": "auth_code"\n      }\n    }\n  }',
+        description='The DeclarativeOAuth specific blob.\nPertains to the fields defined by the connector relating to the OAuth flow.\n\nInterpolation capabilities:\n- The variables placeholders are declared as `{my_var}`.\n- The nested resolution variables like `{{my_nested_var}}` is allowed as well.\n\n- The allowed interpolation context is:\n  + base64Encoder - encode to `base64`, {base64Encoder:{my_var_a}:{my_var_b}}\n  + base64Decorer - decode from `base64` encoded string, {base64Decoder:{my_string_variable_or_string_value}}\n  + urlEncoder - encode the input string to URL-like format, {urlEncoder:https://test.host.com/endpoint}\n  + urlDecorer - decode the input url-encoded string into text format, {urlDecoder:https%3A%2F%2Fairbyte.io}\n  + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {codeChallengeS256:{state_value}}\n\nExamples:\n  - The TikTok Marketing DeclarativeOAuth spec:\n  {\n    "oauth_connector_input_specification": {\n      "type": "object",\n      "additionalProperties": false,\n      "properties": {\n          "consent_url": "https://ads.tiktok.com/marketing_api/auth?{client_id_key}={{client_id_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}&{state_key}={{state_key}}",\n          "access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",\n          "access_token_params": {\n              "{auth_code_key}": "{{auth_code_key}}",\n              "{client_id_key}": "{{client_id_key}}",\n              "{client_secret_key}": "{{client_secret_key}}"\n          },\n          "access_token_headers": {\n              "Content-Type": "application/json",\n              "Accept": "application/json"\n          },\n          "extract_output": ["data.access_token"],\n          "client_id_key": "app_id",\n          "client_secret_key": "secret",\n          "auth_code_key": "auth_code"\n      }\n    }\n  }',
         title="DeclarativeOAuth Connector Specification",
     )
     complete_oauth_output_specification: Optional[Dict[str, Any]] = Field(
@@ -1589,11 +1513,7 @@ class RecordSelector(BaseModel):
         description="Responsible for filtering records to be emitted by the Source.",
         title="Record Filter",
     )
-    schema_normalization: Optional[Union[SchemaNormalization, CustomSchemaNormalization]] = Field(
-        SchemaNormalization.None_,
-        description="Responsible for normalization according to the schema.",
-        title="Schema Normalization",
-    )
+    schema_normalization: Optional[SchemaNormalization] = SchemaNormalization.None_
     parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
 
 
@@ -1781,7 +1701,6 @@ class DeclarativeStream(BaseModel):
                 KeysToLower,
                 KeysToSnakeCase,
                 FlattenFields,
-                KeysReplace,
             ]
         ]
     ] = Field(
@@ -1956,7 +1875,6 @@ class DynamicSchemaLoader(BaseModel):
                 KeysToLower,
                 KeysToSnakeCase,
                 FlattenFields,
-                KeysReplace,
             ]
         ]
     ] = Field(
@@ -2078,10 +1996,6 @@ class AsyncRetriever(BaseModel):
         ...,
         description="Requester component that describes how to prepare HTTP requests to send to the source API to fetch the status of the running async job.",
     )
-    url_requester: Optional[Union[CustomRequester, HttpRequester]] = Field(
-        None,
-        description="Requester component that describes how to prepare HTTP requests to send to the source API to extract the url from polling response by the completed async job.",
-    )
     download_requester: Union[CustomRequester, HttpRequester] = Field(
         ...,
         description="Requester component that describes how to prepare HTTP requests to send to the source API to download the data provided by the completed async job.",

airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py

@@ -82,6 +82,9 @@ from airbyte_cdk.sources.declarative.extractors import (
 from airbyte_cdk.sources.declarative.extractors.record_filter import (
     ClientSideIncrementalRecordFilterDecorator,
 )
+from airbyte_cdk.sources.declarative.extractors.record_selector import (
+    SCHEMA_TRANSFORMER_TYPE_MAPPING,
+)
 from airbyte_cdk.sources.declarative.incremental import (
     ChildPartitionResumableFullRefreshCursor,
     CursorFactory,
@@ -97,9 +100,7 @@ from airbyte_cdk.sources.declarative.interpolation.interpolated_mapping import I
 from airbyte_cdk.sources.declarative.migrations.legacy_to_per_partition_state_migration import (
     LegacyToPerPartitionStateMigration,
 )
-from airbyte_cdk.sources.declarative.models import (
-    CustomStateMigration,
-)
+from airbyte_cdk.sources.declarative.models import CustomStateMigration
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     AddedFieldDefinition as AddedFieldDefinitionModel,
 )
@@ -184,9 +185,6 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     CustomSchemaLoader as CustomSchemaLoader,
 )
-from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
-    CustomSchemaNormalization as CustomSchemaNormalizationModel,
-)
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     CustomTransformation as CustomTransformationModel,
 )
@@ -256,9 +254,6 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     JwtPayload as JwtPayloadModel,
 )
-from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
-    KeysReplace as KeysReplaceModel,
-)
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     KeysToLower as KeysToLowerModel,
 )
@@ -313,9 +308,6 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     ResponseToFileExtractor as ResponseToFileExtractorModel,
 )
-from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
-    SchemaNormalization as SchemaNormalizationModel,
-)
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     SchemaTypeIdentifier as SchemaTypeIdentifierModel,
 )
@@ -425,9 +417,6 @@ from airbyte_cdk.sources.declarative.transformations.add_fields import AddedFiel
 from airbyte_cdk.sources.declarative.transformations.flatten_fields import (
     FlattenFields,
 )
-from airbyte_cdk.sources.declarative.transformations.keys_replace_transformation import (
-    KeysReplaceTransformation,
-)
 from airbyte_cdk.sources.declarative.transformations.keys_to_lower_transformation import (
     KeysToLowerTransformation,
 )
@@ -450,11 +439,6 @@ from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer
 
 ComponentDefinition = Mapping[str, Any]
 
-SCHEMA_TRANSFORMER_TYPE_MAPPING = {
-    SchemaNormalizationModel.None_: TransformConfig.NoTransform,
-    SchemaNormalizationModel.Default: TransformConfig.DefaultSchemaNormalization,
-}
-
 
 class ModelToComponentFactory:
     EPOCH_DATETIME_FORMAT = "%s"
@@ -503,7 +487,6 @@ class ModelToComponentFactory:
             CustomRequesterModel: self.create_custom_component,
             CustomRetrieverModel: self.create_custom_component,
             CustomSchemaLoader: self.create_custom_component,
-            CustomSchemaNormalizationModel: self.create_custom_component,
             CustomStateMigration: self.create_custom_component,
             CustomPaginationStrategyModel: self.create_custom_component,
             CustomPartitionRouterModel: self.create_custom_component,
@@ -526,7 +509,6 @@ class ModelToComponentFactory:
             GzipParserModel: self.create_gzip_parser,
             KeysToLowerModel: self.create_keys_to_lower_transformation,
             KeysToSnakeCaseModel: self.create_keys_to_snake_transformation,
-            KeysReplaceModel: self.create_keys_replace_transformation,
             FlattenFieldsModel: self.create_flatten_fields,
             IterableDecoderModel: self.create_iterable_decoder,
             XmlDecoderModel: self.create_xml_decoder,
@@ -648,19 +630,10 @@ class ModelToComponentFactory:
     ) -> KeysToSnakeCaseTransformation:
         return KeysToSnakeCaseTransformation()
 
-    def create_keys_replace_transformation(
-        self, model: KeysReplaceModel, config: Config, **kwargs: Any
-    ) -> KeysReplaceTransformation:
-        return KeysReplaceTransformation(
-            old=model.old, new=model.new, parameters=model.parameters or {}
-        )
-
     def create_flatten_fields(
         self, model: FlattenFieldsModel, config: Config, **kwargs: Any
     ) -> FlattenFields:
-        return FlattenFields(
-            flatten_lists=model.flatten_lists if model.flatten_lists is not None else True
-        )
+        return FlattenFields()
 
     @staticmethod
     def _json_schema_type_name_to_type(value_type: Optional[ValueType]) -> Optional[Type[Any]]:
@@ -1587,12 +1560,7 @@ class ModelToComponentFactory:
         )
 
     def create_http_requester(
-        self,
-        model: HttpRequesterModel,
-        config: Config,
-        decoder: Decoder = JsonDecoder(parameters={}),
-        *,
-        name: str,
+        self, model: HttpRequesterModel, decoder: Decoder, config: Config, *, name: str
     ) -> HttpRequester:
         authenticator = (
             self._create_component_from_model(
@@ -1885,24 +1853,15 @@ class ModelToComponentFactory:
                 expires_in_name=InterpolatedString.create(
                     model.expires_in_name or "expires_in", parameters=model.parameters or {}
                 ).eval(config),
-                client_id_name=InterpolatedString.create(
-                    model.client_id_name or "client_id", parameters=model.parameters or {}
-                ).eval(config),
                 client_id=InterpolatedString.create(
                     model.client_id, parameters=model.parameters or {}
                 ).eval(config),
-                client_secret_name=InterpolatedString.create(
-                    model.client_secret_name or "client_secret", parameters=model.parameters or {}
-                ).eval(config),
                 client_secret=InterpolatedString.create(
                     model.client_secret, parameters=model.parameters or {}
                 ).eval(config),
                 access_token_config_path=model.refresh_token_updater.access_token_config_path,
                 refresh_token_config_path=model.refresh_token_updater.refresh_token_config_path,
                 token_expiry_date_config_path=model.refresh_token_updater.token_expiry_date_config_path,
-                grant_type_name=InterpolatedString.create(
-                    model.grant_type_name or "grant_type", parameters=model.parameters or {}
-                ).eval(config),
                 grant_type=InterpolatedString.create(
                     model.grant_type or "refresh_token", parameters=model.parameters or {}
                 ).eval(config),
@@ -1920,15 +1879,11 @@ class ModelToComponentFactory:
         return DeclarativeOauth2Authenticator(  # type: ignore
             access_token_name=model.access_token_name or "access_token",
             access_token_value=model.access_token_value,
-            client_id_name=model.client_id_name or "client_id",
             client_id=model.client_id,
-            client_secret_name=model.client_secret_name or "client_secret",
             client_secret=model.client_secret,
             expires_in_name=model.expires_in_name or "expires_in",
-            grant_type_name=model.grant_type_name or "grant_type",
             grant_type=model.grant_type or "refresh_token",
             refresh_request_body=model.refresh_request_body,
-            refresh_token_name=model.refresh_token_name or "refresh_token",
             refresh_token=model.refresh_token,
             scopes=model.scopes,
             token_expiry_date=model.token_expiry_date,
@@ -2021,11 +1976,12 @@ class ModelToComponentFactory:
         config: Config,
         *,
         name: str,
-        transformations: List[RecordTransformation] | None = None,
-        decoder: Decoder | None = None,
-        client_side_incremental_sync: Dict[str, Any] | None = None,
+        transformations: List[RecordTransformation],
+        decoder: Optional[Decoder] = None,
+        client_side_incremental_sync: Optional[Dict[str, Any]] = None,
         **kwargs: Any,
     ) -> RecordSelector:
+        assert model.schema_normalization is not None  # for mypy
         extractor = self._create_component_from_model(
             model=model.extractor, decoder=decoder, config=config
         )
@@ -2043,10 +1999,8 @@ class ModelToComponentFactory:
                 else None,
                 **client_side_incremental_sync,
             )
-        schema_normalization = (
-            TypeTransformer(SCHEMA_TRANSFORMER_TYPE_MAPPING[model.schema_normalization])
-            if isinstance(model.schema_normalization, SchemaNormalizationModel)
-            else self._create_component_from_model(model.schema_normalization, config=config)  # type: ignore[arg-type] # custom normalization model expected here
+        schema_normalization = TypeTransformer(
+            SCHEMA_TRANSFORMER_TYPE_MAPPING[model.schema_normalization]
         )
 
         return RecordSelector(
@@ -2054,7 +2008,7 @@ class ModelToComponentFactory:
             name=name,
             config=config,
             record_filter=record_filter,
-            transformations=transformations or [],
+            transformations=transformations,
             schema_normalization=schema_normalization,
             parameters=model.parameters or {},
         )
@@ -2300,7 +2254,7 @@ class ModelToComponentFactory:
                 extractor=download_extractor,
                 name=name,
                 record_filter=None,
-                transformations=transformations,
+                transformations=[],
                 schema_normalization=TypeTransformer(TransformConfig.NoTransform),
                 config=config,
                 parameters={},
@@ -2337,16 +2291,6 @@ class ModelToComponentFactory:
             if model.delete_requester
             else None
         )
-        url_requester = (
-            self._create_component_from_model(
-                model=model.url_requester,
-                decoder=decoder,
-                config=config,
-                name=f"job extract_url - {name}",
-            )
-            if model.url_requester
-            else None
-        )
         status_extractor = self._create_component_from_model(
             model=model.status_extractor, decoder=decoder, config=config, name=name
         )
@@ -2357,7 +2301,6 @@ class ModelToComponentFactory:
             creation_requester=creation_requester,
             polling_requester=polling_requester,
             download_retriever=download_retriever,
-            url_requester=url_requester,
             abort_requester=abort_requester,
             delete_requester=delete_requester,
             status_extractor=status_extractor,

airbyte_cdk/sources/declarative/requesters/http_job_repository.py

@@ -31,10 +31,6 @@ LOGGER = logging.getLogger("airbyte")
 
 @dataclass
 class AsyncHttpJobRepository(AsyncJobRepository):
-    """
-    See Readme file for more details about flow.
-    """
-
     creation_requester: Requester
     polling_requester: Requester
     download_retriever: SimpleRetriever
@@ -48,9 +44,6 @@ class AsyncHttpJobRepository(AsyncJobRepository):
     record_extractor: RecordExtractor = field(
         init=False, repr=False, default_factory=lambda: ResponseToFileExtractor({})
     )
-    url_requester: Optional[Requester] = (
-        None  # use it in case polling_requester provides some <id> and extra request is needed to obtain list of urls to download from
-    )
 
     def __post_init__(self) -> None:
         self._create_job_response_by_id: Dict[str, Response] = {}
@@ -193,13 +186,10 @@ class AsyncHttpJobRepository(AsyncJobRepository):
 
         """
 
-        for url in self._get_download_url(job):
-            job_slice = job.job_parameters()
-            stream_slice = StreamSlice(
-                partition=job_slice.partition,
-                cursor_slice=job_slice.cursor_slice,
-                extra_fields={**job_slice.extra_fields, "url": url},
-            )
+        for url in self.urls_extractor.extract_records(
+            self._polling_job_response_by_id[job.api_job_id()]
+        ):
+            stream_slice: StreamSlice = StreamSlice(partition={"url": url}, cursor_slice={})
             for message in self.download_retriever.read_records({}, stream_slice):
                 if isinstance(message, Record):
                     yield message.data
@@ -236,22 +226,3 @@ class AsyncHttpJobRepository(AsyncJobRepository):
             cursor_slice={},
         )
         return stream_slice
-
-    def _get_download_url(self, job: AsyncJob) -> Iterable[str]:
-        if not self.url_requester:
-            url_response = self._polling_job_response_by_id[job.api_job_id()]
-        else:
-            stream_slice: StreamSlice = StreamSlice(
-                partition={
-                    "polling_job_response": self._polling_job_response_by_id[job.api_job_id()]
-                },
-                cursor_slice={},
-            )
-            url_response = self.url_requester.send_request(stream_slice=stream_slice)  # type: ignore # we expect url_requester to always be presented, otherwise raise an exception as we cannot proceed with the report
-            if not url_response:
-                raise AirbyteTracedException(
-                    internal_message="Always expect a response or an exception from url_requester",
-                    failure_type=FailureType.system_error,
-                )
-
-        yield from self.urls_extractor.extract_records(url_response)  # type: ignore # we expect urls_extractor to always return list of strings