airbyte-cdk 6.11.2.dev1__tar.gz → 6.12.0rc1__tar.gz

{airbyte_cdk-6.11.2.dev1 → airbyte_cdk-6.12.0rc1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 6.11.2.dev1
+Version: 6.12.0rc1
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://airbyte.com
 License: MIT

{airbyte_cdk-6.11.2.dev1 → airbyte_cdk-6.12.0rc1}/airbyte_cdk/sources/declarative/declarative_component_schema.yaml

@@ -1897,6 +1897,56 @@ definitions:
       $parameters:
         type: object
         additionalProperties: true
+  ZipfileDecoder:
+    title: Zipfile Decoder
+    description: Decoder for response data that is returned as zipfile(s).
+    type: object
+    additionalProperties: true
+    required:
+      - type
+    properties:
+      type:
+        type: string
+        enum: [ZipfileDecoder]
+      parser:
+        title: Parser
+        description: Parser to parse the decompressed data from the zipfile(s).
+        anyOf:
+          - "$ref": "#/definitions/JsonParser"
+          - "$ref": "#/definitions/CustomParser"
+  JsonParser:
+    title: JsonParser
+    description: Parser used for parsing str, bytes, or bytearray data and returning data in a dictionary format.
+    type: object
+    additionalProperties: true
+    required:
+      - type
+    properties:
+      type:
+        type: string
+        enum: [JsonParser]
+  CustomParser:
+    title: Custom Parser
+    description: Use this to implement custom parser logic.
+    type: object
+    additionalProperties: true
+    required:
+      - type
+      - class_name
+    properties:
+      type:
+        type: string
+        enum: [CustomParser]
+      class_name:
+        title: Class Name
+        description: Fully-qualified name of the class that will be implementing the custom decoding. Has to be a sub class of Parser. The format is `source_<name>.<package>.<class_name>`.
+        type: string
+        additionalProperties: true
+        examples:
+          - "source_rivendell.components.ElvishParser"
+      $parameters:
+        type: object
+        additionalProperties: true
   ListPartitionRouter:
     title: List Partition Router
     description: A Partition router that specifies a list of attributes where each attribute describes a portion of the complete data set for a stream. During a sync, each value is iterated over and can be used as input to outbound API requests.
@@ -2130,61 +2180,65 @@ definitions:
           - extract_output
         properties:
           consent_url:
-            title: Consent URL
+            title: DeclarativeOAuth Consent URL
             type: string
             description: |-
               The DeclarativeOAuth Specific string URL string template to initiate the authentication.
               The placeholders are replaced during the processing to provide neccessary values.
             examples:
-              - https://domain.host.com/marketing_api/auth?{client_id_key}={{client_id_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}&{state_key}={{state_key}}
-              - https://endpoint.host.com/oauth2/authorize?{client_id_key}={{client_id_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}&{scope_key}={urlEncoder:{{scope_key}}}&{state_key}={{state_key}}&subdomain={subdomain}
+              - consent_url: https://domain.host.com/marketing_api/auth?{client_id_key}={{client_id_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}&{state_key}={{state_key}}
+              - consent_url: https://endpoint.host.com/oauth2/authorize?{client_id_key}={{client_id_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}&{scope_key}={urlEncoder:{{scope_key}}}&{state_key}={{state_key}}&subdomain={subdomain}
           scope:
-            title: Scopes
+            title: (Optional) DeclarativeOAuth Scope
             type: string
             description: |-
               The DeclarativeOAuth Specific string of the scopes needed to be grant for authenticated user.
             examples:
-              - user:read user:read_orders workspaces:read
+              - scope: user:read user:read_orders workspaces:read
           access_token_url:
-            title: Access Token URL
+            title: DeclarativeOAuth Access Token URL
             type: string
             description: |-
               The DeclarativeOAuth Specific URL templated string to obtain the `access_token`, `refresh_token` etc.
               The placeholders are replaced during the processing to provide neccessary values.
             examples:
-              - https://auth.host.com/oauth2/token?{client_id_key}={{client_id_key}}&{client_secret_key}={{client_secret_key}}&{auth_code_key}={{auth_code_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}
+              - access_token_url: https://auth.host.com/oauth2/token?{client_id_key}={{client_id_key}}&{client_secret_key}={{client_secret_key}}&{auth_code_key}={{auth_code_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}
           access_token_headers:
-            title: Access Token Headers
+            title: (Optional) DeclarativeOAuth Access Token Headers
             type: object
             additionalProperties: true
             description: |-
               The DeclarativeOAuth Specific optional headers to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.
             examples:
-              - {"Authorization": "Basic {base64Encoder:{client_id}:{client_secret}}"}
+              - access_token_headers:
+                  {
+                    "Authorization": "Basic {base64Encoder:{client_id}:{client_secret}}",
+                  }
           access_token_params:
-            title: Access Token Query Params (Json Encoded)
+            title: (Optional) DeclarativeOAuth Access Token Query Params (Json Encoded)
             type: object
             additionalProperties: true
             description: |-
               The DeclarativeOAuth Specific optional query parameters to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.
               When this property is provided, the query params will be encoded as `Json` and included in the outgoing API request.
             examples:
-              - {
-                  "{auth_code_key}": "{{auth_code_key}}",
-                  "{client_id_key}": "{{client_id_key}}",
-                  "{client_secret_key}": "{{client_secret_key}}",
-                }
+              - access_token_params:
+                  {
+                    "{auth_code_key}": "{{auth_code_key}}",
+                    "{client_id_key}": "{{client_id_key}}",
+                    "{client_secret_key}": "{{client_secret_key}}",
+                  }
           extract_output:
-            title: Extract Output
+            title: DeclarativeOAuth Extract Output
             type: array
             items:
               type: string
             description: |-
               The DeclarativeOAuth Specific list of strings to indicate which keys should be extracted and returned back to the input config.
             examples:
-              - ["access_token", "refresh_token", "other_field"]
+              - extract_output: ["access_token", "refresh_token", "other_field"]
           state:
-            title: Configurable State Query Param
+            title: (Optional) DeclarativeOAuth Configurable State Query Param
             type: object
             additionalProperties: true
             required:
@@ -2199,49 +2253,49 @@ definitions:
               max:
                 type: integer
             examples:
-              - { "min": 7, "max": 128 }
+              - state: { "min": 7, "max": 128 }
           client_id_key:
-            title: Client ID Key Override
+            title: (Optional) DeclarativeOAuth Client ID Key Override
             type: string
             description: |-
               The DeclarativeOAuth Specific optional override to provide the custom `client_id` key name, if required by data-provider.
             examples:
-              - "my_custom_client_id_key_name"
+              - client_id_key: "my_custom_client_id_key_name"
           client_secret_key:
-            title: Client Secret Key Override
+            title: (Optional) DeclarativeOAuth Client Secret Key Override
             type: string
             description: |-
               The DeclarativeOAuth Specific optional override to provide the custom `client_secret` key name, if required by data-provider.
             examples:
-              - "my_custom_client_secret_key_name"
+              - client_secret_key: "my_custom_client_secret_key_name"
           scope_key:
-            title: Scopes Key Override
+            title: (Optional) DeclarativeOAuth Scope Key Override
             type: string
             description: |-
               The DeclarativeOAuth Specific optional override to provide the custom `scope` key name, if required by data-provider.
             examples:
-              - "my_custom_scope_key_key_name"
+              - scope_key: "my_custom_scope_key_key_name"
           state_key:
-            title: State Key Override
+            title: (Optional) DeclarativeOAuth State Key Override
             type: string
             description: |-
               The DeclarativeOAuth Specific optional override to provide the custom `state` key name, if required by data-provider.
             examples:
-              - "my_custom_state_key_key_name"
+              - state_key: "my_custom_state_key_key_name"
           auth_code_key:
-            title: Auth Code Key Override
+            title: (Optional) DeclarativeOAuth Auth Code Key Override
             type: string
             description: |-
               The DeclarativeOAuth Specific optional override to provide the custom `code` key name to something like `auth_code` or `custom_auth_code`, if required by data-provider.
             examples:
-              - "my_custom_auth_code_key_name"
+              - auth_code_key: "my_custom_auth_code_key_name"
           redirect_uri_key:
-            title: Redirect URI Key Override
+            title: (Optional) DeclarativeOAuth Redirect URI Key Override
             type: string
             description: |-
               The DeclarativeOAuth Specific optional override to provide the custom `redirect_uri` key name to something like `callback_uri`, if required by data-provider.
             examples:
-              - "my_custom_redirect_uri_key_name"
+              - redirect_uri_key: "my_custom_redirect_uri_key_name"
       complete_oauth_output_specification:
         title: "OAuth output specification"
         description: |-

{airbyte_cdk-6.11.2.dev1 → airbyte_cdk-6.12.0rc1}/airbyte_cdk/sources/declarative/decoders/__init__.py

@@ -7,5 +7,6 @@ from airbyte_cdk.sources.declarative.decoders.json_decoder import JsonDecoder, J
 from airbyte_cdk.sources.declarative.decoders.noop_decoder import NoopDecoder
 from airbyte_cdk.sources.declarative.decoders.pagination_decoder_decorator import PaginationDecoderDecorator
 from airbyte_cdk.sources.declarative.decoders.xml_decoder import XmlDecoder
+from airbyte_cdk.sources.declarative.decoders.zipfile_decoder import ZipfileDecoder
 
-__all__ = ["Decoder", "JsonDecoder", "JsonlDecoder", "IterableDecoder", "GzipJsonDecoder", "NoopDecoder", "PaginationDecoderDecorator", "XmlDecoder"]
+__all__ = ["Decoder", "JsonDecoder", "JsonlDecoder", "IterableDecoder", "GzipJsonDecoder", "NoopDecoder", "PaginationDecoderDecorator", "XmlDecoder", "ZipfileDecoder"]

airbyte_cdk-6.12.0rc1/airbyte_cdk/sources/declarative/decoders/parsers/__init__.py

@@ -0,0 +1,7 @@
+#
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+#
+
+from airbyte_cdk.sources.declarative.decoders.parsers.parsers import Parser, JsonParser
+
+__all__ = ["Parser", "JsonParser"]

airbyte_cdk-6.12.0rc1/airbyte_cdk/sources/declarative/decoders/parsers/parsers.py

@@ -0,0 +1,49 @@
+#
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+#
+
+import json
+import logging
+from abc import abstractmethod
+from dataclasses import InitVar, dataclass
+from typing import Any, Generator, Mapping, MutableMapping, Union
+
+logger = logging.getLogger("airbyte")
+
+
+@dataclass
+class Parser:
+    """
+    Parser strategy to convert str, bytes, or bytearray data into MutableMapping[str, Any].
+    """
+
+    @abstractmethod
+    def parse(
+        self, data: Union[str, bytes, bytearray]
+    ) -> Generator[MutableMapping[str, Any], None, None]:
+        pass
+
+
+@dataclass
+class JsonParser(Parser):
+    """
+    Parser strategy for converting JSON-structure str, bytes, or bytearray data into MutableMapping[str, Any].
+    """
+
+    parameters: InitVar[Mapping[str, Any]]
+
+    def parse(
+        self, data: Union[str, bytes, bytearray]
+    ) -> Generator[MutableMapping[str, Any], None, None]:
+        try:
+            body_json = json.loads(data)
+        except json.JSONDecodeError:
+            logger.warning(f"Data cannot be parsed into json: {data=}")
+            yield {}
+
+        if not isinstance(body_json, list):
+            body_json = [body_json]
+        if len(body_json) == 0:
+            yield {}
+        else:
+            yield from body_json

airbyte_cdk-6.12.0rc1/airbyte_cdk/sources/declarative/decoders/zipfile_decoder.py

@@ -0,0 +1,54 @@
+#
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+#
+
+import gzip
+import io
+import logging
+import zipfile
+from dataclasses import InitVar, dataclass
+from typing import Any, Generator, Mapping, MutableMapping, Optional
+
+import requests
+
+from airbyte_cdk.sources.declarative.decoders import Decoder
+from airbyte_cdk.sources.declarative.decoders.parsers import JsonParser, Parser
+
+logger = logging.getLogger("airbyte")
+
+
+@dataclass
+class ZipfileDecoder(Decoder):
+    parameters: InitVar[Mapping[str, Any]]
+    parser: Optional[Parser] = None
+
+    def __post_init__(self, parameters: Mapping[str, Any]) -> None:
+        self._parser = (
+            self.parser(parameters=parameters) if self.parser else JsonParser(parameters=parameters)
+        )
+
+    def is_stream_response(self) -> bool:
+        return False
+
+    def decode(
+        self, response: requests.Response
+    ) -> Generator[MutableMapping[str, Any], None, None]:
+        try:
+            zip_file = zipfile.ZipFile(io.BytesIO(response.content))
+        except zipfile.BadZipFile as e:
+            logger.exception(e)
+            logger.error(
+                f"Received an invalid zip file in response to URL: {response.request.url}. "
+                f"The size of the response body is: {len(response.content)}"
+            )
+            yield {}
+
+        for gzip_filename in zip_file.namelist():
+            with zip_file.open(gzip_filename) as file:
+                try:
+                    for data in gzip.open(file):
+                        yield from self._parser.parse(data)
+                except gzip.BadGzipFile as e:
+                    logger.exception(e)
+                    logger.error(f"Fail to read contents of zipped response: {e}")
+                    yield {}

{airbyte_cdk-6.11.2.dev1 → airbyte_cdk-6.12.0rc1}/airbyte_cdk/sources/declarative/models/declarative_component_schema.py

@@ -741,6 +741,27 @@ class GzipJsonDecoder(BaseModel):
     parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
 
 
+class JsonParser(BaseModel):
+    class Config:
+        extra = Extra.allow
+
+    type: Literal["JsonParser"]
+
+
+class CustomParser(BaseModel):
+    class Config:
+        extra = Extra.allow
+
+    type: Literal["CustomParser"]
+    class_name: str = Field(
+        ...,
+        description="Fully-qualified name of the class that will be implementing the custom decoding. Has to be a sub class of Parser. The format is `source_<name>.<package>.<class_name>`.",
+        examples=["source_rivendell.components.ElvishParser"],
+        title="Class Name",
+    )
+    parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
+
+
 class MinMaxDatetime(BaseModel):
     type: Literal["MinMaxDatetime"]
     datetime: str = Field(
@@ -795,90 +816,104 @@ class OauthConnectorInputSpecification(BaseModel):
         ...,
         description="The DeclarativeOAuth Specific string URL string template to initiate the authentication.\nThe placeholders are replaced during the processing to provide neccessary values.",
         examples=[
-            "https://domain.host.com/marketing_api/auth?{client_id_key}={{client_id_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}&{state_key}={{state_key}}",
-            "https://endpoint.host.com/oauth2/authorize?{client_id_key}={{client_id_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}&{scope_key}={urlEncoder:{{scope_key}}}&{state_key}={{state_key}}&subdomain={subdomain}",
+            {
+                "consent_url": "https://domain.host.com/marketing_api/auth?{client_id_key}={{client_id_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}&{state_key}={{state_key}}"
+            },
+            {
+                "consent_url": "https://endpoint.host.com/oauth2/authorize?{client_id_key}={{client_id_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}&{scope_key}={urlEncoder:{{scope_key}}}&{state_key}={{state_key}}&subdomain={subdomain}"
+            },
         ],
-        title="Consent URL",
+        title="DeclarativeOAuth Consent URL",
     )
     scope: Optional[str] = Field(
         None,
         description="The DeclarativeOAuth Specific string of the scopes needed to be grant for authenticated user.",
-        examples=["user:read user:read_orders workspaces:read"],
-        title="Scopes",
+        examples=[{"scope": "user:read user:read_orders workspaces:read"}],
+        title="(Optional) DeclarativeOAuth Scope",
     )
     access_token_url: str = Field(
         ...,
         description="The DeclarativeOAuth Specific URL templated string to obtain the `access_token`, `refresh_token` etc.\nThe placeholders are replaced during the processing to provide neccessary values.",
         examples=[
-            "https://auth.host.com/oauth2/token?{client_id_key}={{client_id_key}}&{client_secret_key}={{client_secret_key}}&{auth_code_key}={{auth_code_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}"
+            {
+                "access_token_url": "https://auth.host.com/oauth2/token?{client_id_key}={{client_id_key}}&{client_secret_key}={{client_secret_key}}&{auth_code_key}={{auth_code_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}"
+            }
         ],
-        title="Access Token URL",
+        title="DeclarativeOAuth Access Token URL",
     )
     access_token_headers: Optional[Dict[str, Any]] = Field(
         None,
         description="The DeclarativeOAuth Specific optional headers to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.",
-        examples=[{"Authorization": "Basic {base64Encoder:{client_id}:{client_secret}}"}],
-        title="Access Token Headers",
+        examples=[
+            {
+                "access_token_headers": {
+                    "Authorization": "Basic {base64Encoder:{client_id}:{client_secret}}"
+                }
+            }
+        ],
+        title="(Optional) DeclarativeOAuth Access Token Headers",
     )
     access_token_params: Optional[Dict[str, Any]] = Field(
         None,
         description="The DeclarativeOAuth Specific optional query parameters to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.\nWhen this property is provided, the query params will be encoded as `Json` and included in the outgoing API request.",
         examples=[
             {
-                "{auth_code_key}": "{{auth_code_key}}",
-                "{client_id_key}": "{{client_id_key}}",
-                "{client_secret_key}": "{{client_secret_key}}",
+                "access_token_params": {
+                    "{auth_code_key}": "{{auth_code_key}}",
+                    "{client_id_key}": "{{client_id_key}}",
+                    "{client_secret_key}": "{{client_secret_key}}",
+                }
             }
         ],
-        title="Access Token Query Params (Json Encoded)",
+        title="(Optional) DeclarativeOAuth Access Token Query Params (Json Encoded)",
     )
     extract_output: List[str] = Field(
         ...,
         description="The DeclarativeOAuth Specific list of strings to indicate which keys should be extracted and returned back to the input config.",
-        examples=[["access_token", "refresh_token", "other_field"]],
-        title="Extract Output",
+        examples=[{"extract_output": ["access_token", "refresh_token", "other_field"]}],
+        title="DeclarativeOAuth Extract Output",
     )
     state: Optional[State] = Field(
         None,
         description="The DeclarativeOAuth Specific object to provide the criteria of how the `state` query param should be constructed,\nincluding length and complexity.",
-        examples=[{"min": 7, "max": 128}],
-        title="Configurable State Query Param",
+        examples=[{"state": {"min": 7, "max": 128}}],
+        title="(Optional) DeclarativeOAuth Configurable State Query Param",
     )
     client_id_key: Optional[str] = Field(
         None,
         description="The DeclarativeOAuth Specific optional override to provide the custom `client_id` key name, if required by data-provider.",
-        examples=["my_custom_client_id_key_name"],
-        title="Client ID Key Override",
+        examples=[{"client_id_key": "my_custom_client_id_key_name"}],
+        title="(Optional) DeclarativeOAuth Client ID Key Override",
     )
     client_secret_key: Optional[str] = Field(
         None,
         description="The DeclarativeOAuth Specific optional override to provide the custom `client_secret` key name, if required by data-provider.",
-        examples=["my_custom_client_secret_key_name"],
-        title="Client Secret Key Override",
+        examples=[{"client_secret_key": "my_custom_client_secret_key_name"}],
+        title="(Optional) DeclarativeOAuth Client Secret Key Override",
     )
     scope_key: Optional[str] = Field(
         None,
         description="The DeclarativeOAuth Specific optional override to provide the custom `scope` key name, if required by data-provider.",
-        examples=["my_custom_scope_key_key_name"],
-        title="Scopes Key Override",
+        examples=[{"scope_key": "my_custom_scope_key_key_name"}],
+        title="(Optional) DeclarativeOAuth Scope Key Override",
     )
     state_key: Optional[str] = Field(
         None,
         description="The DeclarativeOAuth Specific optional override to provide the custom `state` key name, if required by data-provider.",
-        examples=["my_custom_state_key_key_name"],
-        title="State Key Override",
+        examples=[{"state_key": "my_custom_state_key_key_name"}],
+        title="(Optional) DeclarativeOAuth State Key Override",
     )
     auth_code_key: Optional[str] = Field(
         None,
         description="The DeclarativeOAuth Specific optional override to provide the custom `code` key name to something like `auth_code` or `custom_auth_code`, if required by data-provider.",
-        examples=["my_custom_auth_code_key_name"],
-        title="Auth Code Key Override",
+        examples=[{"auth_code_key": "my_custom_auth_code_key_name"}],
+        title="(Optional) DeclarativeOAuth Auth Code Key Override",
     )
     redirect_uri_key: Optional[str] = Field(
         None,
         description="The DeclarativeOAuth Specific optional override to provide the custom `redirect_uri` key name to something like `callback_uri`, if required by data-provider.",
-        examples=["my_custom_redirect_uri_key_name"],
-        title="Redirect URI Key Override",
+        examples=[{"redirect_uri_key": "my_custom_redirect_uri_key_name"}],
+        title="(Optional) DeclarativeOAuth Redirect URI Key Override",
     )
 
 
@@ -1454,6 +1489,18 @@ class SessionTokenRequestApiKeyAuthenticator(BaseModel):
     )
 
 
+class ZipfileDecoder(BaseModel):
+    class Config:
+        extra = Extra.allow
+
+    type: Literal["ZipfileDecoder"]
+    parser: Optional[Union[JsonParser, CustomParser]] = Field(
+        None,
+        description="Parser to parse the decompressed data from the zipfile(s).",
+        title="Parser",
+    )
+
+
 class ListPartitionRouter(BaseModel):
     type: Literal["ListPartitionRouter"]
     cursor_field: str = Field(

{airbyte_cdk-6.11.2.dev1 → airbyte_cdk-6.12.0rc1}/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py

@@ -66,7 +66,9 @@ from airbyte_cdk.sources.declarative.decoders import (
     JsonlDecoder,
     PaginationDecoderDecorator,
     XmlDecoder,
+    ZipfileDecoder,
 )
+from airbyte_cdk.sources.declarative.decoders.parsers import JsonParser
 from airbyte_cdk.sources.declarative.extractors import (
     DpathExtractor,
     RecordFilter,
@@ -224,6 +226,9 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     JsonlDecoder as JsonlDecoderModel,
 )
+from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
+    JsonParser as JsonParserModel,
+)
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     JwtAuthenticator as JwtAuthenticatorModel,
 )
@@ -316,6 +321,9 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     XmlDecoder as XmlDecoderModel,
 )
+from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
+    ZipfileDecoder as ZipfileDecoderModel,
+)
 from airbyte_cdk.sources.declarative.partition_routers import (
     CartesianProductStreamSlicer,
     ListPartitionRouter,
@@ -323,9 +331,6 @@ from airbyte_cdk.sources.declarative.partition_routers import (
     SinglePartitionRouter,
     SubstreamPartitionRouter,
 )
-from airbyte_cdk.sources.declarative.partition_routers.async_job_partition_router import (
-    AsyncJobPartitionRouter,
-)
 from airbyte_cdk.sources.declarative.partition_routers.substream_partition_router import (
     ParentStreamConfig,
 )
@@ -473,6 +478,7 @@ class ModelToComponentFactory:
             InlineSchemaLoaderModel: self.create_inline_schema_loader,
             JsonDecoderModel: self.create_json_decoder,
             JsonlDecoderModel: self.create_jsonl_decoder,
+            JsonParserModel: self.create_json_parser,
             GzipJsonDecoderModel: self.create_gzipjson_decoder,
             KeysToLowerModel: self.create_keys_to_lower_transformation,
             IterableDecoderModel: self.create_iterable_decoder,
@@ -508,6 +514,7 @@ class ModelToComponentFactory:
             ConfigComponentsResolverModel: self.create_config_components_resolver,
             StreamConfigModel: self.create_stream_config,
             ComponentMappingDefinitionModel: self.create_components_mapping_definition,
+            ZipfileDecoderModel: self.create_zipfile_decoder,
         }
 
         # Needed for the case where we need to perform a second parse on the fields of a custom component
@@ -1685,6 +1692,20 @@ class ModelToComponentFactory:
     ) -> GzipJsonDecoder:
         return GzipJsonDecoder(parameters={}, encoding=model.encoding)
 
+    def create_zipfile_decoder(
+        self, model: ZipfileDecoderModel, config: Config, **kwargs: Any
+    ) -> ZipfileDecoder:
+        parser = (
+            self._create_component_from_model(model=model.parser, config=config)
+            if model.parser
+            else None
+        )
+        return ZipfileDecoder(parameters={}, parser=parser)
+
+    @staticmethod
+    def create_json_parser(model: JsonParserModel, config: Config, **kwargs: Any) -> JsonParser:
+        return JsonParser(parameters={})
+
     @staticmethod
     def create_json_file_schema_loader(
         model: JsonFileSchemaLoaderModel, config: Config, **kwargs: Any
@@ -2231,24 +2252,18 @@ class ModelToComponentFactory:
             urls_extractor=urls_extractor,
         )
 
-        async_job_partition_router = AsyncJobPartitionRouter(
+        return AsyncRetriever(
             job_orchestrator_factory=lambda stream_slices: AsyncJobOrchestrator(
                 job_repository,
                 stream_slices,
-                JobTracker(1),
-                # FIXME eventually make the number of concurrent jobs in the API configurable. Until then, we limit to 1
+                JobTracker(
+                    1
+                ),  # FIXME eventually make the number of concurrent jobs in the API configurable. Until then, we limit to 1
                 self._message_repository,
-                has_bulk_parent=False,
-                # FIXME work would need to be done here in order to detect if a stream as a parent stream that is bulk
+                has_bulk_parent=False,  # FIXME work would need to be done here in order to detect if a stream as a parent stream that is bulk
             ),
-            stream_slicer=stream_slicer,
-            config=config,
-            parameters=model.parameters or {},
-        )
-
-        return AsyncRetriever(
             record_selector=record_selector,
-            stream_slicer=async_job_partition_router,
+            stream_slicer=stream_slicer,
             config=config,
             parameters=model.parameters or {},
         )

{airbyte_cdk-6.11.2.dev1 → airbyte_cdk-6.12.0rc1}/airbyte_cdk/sources/declarative/partition_routers/__init__.py

@@ -2,18 +2,10 @@
 # Copyright (c) 2022 Airbyte, Inc., all rights reserved.
 #
 
-from airbyte_cdk.sources.declarative.partition_routers.async_job_partition_router import AsyncJobPartitionRouter
 from airbyte_cdk.sources.declarative.partition_routers.cartesian_product_stream_slicer import CartesianProductStreamSlicer
 from airbyte_cdk.sources.declarative.partition_routers.list_partition_router import ListPartitionRouter
 from airbyte_cdk.sources.declarative.partition_routers.single_partition_router import SinglePartitionRouter
 from airbyte_cdk.sources.declarative.partition_routers.substream_partition_router import SubstreamPartitionRouter
 from airbyte_cdk.sources.declarative.partition_routers.partition_router import PartitionRouter
 
-__all__ = [
-    "AsyncJobPartitionRouter",
-    "CartesianProductStreamSlicer",
-    "ListPartitionRouter",
-    "SinglePartitionRouter",
-    "SubstreamPartitionRouter",
-    "PartitionRouter"
-]
+__all__ = ["CartesianProductStreamSlicer", "ListPartitionRouter", "SinglePartitionRouter", "SubstreamPartitionRouter", "PartitionRouter"]