airbyte-cdk 6.12.1__py3-none-any.whl → 6.12.1.dev0__py3-none-any.whl

airbyte_cdk/sources/declarative/declarative_component_schema.yaml

@@ -1234,8 +1234,6 @@ definitions:
             - "$ref": "#/definitions/CustomTransformation"
             - "$ref": "#/definitions/RemoveFields"
             - "$ref": "#/definitions/KeysToLower"
-            - "$ref": "#/definitions/KeysToSnakeCase"
-            - "$ref": "#/definitions/FlattenFields"
       state_migrations:
         title: State Migrations
         description: Array of state migrations to be applied on the input state
@@ -1768,18 +1766,6 @@ definitions:
           - "$ref": "#/definitions/AsyncRetriever"
           - "$ref": "#/definitions/CustomRetriever"
           - "$ref": "#/definitions/SimpleRetriever"
-      schema_transformations:
-        title: Schema Transformations
-        description: A list of transformations to be applied to the schema.
-        type: array
-        items:
-          anyOf:
-            - "$ref": "#/definitions/AddFields"
-            - "$ref": "#/definitions/CustomTransformation"
-            - "$ref": "#/definitions/RemoveFields"
-            - "$ref": "#/definitions/KeysToLower"
-            - "$ref": "#/definitions/KeysToSnakeCase"
-            - "$ref": "#/definitions/FlattenFields"
       schema_type_identifier:
         "$ref": "#/definitions/SchemaTypeIdentifier"
       $parameters:
@@ -1852,32 +1838,6 @@ definitions:
       $parameters:
         type: object
         additionalProperties: true
-  KeysToSnakeCase:
-    title: Key to Snake Case
-    description: A transformation that renames all keys to snake case.
-    type: object
-    required:
-      - type
-    properties:
-      type:
-        type: string
-        enum: [KeysToSnakeCase]
-      $parameters:
-        type: object
-        additionalProperties: true
-  FlattenFields:
-    title: Flatten Fields
-    description: A transformation that flatten record to single level format.
-    type: object
-    required:
-      - type
-    properties:
-      type:
-        type: string
-        enum: [FlattenFields]
-      $parameters:
-        type: object
-        additionalProperties: true
   IterableDecoder:
     title: Iterable Decoder
     description: Use this if the response consists of strings separated by new lines (`\n`). The Decoder will wrap each row into a JSON object with the `record` key.
@@ -2170,63 +2130,65 @@ definitions:
           - extract_output
         properties:
           consent_url:
-            title: Consent URL
+            title: DeclarativeOAuth Consent URL
             type: string
             description: |-
               The DeclarativeOAuth Specific string URL string template to initiate the authentication.
               The placeholders are replaced during the processing to provide neccessary values.
             examples:
-              - https://domain.host.com/marketing_api/auth?{client_id_key}={{client_id_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}&{state_key}={{state_key}}
-              - https://endpoint.host.com/oauth2/authorize?{client_id_key}={{client_id_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}&{scope_key}={urlEncoder:{{scope_key}}}&{state_key}={{state_key}}&subdomain={subdomain}
+              - consent_url: https://domain.host.com/marketing_api/auth?{client_id_key}={{client_id_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}&{state_key}={{state_key}}
+              - consent_url: https://endpoint.host.com/oauth2/authorize?{client_id_key}={{client_id_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}&{scope_key}={urlEncoder:{{scope_key}}}&{state_key}={{state_key}}&subdomain={subdomain}
           scope:
-            title: Scopes
+            title: (Optional) DeclarativeOAuth Scope
             type: string
             description: |-
               The DeclarativeOAuth Specific string of the scopes needed to be grant for authenticated user.
             examples:
-              - user:read user:read_orders workspaces:read
+              - scope: user:read user:read_orders workspaces:read
           access_token_url:
-            title: Access Token URL
+            title: DeclarativeOAuth Access Token URL
             type: string
             description: |-
               The DeclarativeOAuth Specific URL templated string to obtain the `access_token`, `refresh_token` etc.
               The placeholders are replaced during the processing to provide neccessary values.
             examples:
-              - https://auth.host.com/oauth2/token?{client_id_key}={{client_id_key}}&{client_secret_key}={{client_secret_key}}&{auth_code_key}={{auth_code_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}
+              - access_token_url: https://auth.host.com/oauth2/token?{client_id_key}={{client_id_key}}&{client_secret_key}={{client_secret_key}}&{auth_code_key}={{auth_code_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}
           access_token_headers:
-            title: Access Token Headers
+            title: (Optional) DeclarativeOAuth Access Token Headers
             type: object
             additionalProperties: true
             description: |-
               The DeclarativeOAuth Specific optional headers to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.
             examples:
-              - {
-                  "Authorization": "Basic {base64Encoder:{client_id}:{client_secret}}",
-                }
+              - access_token_headers:
+                  {
+                    "Authorization": "Basic {base64Encoder:{client_id}:{client_secret}}",
+                  }
           access_token_params:
-            title: Access Token Query Params (Json Encoded)
+            title: (Optional) DeclarativeOAuth Access Token Query Params (Json Encoded)
             type: object
             additionalProperties: true
             description: |-
               The DeclarativeOAuth Specific optional query parameters to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.
               When this property is provided, the query params will be encoded as `Json` and included in the outgoing API request.
             examples:
-              - {
-                  "{auth_code_key}": "{{auth_code_key}}",
-                  "{client_id_key}": "{{client_id_key}}",
-                  "{client_secret_key}": "{{client_secret_key}}",
-                }
+              - access_token_params:
+                  {
+                    "{auth_code_key}": "{{auth_code_key}}",
+                    "{client_id_key}": "{{client_id_key}}",
+                    "{client_secret_key}": "{{client_secret_key}}",
+                  }
           extract_output:
-            title: Extract Output
+            title: DeclarativeOAuth Extract Output
             type: array
             items:
               type: string
             description: |-
               The DeclarativeOAuth Specific list of strings to indicate which keys should be extracted and returned back to the input config.
             examples:
-              - ["access_token", "refresh_token", "other_field"]
+              - extract_output: ["access_token", "refresh_token", "other_field"]
           state:
-            title: Configurable State Query Param
+            title: (Optional) DeclarativeOAuth Configurable State Query Param
             type: object
             additionalProperties: true
             required:
@@ -2241,49 +2203,49 @@ definitions:
               max:
                 type: integer
             examples:
-              - { "min": 7, "max": 128 }
+              - state: { "min": 7, "max": 128 }
           client_id_key:
-            title: Client ID Key Override
+            title: (Optional) DeclarativeOAuth Client ID Key Override
             type: string
             description: |-
               The DeclarativeOAuth Specific optional override to provide the custom `client_id` key name, if required by data-provider.
             examples:
-              - "my_custom_client_id_key_name"
+              - client_id_key: "my_custom_client_id_key_name"
           client_secret_key:
-            title: Client Secret Key Override
+            title: (Optional) DeclarativeOAuth Client Secret Key Override
             type: string
             description: |-
               The DeclarativeOAuth Specific optional override to provide the custom `client_secret` key name, if required by data-provider.
             examples:
-              - "my_custom_client_secret_key_name"
+              - client_secret_key: "my_custom_client_secret_key_name"
           scope_key:
-            title: Scopes Key Override
+            title: (Optional) DeclarativeOAuth Scope Key Override
             type: string
             description: |-
               The DeclarativeOAuth Specific optional override to provide the custom `scope` key name, if required by data-provider.
             examples:
-              - "my_custom_scope_key_key_name"
+              - scope_key: "my_custom_scope_key_key_name"
           state_key:
-            title: State Key Override
+            title: (Optional) DeclarativeOAuth State Key Override
             type: string
             description: |-
               The DeclarativeOAuth Specific optional override to provide the custom `state` key name, if required by data-provider.
             examples:
-              - "my_custom_state_key_key_name"
+              - state_key: "my_custom_state_key_key_name"
           auth_code_key:
-            title: Auth Code Key Override
+            title: (Optional) DeclarativeOAuth Auth Code Key Override
             type: string
             description: |-
               The DeclarativeOAuth Specific optional override to provide the custom `code` key name to something like `auth_code` or `custom_auth_code`, if required by data-provider.
             examples:
-              - "my_custom_auth_code_key_name"
+              - auth_code_key: "my_custom_auth_code_key_name"
           redirect_uri_key:
-            title: Redirect URI Key Override
+            title: (Optional) DeclarativeOAuth Redirect URI Key Override
             type: string
             description: |-
               The DeclarativeOAuth Specific optional override to provide the custom `redirect_uri` key name to something like `callback_uri`, if required by data-provider.
             examples:
-              - "my_custom_redirect_uri_key_name"
+              - redirect_uri_key: "my_custom_redirect_uri_key_name"
       complete_oauth_output_specification:
         title: "OAuth output specification"
         description: |-

airbyte_cdk/sources/declarative/models/declarative_component_schema.py

@@ -710,16 +710,6 @@ class KeysToLower(BaseModel):
     parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
 
 
-class KeysToSnakeCase(BaseModel):
-    type: Literal["KeysToSnakeCase"]
-    parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
-
-
-class FlattenFields(BaseModel):
-    type: Literal["FlattenFields"]
-    parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
-
-
 class IterableDecoder(BaseModel):
     type: Literal["IterableDecoder"]
 
@@ -805,90 +795,104 @@ class OauthConnectorInputSpecification(BaseModel):
         ...,
         description="The DeclarativeOAuth Specific string URL string template to initiate the authentication.\nThe placeholders are replaced during the processing to provide neccessary values.",
         examples=[
-            "https://domain.host.com/marketing_api/auth?{client_id_key}={{client_id_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}&{state_key}={{state_key}}",
-            "https://endpoint.host.com/oauth2/authorize?{client_id_key}={{client_id_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}&{scope_key}={urlEncoder:{{scope_key}}}&{state_key}={{state_key}}&subdomain={subdomain}",
+            {
+                "consent_url": "https://domain.host.com/marketing_api/auth?{client_id_key}={{client_id_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}&{state_key}={{state_key}}"
+            },
+            {
+                "consent_url": "https://endpoint.host.com/oauth2/authorize?{client_id_key}={{client_id_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}&{scope_key}={urlEncoder:{{scope_key}}}&{state_key}={{state_key}}&subdomain={subdomain}"
+            },
         ],
-        title="Consent URL",
+        title="DeclarativeOAuth Consent URL",
     )
     scope: Optional[str] = Field(
         None,
         description="The DeclarativeOAuth Specific string of the scopes needed to be grant for authenticated user.",
-        examples=["user:read user:read_orders workspaces:read"],
-        title="Scopes",
+        examples=[{"scope": "user:read user:read_orders workspaces:read"}],
+        title="(Optional) DeclarativeOAuth Scope",
     )
     access_token_url: str = Field(
         ...,
         description="The DeclarativeOAuth Specific URL templated string to obtain the `access_token`, `refresh_token` etc.\nThe placeholders are replaced during the processing to provide neccessary values.",
         examples=[
-            "https://auth.host.com/oauth2/token?{client_id_key}={{client_id_key}}&{client_secret_key}={{client_secret_key}}&{auth_code_key}={{auth_code_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}"
+            {
+                "access_token_url": "https://auth.host.com/oauth2/token?{client_id_key}={{client_id_key}}&{client_secret_key}={{client_secret_key}}&{auth_code_key}={{auth_code_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}"
+            }
         ],
-        title="Access Token URL",
+        title="DeclarativeOAuth Access Token URL",
     )
     access_token_headers: Optional[Dict[str, Any]] = Field(
         None,
         description="The DeclarativeOAuth Specific optional headers to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.",
-        examples=[{"Authorization": "Basic {base64Encoder:{client_id}:{client_secret}}"}],
-        title="Access Token Headers",
+        examples=[
+            {
+                "access_token_headers": {
+                    "Authorization": "Basic {base64Encoder:{client_id}:{client_secret}}"
+                }
+            }
+        ],
+        title="(Optional) DeclarativeOAuth Access Token Headers",
     )
     access_token_params: Optional[Dict[str, Any]] = Field(
         None,
         description="The DeclarativeOAuth Specific optional query parameters to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.\nWhen this property is provided, the query params will be encoded as `Json` and included in the outgoing API request.",
         examples=[
             {
-                "{auth_code_key}": "{{auth_code_key}}",
-                "{client_id_key}": "{{client_id_key}}",
-                "{client_secret_key}": "{{client_secret_key}}",
+                "access_token_params": {
+                    "{auth_code_key}": "{{auth_code_key}}",
+                    "{client_id_key}": "{{client_id_key}}",
+                    "{client_secret_key}": "{{client_secret_key}}",
+                }
             }
         ],
-        title="Access Token Query Params (Json Encoded)",
+        title="(Optional) DeclarativeOAuth Access Token Query Params (Json Encoded)",
     )
     extract_output: List[str] = Field(
         ...,
         description="The DeclarativeOAuth Specific list of strings to indicate which keys should be extracted and returned back to the input config.",
-        examples=[["access_token", "refresh_token", "other_field"]],
-        title="Extract Output",
+        examples=[{"extract_output": ["access_token", "refresh_token", "other_field"]}],
+        title="DeclarativeOAuth Extract Output",
     )
     state: Optional[State] = Field(
         None,
         description="The DeclarativeOAuth Specific object to provide the criteria of how the `state` query param should be constructed,\nincluding length and complexity.",
-        examples=[{"min": 7, "max": 128}],
-        title="Configurable State Query Param",
+        examples=[{"state": {"min": 7, "max": 128}}],
+        title="(Optional) DeclarativeOAuth Configurable State Query Param",
     )
     client_id_key: Optional[str] = Field(
         None,
         description="The DeclarativeOAuth Specific optional override to provide the custom `client_id` key name, if required by data-provider.",
-        examples=["my_custom_client_id_key_name"],
-        title="Client ID Key Override",
+        examples=[{"client_id_key": "my_custom_client_id_key_name"}],
+        title="(Optional) DeclarativeOAuth Client ID Key Override",
     )
     client_secret_key: Optional[str] = Field(
         None,
         description="The DeclarativeOAuth Specific optional override to provide the custom `client_secret` key name, if required by data-provider.",
-        examples=["my_custom_client_secret_key_name"],
-        title="Client Secret Key Override",
+        examples=[{"client_secret_key": "my_custom_client_secret_key_name"}],
+        title="(Optional) DeclarativeOAuth Client Secret Key Override",
     )
     scope_key: Optional[str] = Field(
         None,
         description="The DeclarativeOAuth Specific optional override to provide the custom `scope` key name, if required by data-provider.",
-        examples=["my_custom_scope_key_key_name"],
-        title="Scopes Key Override",
+        examples=[{"scope_key": "my_custom_scope_key_key_name"}],
+        title="(Optional) DeclarativeOAuth Scope Key Override",
     )
     state_key: Optional[str] = Field(
         None,
         description="The DeclarativeOAuth Specific optional override to provide the custom `state` key name, if required by data-provider.",
-        examples=["my_custom_state_key_key_name"],
-        title="State Key Override",
+        examples=[{"state_key": "my_custom_state_key_key_name"}],
+        title="(Optional) DeclarativeOAuth State Key Override",
     )
     auth_code_key: Optional[str] = Field(
         None,
         description="The DeclarativeOAuth Specific optional override to provide the custom `code` key name to something like `auth_code` or `custom_auth_code`, if required by data-provider.",
-        examples=["my_custom_auth_code_key_name"],
-        title="Auth Code Key Override",
+        examples=[{"auth_code_key": "my_custom_auth_code_key_name"}],
+        title="(Optional) DeclarativeOAuth Auth Code Key Override",
     )
     redirect_uri_key: Optional[str] = Field(
         None,
         description="The DeclarativeOAuth Specific optional override to provide the custom `redirect_uri` key name to something like `callback_uri`, if required by data-provider.",
-        examples=["my_custom_redirect_uri_key_name"],
-        title="Redirect URI Key Override",
+        examples=[{"redirect_uri_key": "my_custom_redirect_uri_key_name"}],
+        title="(Optional) DeclarativeOAuth Redirect URI Key Override",
     )
 
 
@@ -1664,16 +1668,7 @@ class DeclarativeStream(BaseModel):
         title="Schema Loader",
     )
     transformations: Optional[
-        List[
-            Union[
-                AddFields,
-                CustomTransformation,
-                RemoveFields,
-                KeysToLower,
-                KeysToSnakeCase,
-                FlattenFields,
-            ]
-        ]
+        List[Union[AddFields, CustomTransformation, RemoveFields, KeysToLower]]
     ] = Field(
         None,
         description="A list of transformations to be applied to each output record.",
@@ -1837,22 +1832,6 @@ class DynamicSchemaLoader(BaseModel):
         description="Component used to coordinate how records are extracted across stream slices and request pages.",
         title="Retriever",
     )
-    schema_transformations: Optional[
-        List[
-            Union[
-                AddFields,
-                CustomTransformation,
-                RemoveFields,
-                KeysToLower,
-                KeysToSnakeCase,
-                FlattenFields,
-            ]
-        ]
-    ] = Field(
-        None,
-        description="A list of transformations to be applied to the schema.",
-        title="Schema Transformations",
-    )
     schema_type_identifier: SchemaTypeIdentifier
     parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
 

airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py

@@ -197,9 +197,6 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     ExponentialBackoffStrategy as ExponentialBackoffStrategyModel,
 )
-from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
-    FlattenFields as FlattenFieldsModel,
-)
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     GzipJsonDecoder as GzipJsonDecoderModel,
 )
@@ -239,9 +236,6 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     KeysToLower as KeysToLowerModel,
 )
-from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
-    KeysToSnakeCase as KeysToSnakeCaseModel,
-)
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     LegacySessionTokenAuthenticator as LegacySessionTokenAuthenticatorModel,
 )
@@ -329,9 +323,6 @@ from airbyte_cdk.sources.declarative.partition_routers import (
     SinglePartitionRouter,
     SubstreamPartitionRouter,
 )
-from airbyte_cdk.sources.declarative.partition_routers.async_job_partition_router import (
-    AsyncJobPartitionRouter,
-)
 from airbyte_cdk.sources.declarative.partition_routers.substream_partition_router import (
     ParentStreamConfig,
 )
@@ -396,15 +387,9 @@ from airbyte_cdk.sources.declarative.transformations import (
     RemoveFields,
 )
 from airbyte_cdk.sources.declarative.transformations.add_fields import AddedFieldDefinition
-from airbyte_cdk.sources.declarative.transformations.flatten_fields import (
-    FlattenFields,
-)
 from airbyte_cdk.sources.declarative.transformations.keys_to_lower_transformation import (
     KeysToLowerTransformation,
 )
-from airbyte_cdk.sources.declarative.transformations.keys_to_snake_transformation import (
-    KeysToSnakeCaseTransformation,
-)
 from airbyte_cdk.sources.message import (
     InMemoryMessageRepository,
     LogAppenderMessageRepositoryDecorator,
@@ -487,8 +472,6 @@ class ModelToComponentFactory:
             JsonlDecoderModel: self.create_jsonl_decoder,
             GzipJsonDecoderModel: self.create_gzipjson_decoder,
             KeysToLowerModel: self.create_keys_to_lower_transformation,
-            KeysToSnakeCaseModel: self.create_keys_to_snake_transformation,
-            FlattenFieldsModel: self.create_flatten_fields,
             IterableDecoderModel: self.create_iterable_decoder,
             XmlDecoderModel: self.create_xml_decoder,
             JsonFileSchemaLoaderModel: self.create_json_file_schema_loader,
@@ -604,16 +587,6 @@ class ModelToComponentFactory:
     ) -> KeysToLowerTransformation:
         return KeysToLowerTransformation()
 
-    def create_keys_to_snake_transformation(
-        self, model: KeysToSnakeCaseModel, config: Config, **kwargs: Any
-    ) -> KeysToSnakeCaseTransformation:
-        return KeysToSnakeCaseTransformation()
-
-    def create_flatten_fields(
-        self, model: FlattenFieldsModel, config: Config, **kwargs: Any
-    ) -> FlattenFields:
-        return FlattenFields()
-
     @staticmethod
     def _json_schema_type_name_to_type(value_type: Optional[ValueType]) -> Optional[Type[Any]]:
         if not value_type:
@@ -1665,13 +1638,6 @@ class ModelToComponentFactory:
             model.retriever, stream_slicer
         )
 
-        schema_transformations = []
-        if model.schema_transformations:
-            for transformation_model in model.schema_transformations:
-                schema_transformations.append(
-                    self._create_component_from_model(model=transformation_model, config=config)
-                )
-
         retriever = self._create_component_from_model(
             model=model.retriever,
             config=config,
@@ -1686,7 +1652,6 @@ class ModelToComponentFactory:
         return DynamicSchemaLoader(
             retriever=retriever,
             config=config,
-            schema_transformations=schema_transformations,
             schema_type_identifier=schema_type_identifier,
             parameters=model.parameters or {},
         )
@@ -2263,24 +2228,18 @@ class ModelToComponentFactory:
             urls_extractor=urls_extractor,
         )
 
-        async_job_partition_router = AsyncJobPartitionRouter(
+        return AsyncRetriever(
             job_orchestrator_factory=lambda stream_slices: AsyncJobOrchestrator(
                 job_repository,
                 stream_slices,
-                JobTracker(1),
-                # FIXME eventually make the number of concurrent jobs in the API configurable. Until then, we limit to 1
+                JobTracker(
+                    1
+                ),  # FIXME eventually make the number of concurrent jobs in the API configurable. Until then, we limit to 1
                 self._message_repository,
-                has_bulk_parent=False,
-                # FIXME work would need to be done here in order to detect if a stream as a parent stream that is bulk
+                has_bulk_parent=False,  # FIXME work would need to be done here in order to detect if a stream as a parent stream that is bulk
             ),
-            stream_slicer=stream_slicer,
-            config=config,
-            parameters=model.parameters or {},
-        )
-
-        return AsyncRetriever(
             record_selector=record_selector,
-            stream_slicer=async_job_partition_router,
+            stream_slicer=stream_slicer,
             config=config,
             parameters=model.parameters or {},
         )

airbyte_cdk/sources/declarative/partition_routers/__init__.py

@@ -2,18 +2,10 @@
 # Copyright (c) 2022 Airbyte, Inc., all rights reserved.
 #
 
-from airbyte_cdk.sources.declarative.partition_routers.async_job_partition_router import AsyncJobPartitionRouter
 from airbyte_cdk.sources.declarative.partition_routers.cartesian_product_stream_slicer import CartesianProductStreamSlicer
 from airbyte_cdk.sources.declarative.partition_routers.list_partition_router import ListPartitionRouter
 from airbyte_cdk.sources.declarative.partition_routers.single_partition_router import SinglePartitionRouter
 from airbyte_cdk.sources.declarative.partition_routers.substream_partition_router import SubstreamPartitionRouter
 from airbyte_cdk.sources.declarative.partition_routers.partition_router import PartitionRouter
 
-__all__ = [
-    "AsyncJobPartitionRouter",
-    "CartesianProductStreamSlicer",
-    "ListPartitionRouter",
-    "SinglePartitionRouter",
-    "SubstreamPartitionRouter",
-    "PartitionRouter"
-]
+__all__ = ["CartesianProductStreamSlicer", "ListPartitionRouter", "SinglePartitionRouter", "SubstreamPartitionRouter", "PartitionRouter"]

airbyte_cdk/sources/declarative/retrievers/async_retriever.py

@@ -1,8 +1,8 @@
 # Copyright (c) 2024 Airbyte, Inc., all rights reserved.
 
 
-from dataclasses import InitVar, dataclass
-from typing import Any, Iterable, Mapping, Optional
+from dataclasses import InitVar, dataclass, field
+from typing import Any, Callable, Iterable, Mapping, Optional
 
 from typing_extensions import deprecated
 
@@ -12,10 +12,9 @@ from airbyte_cdk.sources.declarative.async_job.job_orchestrator import (
     AsyncPartition,
 )
 from airbyte_cdk.sources.declarative.extractors.record_selector import RecordSelector
-from airbyte_cdk.sources.declarative.partition_routers.async_job_partition_router import (
-    AsyncJobPartitionRouter,
-)
+from airbyte_cdk.sources.declarative.partition_routers import SinglePartitionRouter
 from airbyte_cdk.sources.declarative.retrievers import Retriever
+from airbyte_cdk.sources.declarative.stream_slicers import StreamSlicer
 from airbyte_cdk.sources.source import ExperimentalClassWarning
 from airbyte_cdk.sources.streams.core import StreamData
 from airbyte_cdk.sources.types import Config, StreamSlice, StreamState
@@ -30,10 +29,15 @@ from airbyte_cdk.utils.traced_exception import AirbyteTracedException
 class AsyncRetriever(Retriever):
     config: Config
     parameters: InitVar[Mapping[str, Any]]
+    job_orchestrator_factory: Callable[[Iterable[StreamSlice]], AsyncJobOrchestrator]
     record_selector: RecordSelector
-    stream_slicer: AsyncJobPartitionRouter
+    stream_slicer: StreamSlicer = field(
+        default_factory=lambda: SinglePartitionRouter(parameters={})
+    )
 
     def __post_init__(self, parameters: Mapping[str, Any]) -> None:
+        self._job_orchestrator_factory = self.job_orchestrator_factory
+        self.__job_orchestrator: Optional[AsyncJobOrchestrator] = None
         self._parameters = parameters
 
     @property
@@ -50,6 +54,17 @@ class AsyncRetriever(Retriever):
         """
         pass
 
+    @property
+    def _job_orchestrator(self) -> AsyncJobOrchestrator:
+        if not self.__job_orchestrator:
+            raise AirbyteTracedException(
+                message="Invalid state within AsyncJobRetriever. Please contact Airbyte Support",
+                internal_message="AsyncPartitionRepository is expected to be accessed only after `stream_slices`",
+                failure_type=FailureType.system_error,
+            )
+
+        return self.__job_orchestrator
+
     def _get_stream_state(self) -> StreamState:
         """
         Gets the current state of the stream.
@@ -84,7 +99,15 @@ class AsyncRetriever(Retriever):
         return stream_slice["partition"]  # type: ignore  # stream_slice["partition"] has been added as an AsyncPartition as part of stream_slices
 
     def stream_slices(self) -> Iterable[Optional[StreamSlice]]:
-        return self.stream_slicer.stream_slices()
+        slices = self.stream_slicer.stream_slices()
+        self.__job_orchestrator = self._job_orchestrator_factory(slices)
+
+        for completed_partition in self._job_orchestrator.create_and_get_completed_partitions():
+            yield StreamSlice(
+                partition=dict(completed_partition.stream_slice.partition)
+                | {"partition": completed_partition},
+                cursor_slice=completed_partition.stream_slice.cursor_slice,
+            )
 
     def read_records(
         self,
@@ -93,7 +116,7 @@ class AsyncRetriever(Retriever):
     ) -> Iterable[StreamData]:
         stream_state: StreamState = self._get_stream_state()
         partition: AsyncPartition = self._validate_and_get_stream_slice_partition(stream_slice)
-        records: Iterable[Mapping[str, Any]] = self.stream_slicer.fetch_records(partition)
+        records: Iterable[Mapping[str, Any]] = self._job_orchestrator.fetch_records(partition)
 
         yield from self.record_selector.filter_and_transform(
             all_data=records,

airbyte_cdk/sources/declarative/schema/dynamic_schema_loader.py

@@ -4,7 +4,7 @@
 
 
 from copy import deepcopy
-from dataclasses import InitVar, dataclass, field
+from dataclasses import InitVar, dataclass
 from typing import Any, List, Mapping, MutableMapping, Optional, Union
 
 import dpath
@@ -13,9 +13,8 @@ from typing_extensions import deprecated
 from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
 from airbyte_cdk.sources.declarative.retrievers.retriever import Retriever
 from airbyte_cdk.sources.declarative.schema.schema_loader import SchemaLoader
-from airbyte_cdk.sources.declarative.transformations import RecordTransformation
 from airbyte_cdk.sources.source import ExperimentalClassWarning
-from airbyte_cdk.sources.types import Config, StreamSlice, StreamState
+from airbyte_cdk.sources.types import Config
 
 AIRBYTE_DATA_TYPES: Mapping[str, Mapping[str, Any]] = {
     "string": {"type": ["null", "string"]},
@@ -104,7 +103,6 @@ class DynamicSchemaLoader(SchemaLoader):
     config: Config
     parameters: InitVar[Mapping[str, Any]]
     schema_type_identifier: SchemaTypeIdentifier
-    schema_transformations: List[RecordTransformation] = field(default_factory=lambda: [])
 
     def get_json_schema(self) -> Mapping[str, Any]:
         """
@@ -130,27 +128,12 @@ class DynamicSchemaLoader(SchemaLoader):
             )
             properties[key] = value
 
-        transformed_properties = self._transform(properties, {})
-
         return {
             "$schema": "http://json-schema.org/draft-07/schema#",
             "type": "object",
-            "properties": transformed_properties,
+            "properties": properties,
         }
 
-    def _transform(
-        self,
-        properties: Mapping[str, Any],
-        stream_state: StreamState,
-        stream_slice: Optional[StreamSlice] = None,
-    ) -> Mapping[str, Any]:
-        for transformation in self.schema_transformations:
-            transformation.transform(
-                properties,  # type: ignore  # properties has type Mapping[str, Any], but Dict[str, Any] expected
-                config=self.config,
-            )
-        return properties
-
     def _get_key(
         self,
         raw_schema: MutableMapping[str, Any],

airbyte_cdk/sources/streams/core.py

@@ -223,17 +223,17 @@ class Stream(ABC):
                     record_counter += 1
 
                     checkpoint_interval = self.state_checkpoint_interval
-                    checkpoint = checkpoint_reader.get_checkpoint()
                     if (
                         should_checkpoint
                         and checkpoint_interval
                         and record_counter % checkpoint_interval == 0
-                        and checkpoint is not None
                     ):
-                        airbyte_state_message = self._checkpoint_state(
-                            checkpoint, state_manager=state_manager
-                        )
-                        yield airbyte_state_message
+                        checkpoint = checkpoint_reader.get_checkpoint()
+                        if checkpoint:
+                            airbyte_state_message = self._checkpoint_state(
+                                checkpoint, state_manager=state_manager
+                            )
+                            yield airbyte_state_message
 
                     if internal_config.is_limit_reached(record_counter):
                         break

airbyte_cdk/sources/streams/http/http_client.py

@@ -262,7 +262,7 @@ class HttpClient:
         user_backoff_handler = user_defined_backoff_handler(max_tries=max_tries, max_time=max_time)(
             self._send
         )
-        rate_limit_backoff_handler = rate_limit_default_backoff_handler(max_tries=max_tries)
+        rate_limit_backoff_handler = rate_limit_default_backoff_handler()
         backoff_handler = http_client_default_backoff_handler(
             max_tries=max_tries, max_time=max_time
         )
@@ -472,9 +472,7 @@ class HttpClient:
 
             elif retry_endlessly:
                 raise RateLimitBackoffException(
-                    request=request,
-                    response=(response if response is not None else exc),
-                    error_message=error_message,
+                    request=request, response=response or exc, error_message=error_message
                 )
 
             raise DefaultBackoffException(

{airbyte_cdk-6.12.1.dist-info → airbyte_cdk-6.12.1.dev0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 6.12.1
+Version: 6.12.1.dev0
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://airbyte.com
 License: MIT
@@ -22,7 +22,6 @@ Provides-Extra: sql
 Provides-Extra: vector-db-based
 Requires-Dist: Jinja2 (>=3.1.2,<3.2.0)
 Requires-Dist: PyYAML (>=6.0.1,<7.0.0)
-Requires-Dist: Unidecode (>=1.3,<2.0)
 Requires-Dist: airbyte-protocol-models-dataclasses (>=0.14,<0.15)
 Requires-Dist: avro (>=1.11.2,<1.12.0) ; extra == "file-based"
 Requires-Dist: backoff

{airbyte_cdk-6.12.1.dist-info → airbyte_cdk-6.12.1.dev0.dist-info}/RECORD

@@ -66,7 +66,7 @@ airbyte_cdk/sources/declarative/concurrent_declarative_source.py,sha256=PxP4p268
 airbyte_cdk/sources/declarative/datetime/__init__.py,sha256=l9LG7Qm6e5r_qgqfVKnx3mXYtg1I9MmMjomVIPfU4XA,177
 airbyte_cdk/sources/declarative/datetime/datetime_parser.py,sha256=SX9JjdesN1edN2WVUVMzU_ptqp2QB1OnsnjZ4mwcX7w,2579
 airbyte_cdk/sources/declarative/datetime/min_max_datetime.py,sha256=0BHBtDNQZfvwM45-tY5pNlTcKAFSGGNxemoi0Jic-0E,5785
-airbyte_cdk/sources/declarative/declarative_component_schema.yaml,sha256=Ls6VUtfX2GvJHR-hv5rs9azjAnunmr8JqZh8vi0DmU4,129264
+airbyte_cdk/sources/declarative/declarative_component_schema.yaml,sha256=bmuzrO0q4yeRXXkAMAFdzgtDx2gF8-k8ERq2MqVMgq4,128593
 airbyte_cdk/sources/declarative/declarative_source.py,sha256=nF7wBqFd3AQmEKAm4CnIo29CJoQL562cJGSCeL8U8bA,1531
 airbyte_cdk/sources/declarative/declarative_stream.py,sha256=JRyNeOIpsFu4ztVZsN6sncqUEIqIE-bUkD2TPgbMgk0,10375
 airbyte_cdk/sources/declarative/decoders/__init__.py,sha256=hNlhaB5FjNC6IfJyglj5ZJWkYD2nEAukMDmzRz5PC6o,671
@@ -104,14 +104,13 @@ airbyte_cdk/sources/declarative/migrations/__init__.py,sha256=47DEQpj8HBSa-_TImW
 airbyte_cdk/sources/declarative/migrations/legacy_to_per_partition_state_migration.py,sha256=iemy3fKLczcU0-Aor7tx5jcT6DRedKMqyK7kCOp01hg,3924
 airbyte_cdk/sources/declarative/migrations/state_migration.py,sha256=KWPjealMLKSMtajXgkdGgKg7EmTLR-CqqD7UIh0-eDU,794
 airbyte_cdk/sources/declarative/models/__init__.py,sha256=nUFxNCiKeYRVXuZEKA7GD-lTHxsiKcQ8FitZjKhPIvE,100
-airbyte_cdk/sources/declarative/models/declarative_component_schema.py,sha256=kESlZ8F2i2q3BMZfEhwEk0XEzxr7SWtJti6AoPsthg0,90831
+airbyte_cdk/sources/declarative/models/declarative_component_schema.py,sha256=-XTzt8EY23Qel7KZINQIbQOoBWqdiK6am-p7A615AI0,90753
 airbyte_cdk/sources/declarative/parsers/__init__.py,sha256=ZnqYNxHsKCgO38IwB34RQyRMXTs4GTvlRi3ImKnIioo,61
 airbyte_cdk/sources/declarative/parsers/custom_exceptions.py,sha256=Rir9_z3Kcd5Es0-LChrzk-0qubAsiK_RSEnLmK2OXm8,553
 airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py,sha256=CXwTfD3wSQq3okcqwigpprbHhSURUokh4GK2OmOyKC8,9132
 airbyte_cdk/sources/declarative/parsers/manifest_reference_resolver.py,sha256=IWUOdF03o-aQn0Occo1BJCxU0Pz-QILk5L67nzw2thw,6803
-airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py,sha256=i2Z4q_9pYWc40uiHJ3UMqAh1hgoVTTXybxRaWzbwNHE,106031
-airbyte_cdk/sources/declarative/partition_routers/__init__.py,sha256=974SY1RFwitUCiiDHuFHDGmSNu1D72z3bSTpvlBwAho,911
-airbyte_cdk/sources/declarative/partition_routers/async_job_partition_router.py,sha256=n82J15S8bjeMZ5uROu--P3hnbQoxkY5v7RPHYx7g7ro,2929
+airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py,sha256=UWUlj6m2WT9SspMtvvhni31FoQBrkqnHGzrwFKem-Qc,104367
+airbyte_cdk/sources/declarative/partition_routers/__init__.py,sha256=1NjaZoGAIefvWwj6wx-LOKIXXWS-UnBlZFnuR7y6uYA,745
 airbyte_cdk/sources/declarative/partition_routers/cartesian_product_stream_slicer.py,sha256=c5cuVFM6NFkuQqG8Z5IwkBuwDrvXZN1CunUOM_L0ezg,6892
 airbyte_cdk/sources/declarative/partition_routers/list_partition_router.py,sha256=t7pRdFWfFWJtQQG19c9PVeMODyO2BknRTakpM5U9N-8,4844
 airbyte_cdk/sources/declarative/partition_routers/partition_router.py,sha256=YyEIzdmLd1FjbVP3QbQ2VFCLW_P-OGbVh6VpZShp54k,2218
@@ -158,12 +157,12 @@ airbyte_cdk/sources/declarative/resolvers/components_resolver.py,sha256=KPjKc0yb
 airbyte_cdk/sources/declarative/resolvers/config_components_resolver.py,sha256=dz4iJV9liD_LzY_Mn4XmAStoUll60R3MIGWV4aN3pgg,5223
 airbyte_cdk/sources/declarative/resolvers/http_components_resolver.py,sha256=ZA2vrHQKfXNMcH3x1iuyFOTGNzYDhUFT2qcaiOzSK0A,4271
 airbyte_cdk/sources/declarative/retrievers/__init__.py,sha256=FVQpUGVwp2Gibk4gp07VmLKX5AafUlsZWFSrDpUDuJM,443
-airbyte_cdk/sources/declarative/retrievers/async_retriever.py,sha256=3jgor7a6_s_9KgqHmPk6cWMDZ-6OugFPjCajIkC3Onw,3721
+airbyte_cdk/sources/declarative/retrievers/async_retriever.py,sha256=WDFnjrXLz3-YEjFhmlMkWAn9AJvnZ0mk9FyC8DAhEYk,4976
 airbyte_cdk/sources/declarative/retrievers/retriever.py,sha256=XPLs593Xv8c5cKMc37XzUAYmzlXd1a7eSsspM-CMuWA,1696
 airbyte_cdk/sources/declarative/retrievers/simple_retriever.py,sha256=N4swGw5mfuTXJ2R7AKX18CHzizsr69pXwt5uSHLPi48,24172
 airbyte_cdk/sources/declarative/schema/__init__.py,sha256=Io9vninzlEjQ2uFmWklxfwNM0cXfljtzOz5zL1OVyT4,701
 airbyte_cdk/sources/declarative/schema/default_schema_loader.py,sha256=KTACrIE23a83wsm3Rd9Eb4K6-20lrGqYxTHNp9yxsso,1820
-airbyte_cdk/sources/declarative/schema/dynamic_schema_loader.py,sha256=H6A3NQ6kPPM-cUNPmdvDPc9xNzR1rQNrK95GbgCW334,8822
+airbyte_cdk/sources/declarative/schema/dynamic_schema_loader.py,sha256=tP5DIEMn-k2JshWeXmo53ZEudDAVb4AJ50Z5tfme_ZU,8063
 airbyte_cdk/sources/declarative/schema/inline_schema_loader.py,sha256=bVETE10hRsatRJq3R3BeyRR0wIoK3gcP1gcpVRQ_P5U,464
 airbyte_cdk/sources/declarative/schema/json_file_schema_loader.py,sha256=5Wl-fqW-pVf_dxJ4yGHMAFfC4JjKHYJhqFJT1xA57F4,4177
 airbyte_cdk/sources/declarative/schema/schema_loader.py,sha256=kjt8v0N5wWKA5zyLnrDLxf1PJKdUqvQq2RVnAOAzNSY,379
@@ -174,9 +173,7 @@ airbyte_cdk/sources/declarative/stream_slicers/declarative_partition_generator.p
 airbyte_cdk/sources/declarative/stream_slicers/stream_slicer.py,sha256=SOkIPBi2Wu7yxIvA15yFzUAB95a3IzA8LPq5DEqHQQc,725
 airbyte_cdk/sources/declarative/transformations/__init__.py,sha256=CPJ8TlMpiUmvG3624VYu_NfTzxwKcfBjM2Q2wJ7fkSA,919
 airbyte_cdk/sources/declarative/transformations/add_fields.py,sha256=r4YdAuAk2bQtNWJMztIIy2CC-NglD9NeK1s1TeO9wkw,5027
-airbyte_cdk/sources/declarative/transformations/flatten_fields.py,sha256=ti9fLVk-EpMeDY7ImduvQq1YGounLYmH9dHzp7MIRxk,1703
 airbyte_cdk/sources/declarative/transformations/keys_to_lower_transformation.py,sha256=RTs5KX4V3hM7A6QN1WlGF21YccTIyNH6qQI9IMb__hw,670
-airbyte_cdk/sources/declarative/transformations/keys_to_snake_transformation.py,sha256=43zwe6_F5ba5C4eY0RgXxPz7ndPKZfXGChHepFn-2lk,2263
 airbyte_cdk/sources/declarative/transformations/remove_fields.py,sha256=EwUP0SZ2p4GRJ6Q8CUzlz9dcUeEidEFDlI2IBye2tlc,2745
 airbyte_cdk/sources/declarative/transformations/transformation.py,sha256=4sXtx9cNY2EHUPq-xHvDs8GQEBUy3Eo6TkRLKHPXx68,1161
 airbyte_cdk/sources/declarative/types.py,sha256=yqx0xlZv_76tkC7fqJKefmvl4GJJ8mXbeddwVV8XRJU,778
@@ -267,7 +264,7 @@ airbyte_cdk/sources/streams/concurrent/partitions/types.py,sha256=frPVvHtY7vLxpG
 airbyte_cdk/sources/streams/concurrent/state_converters/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 airbyte_cdk/sources/streams/concurrent/state_converters/abstract_stream_state_converter.py,sha256=CXHUMOhndu-LOKgsnNTItv5s5qrKpmJDeHOzlH1nBy8,6819
 airbyte_cdk/sources/streams/concurrent/state_converters/datetime_stream_state_converter.py,sha256=syjdxEoElIOzqVS5Jrm5FOR70jsbBdttEO_3Iz12Jyo,7523
-airbyte_cdk/sources/streams/core.py,sha256=z4Oi5qmJPjs-RdMd5tPWHvHqIjkcxhkVKTvIpfAs2uA,32211
+airbyte_cdk/sources/streams/core.py,sha256=jiYW6w8cjNjzXMd8U8Gt-02fYYU7b0ciXSSSnGvFRak,32219
 airbyte_cdk/sources/streams/http/__init__.py,sha256=NXaNlkzZMkh5kS8S5ujEaKEE6855sk6_HljF_GFjKZI,311
 airbyte_cdk/sources/streams/http/availability_strategy.py,sha256=sovoGFThZr-doMN9vJvTuJBrvkwQVIO0qTQO64pGZPY,2428
 airbyte_cdk/sources/streams/http/error_handlers/__init__.py,sha256=R8OgTcratGH4f6BbYM2Hp8qYyEk7wMYuyda5H9ohGW8,665
@@ -281,7 +278,7 @@ airbyte_cdk/sources/streams/http/error_handlers/json_error_message_parser.py,sha
 airbyte_cdk/sources/streams/http/error_handlers/response_models.py,sha256=xGIVELBFY0TmH9aUq1ikoqJz8oHLr6di2JLvKWVEO-s,2236
 airbyte_cdk/sources/streams/http/exceptions.py,sha256=njC7MlMJoFYcSGz4mIp6-bqLFTr6vC8ej25X0oSeyjE,1824
 airbyte_cdk/sources/streams/http/http.py,sha256=JAMpiTdS9HFNOlwayWNvQdxoqs2rpW9wdYlhFHv_1Q4,28496
-airbyte_cdk/sources/streams/http/http_client.py,sha256=tDE0ROtxjGMVphvsw8INvGMtZ97hIF-v47pZ3jIyiwc,23011
+airbyte_cdk/sources/streams/http/http_client.py,sha256=dyNrbcahEnDfGTrhqkr1XdfPiuVVRlKOdj-TJ5WRVrk,22923
 airbyte_cdk/sources/streams/http/rate_limiting.py,sha256=IwdjrHKUnU97XO4qONgYRv4YYW51xQ8SJm4WLafXDB8,6351
 airbyte_cdk/sources/streams/http/requests_native_auth/__init__.py,sha256=RN0D3nOX1xLgwEwKWu6pkGy3XqBFzKSNZ8Lf6umU2eY,413
 airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py,sha256=nxI94yJ3bGfpDO8RR3QvOJ-PSW0n9CElSAkgl5ae80Y,10321
@@ -339,8 +336,8 @@ airbyte_cdk/utils/slice_hasher.py,sha256=-pHexlNYoWYPnXNH-M7HEbjmeJe9Zk7SJijdQ7d
 airbyte_cdk/utils/spec_schema_transformations.py,sha256=-5HTuNsnDBAhj-oLeQXwpTGA0HdcjFOf2zTEMUTTg_Y,816
 airbyte_cdk/utils/stream_status_utils.py,sha256=ZmBoiy5HVbUEHAMrUONxZvxnvfV9CesmQJLDTAIWnWw,1171
 airbyte_cdk/utils/traced_exception.py,sha256=C8uIBuCL_E4WnBAOPSxBicD06JAldoN9fGsQDp463OY,6292
-airbyte_cdk-6.12.1.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
-airbyte_cdk-6.12.1.dist-info/METADATA,sha256=w7op06XAtTqV47wgvbGddUc7pHN6TQB2YvBh8vjQ7FM,5988
-airbyte_cdk-6.12.1.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-airbyte_cdk-6.12.1.dist-info/entry_points.txt,sha256=fj-e3PAQvsxsQzyyq8UkG1k8spunWnD4BAH2AwlR6NM,95
-airbyte_cdk-6.12.1.dist-info/RECORD,,
+airbyte_cdk-6.12.1.dev0.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
+airbyte_cdk-6.12.1.dev0.dist-info/METADATA,sha256=wuEYLVSyehdNYlyYVlLw9n2uOyx1aT_WRmRnDlfHu0s,5955
+airbyte_cdk-6.12.1.dev0.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+airbyte_cdk-6.12.1.dev0.dist-info/entry_points.txt,sha256=fj-e3PAQvsxsQzyyq8UkG1k8spunWnD4BAH2AwlR6NM,95
+airbyte_cdk-6.12.1.dev0.dist-info/RECORD,,

airbyte_cdk/sources/declarative/partition_routers/async_job_partition_router.py

@@ -1,65 +0,0 @@
-# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
-
-from dataclasses import InitVar, dataclass, field
-from typing import Any, Callable, Iterable, Mapping, Optional
-
-from airbyte_cdk.models import FailureType
-from airbyte_cdk.sources.declarative.async_job.job_orchestrator import (
-    AsyncJobOrchestrator,
-    AsyncPartition,
-)
-from airbyte_cdk.sources.declarative.partition_routers.single_partition_router import (
-    SinglePartitionRouter,
-)
-from airbyte_cdk.sources.streams.concurrent.partitions.stream_slicer import StreamSlicer
-from airbyte_cdk.sources.types import Config, StreamSlice
-from airbyte_cdk.utils.traced_exception import AirbyteTracedException
-
-
-@dataclass
-class AsyncJobPartitionRouter(StreamSlicer):
-    """
-    Partition router that creates async jobs in a source API, periodically polls for job
-    completion, and supplies the completed job URL locations as stream slices so that
-    records can be extracted.
-    """
-
-    config: Config
-    parameters: InitVar[Mapping[str, Any]]
-    job_orchestrator_factory: Callable[[Iterable[StreamSlice]], AsyncJobOrchestrator]
-    stream_slicer: StreamSlicer = field(
-        default_factory=lambda: SinglePartitionRouter(parameters={})
-    )
-
-    def __post_init__(self, parameters: Mapping[str, Any]) -> None:
-        self._job_orchestrator_factory = self.job_orchestrator_factory
-        self._job_orchestrator: Optional[AsyncJobOrchestrator] = None
-        self._parameters = parameters
-
-    def stream_slices(self) -> Iterable[StreamSlice]:
-        slices = self.stream_slicer.stream_slices()
-        self._job_orchestrator = self._job_orchestrator_factory(slices)
-
-        for completed_partition in self._job_orchestrator.create_and_get_completed_partitions():
-            yield StreamSlice(
-                partition=dict(completed_partition.stream_slice.partition)
-                | {"partition": completed_partition},
-                cursor_slice=completed_partition.stream_slice.cursor_slice,
-            )
-
-    def fetch_records(self, partition: AsyncPartition) -> Iterable[Mapping[str, Any]]:
-        """
-        This method of fetching records extends beyond what a PartitionRouter/StreamSlicer should
-        be responsible for. However, this was added in because the JobOrchestrator is required to
-        retrieve records. And without defining fetch_records() on this class, we're stuck with either
-        passing the JobOrchestrator to the AsyncRetriever or storing it on multiple classes.
-        """
-
-        if not self._job_orchestrator:
-            raise AirbyteTracedException(
-                message="Invalid state within AsyncJobRetriever. Please contact Airbyte Support",
-                internal_message="AsyncPartitionRepository is expected to be accessed only after `stream_slices`",
-                failure_type=FailureType.system_error,
-            )
-
-        return self._job_orchestrator.fetch_records(partition=partition)

airbyte_cdk/sources/declarative/transformations/flatten_fields.py

@@ -1,50 +0,0 @@
-#
-# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
-#
-
-from dataclasses import dataclass
-from typing import Any, Dict, Optional
-
-from airbyte_cdk.sources.declarative.transformations import RecordTransformation
-from airbyte_cdk.sources.types import Config, StreamSlice, StreamState
-
-
-@dataclass
-class FlattenFields(RecordTransformation):
-    def transform(
-        self,
-        record: Dict[str, Any],
-        config: Optional[Config] = None,
-        stream_state: Optional[StreamState] = None,
-        stream_slice: Optional[StreamSlice] = None,
-    ) -> None:
-        transformed_record = self.flatten_record(record)
-        record.clear()
-        record.update(transformed_record)
-
-    def flatten_record(self, record: Dict[str, Any]) -> Dict[str, Any]:
-        stack = [(record, "_")]
-        transformed_record: Dict[str, Any] = {}
-        force_with_parent_name = False
-
-        while stack:
-            current_record, parent_key = stack.pop()
-
-            if isinstance(current_record, dict):
-                for current_key, value in current_record.items():
-                    new_key = (
-                        f"{parent_key}.{current_key}"
-                        if (current_key in transformed_record or force_with_parent_name)
-                        else current_key
-                    )
-                    stack.append((value, new_key))
-
-            elif isinstance(current_record, list):
-                for i, item in enumerate(current_record):
-                    force_with_parent_name = True
-                    stack.append((item, f"{parent_key}.{i}"))
-
-            else:
-                transformed_record[parent_key] = current_record
-
-        return transformed_record

airbyte_cdk/sources/declarative/transformations/keys_to_snake_transformation.py

@@ -1,68 +0,0 @@
-#
-# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
-#
-
-import re
-from dataclasses import dataclass
-from typing import Any, Dict, List, Optional
-
-import unidecode
-
-from airbyte_cdk.sources.declarative.transformations import RecordTransformation
-from airbyte_cdk.sources.types import Config, StreamSlice, StreamState
-
-
-@dataclass
-class KeysToSnakeCaseTransformation(RecordTransformation):
-    token_pattern: re.Pattern[str] = re.compile(
-        r"[A-Z]+[a-z]*|[a-z]+|\d+|(?P<NoToken>[^a-zA-Z\d]+)"
-    )
-
-    def transform(
-        self,
-        record: Dict[str, Any],
-        config: Optional[Config] = None,
-        stream_state: Optional[StreamState] = None,
-        stream_slice: Optional[StreamSlice] = None,
-    ) -> None:
-        transformed_record = self._transform_record(record)
-        record.clear()
-        record.update(transformed_record)
-
-    def _transform_record(self, record: Dict[str, Any]) -> Dict[str, Any]:
-        transformed_record = {}
-        for key, value in record.items():
-            transformed_key = self.process_key(key)
-            transformed_value = value
-
-            if isinstance(value, dict):
-                transformed_value = self._transform_record(value)
-
-            transformed_record[transformed_key] = transformed_value
-        return transformed_record
-
-    def process_key(self, key: str) -> str:
-        key = self.normalize_key(key)
-        tokens = self.tokenize_key(key)
-        tokens = self.filter_tokens(tokens)
-        return self.tokens_to_snake_case(tokens)
-
-    def normalize_key(self, key: str) -> str:
-        return unidecode.unidecode(key)
-
-    def tokenize_key(self, key: str) -> List[str]:
-        tokens = []
-        for match in self.token_pattern.finditer(key):
-            token = match.group(0) if match.group("NoToken") is None else ""
-            tokens.append(token)
-        return tokens
-
-    def filter_tokens(self, tokens: List[str]) -> List[str]:
-        if len(tokens) >= 3:
-            tokens = tokens[:1] + [t for t in tokens[1:-1] if t] + tokens[-1:]
-        if tokens and tokens[0].isdigit():
-            tokens.insert(0, "")
-        return tokens
-
-    def tokens_to_snake_case(self, tokens: List[str]) -> str:
-        return "_".join(token.lower() for token in tokens)