airbyte-cdk 6.12.4.dev0__py3-none-any.whl → 6.13.0__py3-none-any.whl

airbyte_cdk/__init__.py

@@ -48,27 +48,46 @@ API Reference
 # Once those issues are resolved, the below can be sorted with isort.
 import dunamai as _dunamai
 
-from .destinations import Destination
-from .models import AirbyteConnectionStatus, AirbyteMessage, ConfiguredAirbyteCatalog, Status, Type, FailureType, AirbyteStream, AdvancedAuth, DestinationSyncMode, ConnectorSpecification, OAuthConfigSpecification, OrchestratorType, ConfiguredAirbyteStream, SyncMode, AirbyteLogMessage, Level, AirbyteRecordMessage
-
-from .sources import Source
-from .config_observation import create_connector_config_control_message, emit_configuration_as_airbyte_control_message
+from .config_observation import (
+    create_connector_config_control_message,
+    emit_configuration_as_airbyte_control_message,
+)
 from .connector import BaseConnector, Connector
-
-from .entrypoint import launch, AirbyteEntrypoint
-
+from .destinations import Destination
+from .entrypoint import AirbyteEntrypoint, launch
 from .logger import AirbyteLogFormatter, init_logger
-from .sources import AbstractSource
+from .models import (
+    AdvancedAuth,
+    AirbyteConnectionStatus,
+    AirbyteLogMessage,
+    AirbyteMessage,
+    AirbyteRecordMessage,
+    AirbyteStream,
+    ConfiguredAirbyteCatalog,
+    ConfiguredAirbyteStream,
+    ConnectorSpecification,
+    DestinationSyncMode,
+    FailureType,
+    Level,
+    OAuthConfigSpecification,
+    OrchestratorType,
+    Status,
+    SyncMode,
+    Type,
+)
+from .sources import AbstractSource, Source
 from .sources.concurrent_source.concurrent_source import ConcurrentSource
 from .sources.concurrent_source.concurrent_source_adapter import ConcurrentSourceAdapter
 from .sources.config import BaseConfig
-from .sources.types import Config, Record, StreamSlice
 from .sources.connector_state_manager import ConnectorStateManager
 from .sources.declarative.auth import DeclarativeOauth2Authenticator
-from .sources.declarative.auth.declarative_authenticator import DeclarativeAuthenticator
-from .sources.declarative.auth.declarative_authenticator import NoAuth
+from .sources.declarative.auth.declarative_authenticator import DeclarativeAuthenticator, NoAuth
 from .sources.declarative.auth.oauth import DeclarativeSingleUseRefreshTokenOauth2Authenticator
-from .sources.declarative.auth.token import BasicHttpAuthenticator, BearerAuthenticator, ApiKeyAuthenticator
+from .sources.declarative.auth.token import (
+    ApiKeyAuthenticator,
+    BasicHttpAuthenticator,
+    BearerAuthenticator,
+)
 from .sources.declarative.datetime.min_max_datetime import MinMaxDatetime
 from .sources.declarative.declarative_stream import DeclarativeStream
 from .sources.declarative.decoders import Decoder, JsonDecoder
@@ -77,48 +96,89 @@ from .sources.declarative.extractors import DpathExtractor, RecordSelector
 from .sources.declarative.extractors.record_extractor import RecordExtractor
 from .sources.declarative.extractors.record_filter import RecordFilter
 from .sources.declarative.incremental import DatetimeBasedCursor
-from .sources.declarative.interpolation import InterpolatedString, InterpolatedBoolean
+from .sources.declarative.interpolation import InterpolatedBoolean, InterpolatedString
 from .sources.declarative.manifest_declarative_source import ManifestDeclarativeSource
-from .sources.declarative.migrations.legacy_to_per_partition_state_migration import LegacyToPerPartitionStateMigration
-
-from .sources.declarative.partition_routers import CartesianProductStreamSlicer, SinglePartitionRouter, SubstreamPartitionRouter
+from .sources.declarative.migrations.legacy_to_per_partition_state_migration import (
+    LegacyToPerPartitionStateMigration,
+)
+from .sources.declarative.partition_routers import (
+    CartesianProductStreamSlicer,
+    SinglePartitionRouter,
+    SubstreamPartitionRouter,
+)
 from .sources.declarative.partition_routers.substream_partition_router import ParentStreamConfig
-from .sources.declarative.requesters import Requester, HttpRequester
-
+from .sources.declarative.requesters import HttpRequester, Requester
 from .sources.declarative.requesters.error_handlers import BackoffStrategy
 from .sources.declarative.requesters.paginators import DefaultPaginator, PaginationStrategy
-from .sources.declarative.requesters.paginators.strategies import OffsetIncrement, CursorPaginationStrategy, PageIncrement, StopConditionPaginationStrategyDecorator
-
+from .sources.declarative.requesters.paginators.strategies import (
+    CursorPaginationStrategy,
+    OffsetIncrement,
+    PageIncrement,
+    StopConditionPaginationStrategyDecorator,
+)
 from .sources.declarative.requesters.request_option import RequestOption, RequestOptionType
-
-from .sources.declarative.requesters.request_options.default_request_options_provider import DefaultRequestOptionsProvider
-from .sources.declarative.requesters.request_options.interpolated_request_input_provider import InterpolatedRequestInputProvider
+from .sources.declarative.requesters.request_options.default_request_options_provider import (
+    DefaultRequestOptionsProvider,
+)
+from .sources.declarative.requesters.request_options.interpolated_request_input_provider import (
+    InterpolatedRequestInputProvider,
+)
 from .sources.declarative.requesters.requester import HttpMethod
 from .sources.declarative.retrievers import SimpleRetriever
 from .sources.declarative.schema import JsonFileSchemaLoader
-from .sources.declarative.transformations.add_fields import AddFields, AddedFieldDefinition
+from .sources.declarative.transformations.add_fields import AddedFieldDefinition, AddFields
 from .sources.declarative.transformations.transformation import RecordTransformation
 from .sources.declarative.types import FieldPointer
 from .sources.declarative.yaml_declarative_source import YamlDeclarativeSource
 from .sources.message import InMemoryMessageRepository, MessageRepository
 from .sources.source import TState
 from .sources.streams.availability_strategy import AvailabilityStrategy
-from .sources.streams.call_rate import AbstractAPIBudget, HttpAPIBudget, HttpRequestMatcher, MovingWindowCallRatePolicy, Rate, CachedLimiterSession, LimiterSession
+from .sources.streams.call_rate import (
+    AbstractAPIBudget,
+    CachedLimiterSession,
+    HttpAPIBudget,
+    HttpRequestMatcher,
+    LimiterSession,
+    MovingWindowCallRatePolicy,
+    Rate,
+)
 from .sources.streams.checkpoint import Cursor as LegacyCursor
 from .sources.streams.checkpoint import ResumableFullRefreshCursor
 from .sources.streams.concurrent.adapters import StreamFacade
-from .sources.streams.concurrent.cursor import ConcurrentCursor, CursorField, FinalStateCursor
-from .sources.streams.concurrent.cursor import Cursor
-from .sources.streams.concurrent.state_converters.datetime_stream_state_converter import EpochValueConcurrentStreamStateConverter, IsoMillisConcurrentStreamStateConverter
-from .sources.streams.core import Stream, IncrementalMixin, package_name_from_class
+from .sources.streams.concurrent.cursor import (
+    ConcurrentCursor,
+    Cursor,
+    CursorField,
+    FinalStateCursor,
+)
+from .sources.streams.concurrent.state_converters.datetime_stream_state_converter import (
+    EpochValueConcurrentStreamStateConverter,
+    IsoMillisConcurrentStreamStateConverter,
+)
+from .sources.streams.core import IncrementalMixin, Stream, package_name_from_class
 from .sources.streams.http import HttpStream, HttpSubStream
 from .sources.streams.http.availability_strategy import HttpAvailabilityStrategy
-from .sources.streams.http.exceptions import BaseBackoffException, DefaultBackoffException, UserDefinedBackoffException
+from .sources.streams.http.exceptions import (
+    BaseBackoffException,
+    DefaultBackoffException,
+    UserDefinedBackoffException,
+)
 from .sources.streams.http.rate_limiting import default_backoff_handler
-from .sources.streams.http.requests_native_auth import Oauth2Authenticator, TokenAuthenticator, SingleUseRefreshTokenOauth2Authenticator
+from .sources.streams.http.requests_native_auth import (
+    Oauth2Authenticator,
+    SingleUseRefreshTokenOauth2Authenticator,
+    TokenAuthenticator,
+)
 from .sources.streams.http.requests_native_auth.abstract_token import AbstractHeaderAuthenticator
+from .sources.types import Config, Record, StreamSlice
 from .sources.utils import casing
-from .sources.utils.schema_helpers import InternalConfig, ResourceSchemaLoader, check_config_against_spec_or_exit, split_config, expand_refs
+from .sources.utils.schema_helpers import (
+    InternalConfig,
+    ResourceSchemaLoader,
+    check_config_against_spec_or_exit,
+    expand_refs,
+    split_config,
+)
 from .sources.utils.transform import TransformConfig, TypeTransformer
 from .utils import AirbyteTracedException, is_cloud_environment
 from .utils.constants import ENV_REQUEST_CACHE_PATH
@@ -127,7 +187,6 @@ from .utils.oneof_option_config import OneOfOptionConfig
 from .utils.spec_schema_transformations import resolve_refs
 from .utils.stream_status_utils import as_airbyte_message
 
-
 __all__ = [
     # Availability strategy
     "AvailabilityStrategy",

airbyte_cdk/cli/source_declarative_manifest/__init__.py

@@ -1,6 +1,5 @@
 from airbyte_cdk.cli.source_declarative_manifest._run import run
 
-
 __all__ = [
     "run",
 ]

airbyte_cdk/models/__init__.py

@@ -7,7 +7,6 @@
 # of airbyte-cdk rather than a standalone package.
 from .airbyte_protocol import (
     AdvancedAuth,
-    AirbyteStateStats,
     AirbyteAnalyticsTraceMessage,
     AirbyteCatalog,
     AirbyteConnectionStatus,
@@ -22,13 +21,14 @@ from .airbyte_protocol import (
     AirbyteRecordMessage,
     AirbyteStateBlob,
     AirbyteStateMessage,
+    AirbyteStateStats,
     AirbyteStateType,
     AirbyteStream,
     AirbyteStreamState,
     AirbyteStreamStatus,
-    AirbyteStreamStatusTraceMessage,
     AirbyteStreamStatusReason,
     AirbyteStreamStatusReasonType,
+    AirbyteStreamStatusTraceMessage,
     AirbyteTraceMessage,
     AuthFlowType,
     ConfiguredAirbyteCatalog,
@@ -48,6 +48,14 @@ from .airbyte_protocol import (
     TraceType,
     Type,
 )
+from .airbyte_protocol_serializers import (
+    AirbyteMessageSerializer,
+    AirbyteStateMessageSerializer,
+    AirbyteStreamStateSerializer,
+    ConfiguredAirbyteCatalogSerializer,
+    ConfiguredAirbyteStreamSerializer,
+    ConnectorSpecificationSerializer,
+)
 from .well_known_types import (
     BinaryData,
     Boolean,
@@ -61,12 +69,3 @@ from .well_known_types import (
     TimeWithoutTimezone,
     TimeWithTimezone,
 )
-
-from .airbyte_protocol_serializers import (
-AirbyteStreamStateSerializer,
-AirbyteStateMessageSerializer,
-AirbyteMessageSerializer,
-ConfiguredAirbyteCatalogSerializer,
-ConfiguredAirbyteStreamSerializer,
-ConnectorSpecificationSerializer,
-)

airbyte_cdk/sources/declarative/async_job/job_orchestrator.py

@@ -20,7 +20,6 @@ from typing import (
     TypeVar,
 )
 
-from airbyte_cdk import StreamSlice
 from airbyte_cdk.logger import lazy_log
 from airbyte_cdk.models import FailureType
 from airbyte_cdk.sources.declarative.async_job.job import AsyncJob
@@ -31,6 +30,7 @@ from airbyte_cdk.sources.declarative.async_job.job_tracker import (
 from airbyte_cdk.sources.declarative.async_job.repository import AsyncJobRepository
 from airbyte_cdk.sources.declarative.async_job.status import AsyncJobStatus
 from airbyte_cdk.sources.message import MessageRepository
+from airbyte_cdk.sources.types import StreamSlice
 from airbyte_cdk.utils.airbyte_secrets_utils import filter_secrets
 from airbyte_cdk.utils.traced_exception import AirbyteTracedException
 

airbyte_cdk/sources/declarative/auth/__init__.py

@@ -2,10 +2,7 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
 
-from airbyte_cdk.sources.declarative.auth.oauth import DeclarativeOauth2Authenticator
 from airbyte_cdk.sources.declarative.auth.jwt import JwtAuthenticator
+from airbyte_cdk.sources.declarative.auth.oauth import DeclarativeOauth2Authenticator
 
-__all__ = [
-    "DeclarativeOauth2Authenticator",
-    "JwtAuthenticator"
-]
+__all__ = ["DeclarativeOauth2Authenticator", "JwtAuthenticator"]

airbyte_cdk/sources/declarative/auth/oauth.py

@@ -43,11 +43,11 @@ class DeclarativeOauth2Authenticator(AbstractOauth2Authenticator, DeclarativeAut
         message_repository (MessageRepository): the message repository used to emit logs on HTTP requests
     """
 
-    token_refresh_endpoint: Union[InterpolatedString, str]
     client_id: Union[InterpolatedString, str]
     client_secret: Union[InterpolatedString, str]
     config: Mapping[str, Any]
     parameters: InitVar[Mapping[str, Any]]
+    token_refresh_endpoint: Optional[Union[InterpolatedString, str]] = None
     refresh_token: Optional[Union[InterpolatedString, str]] = None
     scopes: Optional[List[str]] = None
     token_expiry_date: Optional[Union[InterpolatedString, str]] = None
@@ -55,6 +55,7 @@ class DeclarativeOauth2Authenticator(AbstractOauth2Authenticator, DeclarativeAut
     token_expiry_date_format: Optional[str] = None
     token_expiry_is_time_of_expiration: bool = False
     access_token_name: Union[InterpolatedString, str] = "access_token"
+    access_token_value: Optional[Union[InterpolatedString, str]] = None
     expires_in_name: Union[InterpolatedString, str] = "expires_in"
     refresh_request_body: Optional[Mapping[str, Any]] = None
     grant_type: Union[InterpolatedString, str] = "refresh_token"
@@ -62,9 +63,12 @@ class DeclarativeOauth2Authenticator(AbstractOauth2Authenticator, DeclarativeAut
 
     def __post_init__(self, parameters: Mapping[str, Any]) -> None:
         super().__init__()
-        self._token_refresh_endpoint = InterpolatedString.create(
-            self.token_refresh_endpoint, parameters=parameters
-        )
+        if self.token_refresh_endpoint is not None:
+            self._token_refresh_endpoint: Optional[InterpolatedString] = InterpolatedString.create(
+                self.token_refresh_endpoint, parameters=parameters
+            )
+        else:
+            self._token_refresh_endpoint = None
         self._client_id = InterpolatedString.create(self.client_id, parameters=parameters)
         self._client_secret = InterpolatedString.create(self.client_secret, parameters=parameters)
         if self.refresh_token is not None:
@@ -92,20 +96,31 @@ class DeclarativeOauth2Authenticator(AbstractOauth2Authenticator, DeclarativeAut
             if self.token_expiry_date
             else pendulum.now().subtract(days=1)  # type: ignore # substract does not have type hints
         )
-        self._access_token: Optional[str] = None  # access_token is initialized by a setter
+        if self.access_token_value is not None:
+            self._access_token_value = InterpolatedString.create(
+                self.access_token_value, parameters=parameters
+            ).eval(self.config)
+        else:
+            self._access_token_value = None
+
+        self._access_token: Optional[str] = (
+            self._access_token_value if self.access_token_value else None
+        )
 
         if self.get_grant_type() == "refresh_token" and self._refresh_token is None:
             raise ValueError(
                 "OAuthAuthenticator needs a refresh_token parameter if grant_type is set to `refresh_token`"
             )
 
-    def get_token_refresh_endpoint(self) -> str:
-        refresh_token: str = self._token_refresh_endpoint.eval(self.config)
-        if not refresh_token:
-            raise ValueError(
-                "OAuthAuthenticator was unable to evaluate token_refresh_endpoint parameter"
-            )
-        return refresh_token
+    def get_token_refresh_endpoint(self) -> Optional[str]:
+        if self._token_refresh_endpoint is not None:
+            refresh_token_endpoint: str = self._token_refresh_endpoint.eval(self.config)
+            if not refresh_token_endpoint:
+                raise ValueError(
+                    "OAuthAuthenticator was unable to evaluate token_refresh_endpoint parameter"
+                )
+            return refresh_token_endpoint
+        return None
 
     def get_client_id(self) -> str:
         client_id: str = self._client_id.eval(self.config)

airbyte_cdk/sources/declarative/concurrent_declarative_source.py

@@ -20,9 +20,6 @@ from airbyte_cdk.sources.declarative.extractors.record_filter import (
     ClientSideIncrementalRecordFilterDecorator,
 )
 from airbyte_cdk.sources.declarative.incremental.datetime_based_cursor import DatetimeBasedCursor
-from airbyte_cdk.sources.declarative.incremental.per_partition_with_global import (
-    PerPartitionWithGlobalCursor,
-)
 from airbyte_cdk.sources.declarative.interpolation import InterpolatedString
 from airbyte_cdk.sources.declarative.manifest_declarative_source import ManifestDeclarativeSource
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
@@ -303,60 +300,6 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
                             cursor=final_state_cursor,
                         )
                     )
-                elif (
-                    incremental_sync_component_definition
-                    and incremental_sync_component_definition.get("type", "")
-                    == DatetimeBasedCursorModel.__name__
-                    and self._stream_supports_concurrent_partition_processing(
-                        declarative_stream=declarative_stream
-                    )
-                    and hasattr(declarative_stream.retriever, "stream_slicer")
-                    and isinstance(
-                        declarative_stream.retriever.stream_slicer, PerPartitionWithGlobalCursor
-                    )
-                ):
-                    stream_state = state_manager.get_stream_state(
-                        stream_name=declarative_stream.name, namespace=declarative_stream.namespace
-                    )
-                    partition_router = declarative_stream.retriever.stream_slicer._partition_router
-
-                    cursor = self._constructor.create_concurrent_cursor_from_perpartition_cursor(
-                        state_manager=state_manager,
-                        model_type=DatetimeBasedCursorModel,
-                        component_definition=incremental_sync_component_definition,
-                        stream_name=declarative_stream.name,
-                        stream_namespace=declarative_stream.namespace,
-                        config=config or {},
-                        stream_state=stream_state,
-                        partition_router=partition_router,
-                    )
-
-                    partition_generator = StreamSlicerPartitionGenerator(
-                        DeclarativePartitionFactory(
-                            declarative_stream.name,
-                            declarative_stream.get_json_schema(),
-                            self._retriever_factory(
-                                name_to_stream_mapping[declarative_stream.name],
-                                config,
-                                stream_state,
-                            ),
-                            self.message_repository,
-                        ),
-                        cursor,
-                    )
-
-                    concurrent_streams.append(
-                        DefaultStream(
-                            partition_generator=partition_generator,
-                            name=declarative_stream.name,
-                            json_schema=declarative_stream.get_json_schema(),
-                            availability_strategy=AlwaysAvailableAvailabilityStrategy(),
-                            primary_key=get_primary_key_from_stream(declarative_stream.primary_key),
-                            cursor_field=cursor.cursor_field.cursor_field_key,
-                            logger=self.logger,
-                            cursor=cursor,
-                        )
-                    )
                 else:
                     synchronous_streams.append(declarative_stream)
             else:

airbyte_cdk/sources/declarative/declarative_component_schema.yaml

@@ -1021,7 +1021,6 @@ definitions:
       - type
       - client_id
       - client_secret
-      - token_refresh_endpoint
     properties:
       type:
         type: string
@@ -1060,6 +1059,12 @@ definitions:
         default: "access_token"
         examples:
           - access_token
+      access_token_value:
+        title: Access Token Value
+        description: The value of the access_token to bypass the token refreshing using `refresh_token`.
+        type: string
+        examples:
+          - secret_access_token_value
       expires_in_name:
         title: Token Expiry Property Name
         description: The name of the property which contains the expiry date in the response from the token refresh endpoint.
@@ -1235,6 +1240,7 @@ definitions:
             - "$ref": "#/definitions/RemoveFields"
             - "$ref": "#/definitions/KeysToLower"
             - "$ref": "#/definitions/KeysToSnakeCase"
+            - "$ref": "#/definitions/FlattenFields"
       state_migrations:
         title: State Migrations
         description: Array of state migrations to be applied on the input state
@@ -1767,6 +1773,18 @@ definitions:
           - "$ref": "#/definitions/AsyncRetriever"
           - "$ref": "#/definitions/CustomRetriever"
           - "$ref": "#/definitions/SimpleRetriever"
+      schema_transformations:
+        title: Schema Transformations
+        description: A list of transformations to be applied to the schema.
+        type: array
+        items:
+          anyOf:
+            - "$ref": "#/definitions/AddFields"
+            - "$ref": "#/definitions/CustomTransformation"
+            - "$ref": "#/definitions/RemoveFields"
+            - "$ref": "#/definitions/KeysToLower"
+            - "$ref": "#/definitions/KeysToSnakeCase"
+            - "$ref": "#/definitions/FlattenFields"
       schema_type_identifier:
         "$ref": "#/definitions/SchemaTypeIdentifier"
       $parameters:
@@ -2748,9 +2766,64 @@ definitions:
           - "$ref": "#/definitions/IterableDecoder"
           - "$ref": "#/definitions/XmlDecoder"
           - "$ref": "#/definitions/GzipJsonDecoder"
+          - "$ref": "#/definitions/CompositeRawDecoder"
       $parameters:
         type: object
         additionalProperties: true
+  CompositeRawDecoder:
+    description: "(This is experimental, use at your own risk)"
+    type: object
+    required:
+      - type
+      - parser
+    properties:
+      type:
+        type: string
+        enum: [CompositeRawDecoder]
+      parser:
+        anyOf:
+          - "$ref": "#/definitions/GzipParser"
+          - "$ref": "#/definitions/JsonLineParser"
+          - "$ref": "#/definitions/CsvParser"
+  #    PARSERS
+  GzipParser:
+    type: object
+    required:
+      - type
+      - inner_parser
+    properties:
+      type:
+        type: string
+        enum: [GzipParser]
+      inner_parser:
+        anyOf:
+          - "$ref": "#/definitions/JsonLineParser"
+          - "$ref": "#/definitions/CsvParser"
+  JsonLineParser:
+    type: object
+    required:
+      - type
+    properties:
+      type:
+        type: string
+        enum: [JsonLineParser]
+      encoding:
+        type: string
+        default: utf-8
+  CsvParser:
+    type: object
+    required:
+      - type
+    properties:
+      type:
+        type: string
+        enum: [CsvParser]
+      encoding:
+        type: string
+        default: utf-8
+      delimiter:
+        type: string
+        default: ","
   AsyncJobStatusMap:
     description: Matches the api job status to Async Job Status.
     type: object
@@ -3036,6 +3109,7 @@ definitions:
         interpolation_context:
           - config
           - components_values
+          - stream_slice
           - stream_template_config
         examples:
           - ["data"]
@@ -3052,10 +3126,13 @@ definitions:
           - config
           - stream_template_config
           - components_values
+          - stream_slice
         examples:
           - "{{ components_values['updates'] }}"
           - "{{ components_values['MetaData']['LastUpdatedTime'] }}"
           - "{{ config['segment_id'] }}"
+          - "{{ stream_slice['parent_id'] }}"
+          - "{{ stream_slice['extra_fields']['name'] }}"
       value_type:
         title: Value Type
         description: The expected data type of the value. If omitted, the type will be inferred from the value provided.

airbyte_cdk/sources/declarative/decoders/__init__.py

@@ -2,10 +2,28 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
 
+from airbyte_cdk.sources.declarative.decoders.composite_raw_decoder import CompositeRawDecoder
 from airbyte_cdk.sources.declarative.decoders.decoder import Decoder
-from airbyte_cdk.sources.declarative.decoders.json_decoder import JsonDecoder, JsonlDecoder, IterableDecoder, GzipJsonDecoder
+from airbyte_cdk.sources.declarative.decoders.json_decoder import (
+    GzipJsonDecoder,
+    IterableDecoder,
+    JsonDecoder,
+    JsonlDecoder,
+)
 from airbyte_cdk.sources.declarative.decoders.noop_decoder import NoopDecoder
-from airbyte_cdk.sources.declarative.decoders.pagination_decoder_decorator import PaginationDecoderDecorator
+from airbyte_cdk.sources.declarative.decoders.pagination_decoder_decorator import (
+    PaginationDecoderDecorator,
+)
 from airbyte_cdk.sources.declarative.decoders.xml_decoder import XmlDecoder
 
-__all__ = ["Decoder", "JsonDecoder", "JsonlDecoder", "IterableDecoder", "GzipJsonDecoder", "NoopDecoder", "PaginationDecoderDecorator", "XmlDecoder"]
+__all__ = [
+    "Decoder",
+    "CompositeRawDecoder",
+    "JsonDecoder",
+    "JsonlDecoder",
+    "IterableDecoder",
+    "GzipJsonDecoder",
+    "NoopDecoder",
+    "PaginationDecoderDecorator",
+    "XmlDecoder",
+]

airbyte_cdk/sources/declarative/decoders/composite_raw_decoder.py

@@ -0,0 +1,97 @@
+import csv
+import gzip
+import json
+import logging
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from io import BufferedIOBase, TextIOWrapper
+from typing import Any, Generator, MutableMapping, Optional
+
+import requests
+
+from airbyte_cdk.sources.declarative.decoders.decoder import Decoder
+
+logger = logging.getLogger("airbyte")
+
+
+@dataclass
+class Parser(ABC):
+    @abstractmethod
+    def parse(
+        self,
+        data: BufferedIOBase,
+    ) -> Generator[MutableMapping[str, Any], None, None]:
+        """
+        Parse data and yield dictionaries.
+        """
+        pass
+
+
+@dataclass
+class GzipParser(Parser):
+    inner_parser: Parser
+
+    def parse(
+        self,
+        data: BufferedIOBase,
+    ) -> Generator[MutableMapping[str, Any], None, None]:
+        """
+        Decompress gzipped bytes and pass decompressed data to the inner parser.
+        """
+        with gzip.GzipFile(fileobj=data, mode="rb") as gzipobj:
+            yield from self.inner_parser.parse(gzipobj)
+
+
+@dataclass
+class JsonLineParser(Parser):
+    encoding: Optional[str] = "utf-8"
+
+    def parse(
+        self,
+        data: BufferedIOBase,
+    ) -> Generator[MutableMapping[str, Any], None, None]:
+        for line in data:
+            try:
+                yield json.loads(line.decode(encoding=self.encoding or "utf-8"))
+            except json.JSONDecodeError as e:
+                logger.warning(f"Cannot decode/parse line {line!r} as JSON, error: {e}")
+
+
+@dataclass
+class CsvParser(Parser):
+    # TODO: migrate implementation to re-use file-base classes
+    encoding: Optional[str] = "utf-8"
+    delimiter: Optional[str] = ","
+
+    def parse(
+        self,
+        data: BufferedIOBase,
+    ) -> Generator[MutableMapping[str, Any], None, None]:
+        """
+        Parse CSV data from decompressed bytes.
+        """
+        text_data = TextIOWrapper(data, encoding=self.encoding)  # type: ignore
+        reader = csv.DictReader(text_data, delimiter=self.delimiter or ",")
+        yield from reader
+
+
+@dataclass
+class CompositeRawDecoder(Decoder):
+    """
+    Decoder strategy to transform a requests.Response into a Generator[MutableMapping[str, Any], None, None]
+    passed response.raw to parser(s).
+    Note: response.raw is not decoded/decompressed by default.
+    parsers should be instantiated recursively.
+    Example:
+    composite_raw_decoder = CompositeRawDecoder(parser=GzipParser(inner_parser=JsonLineParser(encoding="iso-8859-1")))
+    """
+
+    parser: Parser
+
+    def is_stream_response(self) -> bool:
+        return True
+
+    def decode(
+        self, response: requests.Response
+    ) -> Generator[MutableMapping[str, Any], None, None]:
+        yield from self.parser.parse(data=response.raw)  # type: ignore[arg-type]