airbyte-cdk 6.5.3rc2__py3-none-any.whl → 6.6.0__py3-none-any.whl

airbyte_cdk/sources/declarative/checks/connection_checker.py

@@ -15,7 +15,9 @@ class ConnectionChecker(ABC):
     """
 
     @abstractmethod
-    def check_connection(self, source: AbstractSource, logger: logging.Logger, config: Mapping[str, Any]) -> Tuple[bool, Any]:
+    def check_connection(
+        self, source: AbstractSource, logger: logging.Logger, config: Mapping[str, Any]
+    ) -> Tuple[bool, Any]:
         """
         Tests if the input configuration can be used to successfully connect to the integration e.g: if a provided Stripe API token can be used to connect
         to the Stripe API.

airbyte_cdk/sources/declarative/concurrency_level/concurrency_level.py

@@ -28,15 +28,23 @@ class ConcurrencyLevel:
         if isinstance(self.default_concurrency, int):
             self._default_concurrency: Union[int, InterpolatedString] = self.default_concurrency
         elif "config" in self.default_concurrency and not self.max_concurrency:
-            raise ValueError("ConcurrencyLevel requires that max_concurrency be defined if the default_concurrency can be used-specified")
+            raise ValueError(
+                "ConcurrencyLevel requires that max_concurrency be defined if the default_concurrency can be used-specified"
+            )
         else:
-            self._default_concurrency = InterpolatedString.create(self.default_concurrency, parameters=parameters)
+            self._default_concurrency = InterpolatedString.create(
+                self.default_concurrency, parameters=parameters
+            )
 
     def get_concurrency_level(self) -> int:
         if isinstance(self._default_concurrency, InterpolatedString):
             evaluated_default_concurrency = self._default_concurrency.eval(config=self.config)
             if not isinstance(evaluated_default_concurrency, int):
                 raise ValueError("default_concurrency did not evaluate to an integer")
-            return min(evaluated_default_concurrency, self.max_concurrency) if self.max_concurrency else evaluated_default_concurrency
+            return (
+                min(evaluated_default_concurrency, self.max_concurrency)
+                if self.max_concurrency
+                else evaluated_default_concurrency
+            )
         else:
             return self._default_concurrency

airbyte_cdk/sources/declarative/concurrent_declarative_source.py

@@ -3,20 +3,34 @@
 #
 
 import logging
-from typing import Any, Callable, Generic, Iterator, List, Mapping, Optional, Tuple, Union
-
-from airbyte_cdk.models import AirbyteCatalog, AirbyteMessage, AirbyteStateMessage, ConfiguredAirbyteCatalog
+from typing import Any, Generic, Iterator, List, Mapping, Optional, Tuple, Union
+
+from airbyte_cdk.models import (
+    AirbyteCatalog,
+    AirbyteMessage,
+    AirbyteStateMessage,
+    ConfiguredAirbyteCatalog,
+)
 from airbyte_cdk.sources.concurrent_source.concurrent_source import ConcurrentSource
 from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager
 from airbyte_cdk.sources.declarative.concurrency_level import ConcurrencyLevel
 from airbyte_cdk.sources.declarative.declarative_stream import DeclarativeStream
 from airbyte_cdk.sources.declarative.extractors import RecordSelector
+from airbyte_cdk.sources.declarative.extractors.record_filter import (
+    ClientSideIncrementalRecordFilterDecorator,
+)
 from airbyte_cdk.sources.declarative.incremental.datetime_based_cursor import DatetimeBasedCursor
 from airbyte_cdk.sources.declarative.interpolation import InterpolatedString
 from airbyte_cdk.sources.declarative.manifest_declarative_source import ManifestDeclarativeSource
-from airbyte_cdk.sources.declarative.models.declarative_component_schema import ConcurrencyLevel as ConcurrencyLevelModel
-from airbyte_cdk.sources.declarative.models.declarative_component_schema import DatetimeBasedCursor as DatetimeBasedCursorModel
-from airbyte_cdk.sources.declarative.parsers.model_to_component_factory import ModelToComponentFactory
+from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
+    ConcurrencyLevel as ConcurrencyLevelModel,
+)
+from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
+    DatetimeBasedCursor as DatetimeBasedCursorModel,
+)
+from airbyte_cdk.sources.declarative.parsers.model_to_component_factory import (
+    ModelToComponentFactory,
+)
 from airbyte_cdk.sources.declarative.requesters import HttpRequester
 from airbyte_cdk.sources.declarative.retrievers import SimpleRetriever
 from airbyte_cdk.sources.declarative.transformations.add_fields import AddFields
@@ -25,13 +39,14 @@ from airbyte_cdk.sources.source import TState
 from airbyte_cdk.sources.streams import Stream
 from airbyte_cdk.sources.streams.concurrent.abstract_stream import AbstractStream
 from airbyte_cdk.sources.streams.concurrent.adapters import CursorPartitionGenerator
-from airbyte_cdk.sources.streams.concurrent.availability_strategy import AlwaysAvailableAvailabilityStrategy
+from airbyte_cdk.sources.streams.concurrent.availability_strategy import (
+    AlwaysAvailableAvailabilityStrategy,
+)
 from airbyte_cdk.sources.streams.concurrent.default_stream import DefaultStream
 from airbyte_cdk.sources.streams.concurrent.helpers import get_primary_key_from_stream
 
 
 class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
-
     # By default, we defer to a value of 1 which represents running a connector using the Concurrent CDK engine on only one thread.
     SINGLE_THREADED_CONCURRENCY_LEVEL = 1
 
@@ -63,7 +78,9 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
         # any other arguments, but the existing entrypoint.py isn't designed to support this. Just noting this
         # for our future improvements to the CDK.
         if config:
-            self._concurrent_streams, self._synchronous_streams = self._group_streams(config=config or {})
+            self._concurrent_streams, self._synchronous_streams = self._group_streams(
+                config=config or {}
+            )
         else:
             self._concurrent_streams = None
             self._synchronous_streams = None
@@ -71,10 +88,14 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
         concurrency_level_from_manifest = self._source_config.get("concurrency_level")
         if concurrency_level_from_manifest:
             concurrency_level_component = self._constructor.create_component(
-                model_type=ConcurrencyLevelModel, component_definition=concurrency_level_from_manifest, config=config or {}
+                model_type=ConcurrencyLevelModel,
+                component_definition=concurrency_level_from_manifest,
+                config=config or {},
             )
             if not isinstance(concurrency_level_component, ConcurrencyLevel):
-                raise ValueError(f"Expected to generate a ConcurrencyLevel component, but received {concurrency_level_component.__class__}")
+                raise ValueError(
+                    f"Expected to generate a ConcurrencyLevel component, but received {concurrency_level_component.__class__}"
+                )
 
             concurrency_level = concurrency_level_component.get_concurrency_level()
             initial_number_of_partitions_to_generate = max(
@@ -99,13 +120,16 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
         catalog: ConfiguredAirbyteCatalog,
         state: Optional[Union[List[AirbyteStateMessage]]] = None,
     ) -> Iterator[AirbyteMessage]:
-
         # ConcurrentReadProcessor pops streams that are finished being read so before syncing, the names of the concurrent
         # streams must be saved so that they can be removed from the catalog before starting synchronous streams
         if self._concurrent_streams:
-            concurrent_stream_names = set([concurrent_stream.name for concurrent_stream in self._concurrent_streams])
+            concurrent_stream_names = set(
+                [concurrent_stream.name for concurrent_stream in self._concurrent_streams]
+            )
 
-            selected_concurrent_streams = self._select_streams(streams=self._concurrent_streams, configured_catalog=catalog)
+            selected_concurrent_streams = self._select_streams(
+                streams=self._concurrent_streams, configured_catalog=catalog
+            )
             # It would appear that passing in an empty set of streams causes an infinite loop in ConcurrentReadProcessor.
             # This is also evident in concurrent_source_adapter.py so I'll leave this out of scope to fix for now
             if selected_concurrent_streams:
@@ -125,7 +149,11 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
     def discover(self, logger: logging.Logger, config: Mapping[str, Any]) -> AirbyteCatalog:
         concurrent_streams = self._concurrent_streams or []
         synchronous_streams = self._synchronous_streams or []
-        return AirbyteCatalog(streams=[stream.as_airbyte_stream() for stream in concurrent_streams + synchronous_streams])
+        return AirbyteCatalog(
+            streams=[
+                stream.as_airbyte_stream() for stream in concurrent_streams + synchronous_streams
+            ]
+        )
 
     def streams(self, config: Mapping[str, Any]) -> List[Stream]:
         """
@@ -138,25 +166,34 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
         """
         return super().streams(config)
 
-    def _group_streams(self, config: Mapping[str, Any]) -> Tuple[List[AbstractStream], List[Stream]]:
+    def _group_streams(
+        self, config: Mapping[str, Any]
+    ) -> Tuple[List[AbstractStream], List[Stream]]:
         concurrent_streams: List[AbstractStream] = []
         synchronous_streams: List[Stream] = []
 
         state_manager = ConnectorStateManager(state=self._state)  # type: ignore  # state is always in the form of List[AirbyteStateMessage]. The ConnectorStateManager should use generics, but this can be done later
 
-        name_to_stream_mapping = {stream["name"]: stream for stream in self.resolved_manifest["streams"]}
+        name_to_stream_mapping = {
+            stream["name"]: stream for stream in self.resolved_manifest["streams"]
+        }
 
         for declarative_stream in self.streams(config=config):
             # Some low-code sources use a combination of DeclarativeStream and regular Python streams. We can't inspect
             # these legacy Python streams the way we do low-code streams to determine if they are concurrent compatible,
             # so we need to treat them as synchronous
             if isinstance(declarative_stream, DeclarativeStream):
-                datetime_based_cursor_component_definition = name_to_stream_mapping[declarative_stream.name].get("incremental_sync")
+                datetime_based_cursor_component_definition = name_to_stream_mapping[
+                    declarative_stream.name
+                ].get("incremental_sync")
 
                 if (
                     datetime_based_cursor_component_definition
-                    and datetime_based_cursor_component_definition.get("type", "") == DatetimeBasedCursorModel.__name__
-                    and self._stream_supports_concurrent_partition_processing(declarative_stream=declarative_stream)
+                    and datetime_based_cursor_component_definition.get("type", "")
+                    == DatetimeBasedCursorModel.__name__
+                    and self._stream_supports_concurrent_partition_processing(
+                        declarative_stream=declarative_stream
+                    )
                     and hasattr(declarative_stream.retriever, "stream_slicer")
                     and isinstance(declarative_stream.retriever.stream_slicer, DatetimeBasedCursor)
                 ):
@@ -164,30 +201,38 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
                         stream_name=declarative_stream.name, namespace=declarative_stream.namespace
                     )
 
-                    cursor, connector_state_converter = self._constructor.create_concurrent_cursor_from_datetime_based_cursor(
-                        state_manager=state_manager,
-                        model_type=DatetimeBasedCursorModel,
-                        component_definition=datetime_based_cursor_component_definition,
-                        stream_name=declarative_stream.name,
-                        stream_namespace=declarative_stream.namespace,
-                        config=config or {},
-                        stream_state=stream_state,
+                    cursor, connector_state_converter = (
+                        self._constructor.create_concurrent_cursor_from_datetime_based_cursor(
+                            state_manager=state_manager,
+                            model_type=DatetimeBasedCursorModel,
+                            component_definition=datetime_based_cursor_component_definition,
+                            stream_name=declarative_stream.name,
+                            stream_namespace=declarative_stream.namespace,
+                            config=config or {},
+                            stream_state=stream_state,
+                        )
                     )
 
                     # This is an optimization so that we don't invoke any cursor or state management flows within the
                     # low-code framework because state management is handled through the ConcurrentCursor.
-                    if declarative_stream and declarative_stream.retriever and isinstance(declarative_stream.retriever, SimpleRetriever):
+                    if (
+                        declarative_stream
+                        and declarative_stream.retriever
+                        and isinstance(declarative_stream.retriever, SimpleRetriever)
+                    ):
                         # Also a temporary hack. In the legacy Stream implementation, as part of the read, set_initial_state() is
                         # called to instantiate incoming state on the cursor. Although we no longer rely on the legacy low-code cursor
                         # for concurrent checkpointing, low-code components like StopConditionPaginationStrategyDecorator and
                         # ClientSideIncrementalRecordFilterDecorator still rely on a DatetimeBasedCursor that is properly initialized
                         # with state.
                         if declarative_stream.retriever.cursor:
-                            declarative_stream.retriever.cursor.set_initial_state(stream_state=stream_state)
+                            declarative_stream.retriever.cursor.set_initial_state(
+                                stream_state=stream_state
+                            )
                         declarative_stream.retriever.cursor = None
 
                     partition_generator = CursorPartitionGenerator(
-                        stream_factory=self._new_stream_instance_factory(declarative_stream, config),
+                        stream=declarative_stream,
                         message_repository=self.message_repository,  # type: ignore  # message_repository is always instantiated with a value by factory
                         cursor=cursor,
                         connector_state_converter=connector_state_converter,
@@ -214,7 +259,9 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
 
         return concurrent_streams, synchronous_streams
 
-    def _stream_supports_concurrent_partition_processing(self, declarative_stream: DeclarativeStream) -> bool:
+    def _stream_supports_concurrent_partition_processing(
+        self, declarative_stream: DeclarativeStream
+    ) -> bool:
         """
         Many connectors make use of stream_state during interpolation on a per-partition basis under the assumption that
         state is updated sequentially. Because the concurrent CDK engine processes different partitions in parallel,
@@ -226,7 +273,9 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
         cdk-migrations.md for the full list of connectors.
         """
 
-        if isinstance(declarative_stream.retriever, SimpleRetriever) and isinstance(declarative_stream.retriever.requester, HttpRequester):
+        if isinstance(declarative_stream.retriever, SimpleRetriever) and isinstance(
+            declarative_stream.retriever.requester, HttpRequester
+        ):
             http_requester = declarative_stream.retriever.requester
             if "stream_state" in http_requester._path.string:
                 self.logger.warning(
@@ -243,14 +292,22 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
 
             record_selector = declarative_stream.retriever.record_selector
             if isinstance(record_selector, RecordSelector):
-                if record_selector.record_filter and "stream_state" in record_selector.record_filter.condition:
+                if (
+                    record_selector.record_filter
+                    and not isinstance(
+                        record_selector.record_filter, ClientSideIncrementalRecordFilterDecorator
+                    )
+                    and "stream_state" in record_selector.record_filter.condition
+                ):
                     self.logger.warning(
                         f"Low-code stream '{declarative_stream.name}' uses interpolation of stream_state in the RecordFilter which is not thread-safe. Defaulting to synchronous processing"
                     )
                     return False
 
                 for add_fields in [
-                    transformation for transformation in record_selector.transformations if isinstance(transformation, AddFields)
+                    transformation
+                    for transformation in record_selector.transformations
+                    if isinstance(transformation, AddFields)
                 ]:
                     for field in add_fields.fields:
                         if isinstance(field.value, str) and "stream_state" in field.value:
@@ -258,7 +315,10 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
                                 f"Low-code stream '{declarative_stream.name}' uses interpolation of stream_state in the AddFields which is not thread-safe. Defaulting to synchronous processing"
                             )
                             return False
-                        if isinstance(field.value, InterpolatedString) and "stream_state" in field.value.string:
+                        if (
+                            isinstance(field.value, InterpolatedString)
+                            and "stream_state" in field.value.string
+                        ):
                             self.logger.warning(
                                 f"Low-code stream '{declarative_stream.name}' uses interpolation of stream_state in the AddFields which is not thread-safe. Defaulting to synchronous processing"
                             )
@@ -266,7 +326,9 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
         return True
 
     @staticmethod
-    def _select_streams(streams: List[AbstractStream], configured_catalog: ConfiguredAirbyteCatalog) -> List[AbstractStream]:
+    def _select_streams(
+        streams: List[AbstractStream], configured_catalog: ConfiguredAirbyteCatalog
+    ) -> List[AbstractStream]:
         stream_name_to_instance: Mapping[str, AbstractStream] = {s.name: s for s in streams}
         abstract_streams: List[AbstractStream] = []
         for configured_stream in configured_catalog.streams:
@@ -281,16 +343,10 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
         catalog: ConfiguredAirbyteCatalog,
         concurrent_stream_names: set[str],
     ) -> ConfiguredAirbyteCatalog:
-        return ConfiguredAirbyteCatalog(streams=[stream for stream in catalog.streams if stream.stream.name not in concurrent_stream_names])
-
-    def _new_stream_instance_factory(self, stream_to_copy: Stream, config: Mapping[str, Any]) -> Callable[[], Stream]:
-        """
-        Some of the declarative components are stateful. Therefore, we create one stream per thread in order to avoid threads updating
-        the same field for a specific instance.
-        """
-        def _create_new_stream() -> Stream:
-            streams_with_same_name = list(filter(lambda stream: stream.name == stream_to_copy.name, self.streams(config)))
-            if len(streams_with_same_name) == 1:
-                return streams_with_same_name[0]
-            raise ValueError(f"Expected one stream with name `{stream_to_copy.name}` but got {len(streams_with_same_name)}")
-        return _create_new_stream
+        return ConfiguredAirbyteCatalog(
+            streams=[
+                stream
+                for stream in catalog.streams
+                if stream.stream.name not in concurrent_stream_names
+            ]
+        )

airbyte_cdk/sources/declarative/datetime/min_max_datetime.py

@@ -32,7 +32,7 @@ class MinMaxDatetime:
     parameters: InitVar[Mapping[str, Any]]
     # datetime_format is a unique case where we inherit it from the parent if it is not specified before using the default value
     # which is why we need dedicated getter/setter methods and private dataclass field
-    datetime_format: str = ""
+    datetime_format: str
     _datetime_format: str = field(init=False, repr=False, default="")
     min_datetime: Union[InterpolatedString, str] = ""
     max_datetime: Union[InterpolatedString, str] = ""
@@ -40,10 +40,20 @@ class MinMaxDatetime:
     def __post_init__(self, parameters: Mapping[str, Any]) -> None:
         self.datetime = InterpolatedString.create(self.datetime, parameters=parameters or {})
         self._parser = DatetimeParser()
-        self.min_datetime = InterpolatedString.create(self.min_datetime, parameters=parameters) if self.min_datetime else None  # type: ignore
-        self.max_datetime = InterpolatedString.create(self.max_datetime, parameters=parameters) if self.max_datetime else None  # type: ignore
+        self.min_datetime = (
+            InterpolatedString.create(self.min_datetime, parameters=parameters)
+            if self.min_datetime
+            else None
+        )  # type: ignore
+        self.max_datetime = (
+            InterpolatedString.create(self.max_datetime, parameters=parameters)
+            if self.max_datetime
+            else None
+        )  # type: ignore
 
-    def get_datetime(self, config: Mapping[str, Any], **additional_parameters: Mapping[str, Any]) -> dt.datetime:
+    def get_datetime(
+        self, config: Mapping[str, Any], **additional_parameters: Mapping[str, Any]
+    ) -> dt.datetime:
         """
         Evaluates and returns the datetime
         :param config: The user-provided configuration as specified by the source's spec
@@ -55,7 +65,9 @@ class MinMaxDatetime:
         if not datetime_format:
             datetime_format = "%Y-%m-%dT%H:%M:%S.%f%z"
 
-        time = self._parser.parse(str(self.datetime.eval(config, **additional_parameters)), datetime_format)  # type: ignore # datetime is always cast to an interpolated string
+        time = self._parser.parse(
+            str(self.datetime.eval(config, **additional_parameters)), datetime_format
+        )  # type: ignore # datetime is always cast to an interpolated string
 
         if self.min_datetime:
             min_time = str(self.min_datetime.eval(config, **additional_parameters))  # type: ignore # min_datetime is always cast to an interpolated string
@@ -93,6 +105,8 @@ class MinMaxDatetime:
         if isinstance(interpolated_string_or_min_max_datetime, InterpolatedString) or isinstance(
             interpolated_string_or_min_max_datetime, str
         ):
-            return MinMaxDatetime(datetime=interpolated_string_or_min_max_datetime, parameters=parameters)
+            return MinMaxDatetime(
+                datetime=interpolated_string_or_min_max_datetime, parameters=parameters
+            )
         else:
             return interpolated_string_or_min_max_datetime

airbyte_cdk/sources/declarative/declarative_component_schema.yaml

@@ -1750,6 +1750,45 @@ definitions:
       type:
         type: string
         enum: [XmlDecoder]
+  CustomDecoder:
+    title: Custom Decoder
+    description: Use this to implement custom decoder logic.
+    type: object
+    additionalProperties: true
+    required:
+      - type
+      - class_name
+    properties:
+      type:
+        type: string
+        enum: [CustomDecoder]
+      class_name:
+        title: Class Name
+        description: Fully-qualified name of the class that will be implementing the custom decoding. Has to be a sub class of Decoder. The format is `source_<name>.<package>.<class_name>`.
+        type: string
+        additionalProperties: true
+        examples:
+          - "source_amazon_ads.components.GzipJsonlDecoder"
+      $parameters:
+        type: object
+        additionalProperties: true
+  GzipJsonDecoder:
+    title: GzipJson Decoder
+    description: Use this if the response is Gzip compressed Json.
+    type: object
+    additionalProperties: true
+    required:
+      - type
+    properties:
+      type:
+        type: string
+        enum: [GzipJsonDecoder]
+      encoding:
+        type: string
+        default: utf-8
+      $parameters:
+        type: object
+        additionalProperties: true
   ListPartitionRouter:
     title: List Partition Router
     description: A Partition router that specifies a list of attributes where each attribute describes a portion of the complete data set for a stream. During a sync, each value is iterated over and can be used as input to outbound API requests.
@@ -2404,10 +2443,12 @@ definitions:
         title: Decoder
         description: Component decoding the response so records can be extracted.
         anyOf:
+          - "$ref": "#/definitions/CustomDecoder"
           - "$ref": "#/definitions/JsonDecoder"
           - "$ref": "#/definitions/JsonlDecoder"
           - "$ref": "#/definitions/IterableDecoder"
           - "$ref": "#/definitions/XmlDecoder"
+          - "$ref": "#/definitions/GzipJsonDecoder"
       $parameters:
         type: object
         additionalProperties: true
@@ -2520,10 +2561,12 @@ definitions:
         title: Decoder
         description: Component decoding the response so records can be extracted.
         anyOf:
+          - "$ref": "#/definitions/CustomDecoder"
           - "$ref": "#/definitions/JsonDecoder"
           - "$ref": "#/definitions/JsonlDecoder"
           - "$ref": "#/definitions/IterableDecoder"
           - "$ref": "#/definitions/XmlDecoder"
+          - "$ref": "#/definitions/GzipJsonDecoder"
       $parameters:
         type: object
         additionalProperties: true

airbyte_cdk/sources/declarative/declarative_source.py

@@ -20,7 +20,9 @@ class DeclarativeSource(AbstractSource):
     def connection_checker(self) -> ConnectionChecker:
         """Returns the ConnectionChecker to use for the `check` operation"""
 
-    def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> Tuple[bool, Any]:
+    def check_connection(
+        self, logger: logging.Logger, config: Mapping[str, Any]
+    ) -> Tuple[bool, Any]:
         """
         :param logger: The source logger
         :param config: The user-provided configuration as specified by the source's spec.

airbyte_cdk/sources/declarative/declarative_stream.py

@@ -6,14 +6,23 @@ from dataclasses import InitVar, dataclass, field
 from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Union
 
 from airbyte_cdk.models import SyncMode
-from airbyte_cdk.sources.declarative.incremental import GlobalSubstreamCursor, PerPartitionCursor, PerPartitionWithGlobalCursor
+from airbyte_cdk.sources.declarative.incremental import (
+    GlobalSubstreamCursor,
+    PerPartitionCursor,
+    PerPartitionWithGlobalCursor,
+)
 from airbyte_cdk.sources.declarative.interpolation import InterpolatedString
 from airbyte_cdk.sources.declarative.migrations.state_migration import StateMigration
 from airbyte_cdk.sources.declarative.retrievers import SimpleRetriever
 from airbyte_cdk.sources.declarative.retrievers.retriever import Retriever
 from airbyte_cdk.sources.declarative.schema import DefaultSchemaLoader
 from airbyte_cdk.sources.declarative.schema.schema_loader import SchemaLoader
-from airbyte_cdk.sources.streams.checkpoint import CheckpointMode, CheckpointReader, Cursor, CursorBasedCheckpointReader
+from airbyte_cdk.sources.streams.checkpoint import (
+    CheckpointMode,
+    CheckpointReader,
+    Cursor,
+    CursorBasedCheckpointReader,
+)
 from airbyte_cdk.sources.streams.core import Stream
 from airbyte_cdk.sources.types import Config, StreamSlice
 
@@ -50,7 +59,11 @@ class DeclarativeStream(Stream):
             if isinstance(self.stream_cursor_field, str)
             else self.stream_cursor_field
         )
-        self._schema_loader = self.schema_loader if self.schema_loader else DefaultSchemaLoader(config=self.config, parameters=parameters)
+        self._schema_loader = (
+            self.schema_loader
+            if self.schema_loader
+            else DefaultSchemaLoader(config=self.config, parameters=parameters)
+        )
 
     @property  # type: ignore
     def primary_key(self) -> Optional[Union[str, List[str], List[List[str]]]]:
@@ -133,7 +146,9 @@ class DeclarativeStream(Stream):
             # empty slice which seems to make sense.
             stream_slice = StreamSlice(partition={}, cursor_slice={})
         if not isinstance(stream_slice, StreamSlice):
-            raise ValueError(f"DeclarativeStream does not support stream_slices that are not StreamSlice. Got {stream_slice}")
+            raise ValueError(
+                f"DeclarativeStream does not support stream_slices that are not StreamSlice. Got {stream_slice}"
+            )
         yield from self.retriever.read_records(self.get_json_schema(), stream_slice)  # type: ignore # records are of the correct type
 
     def get_json_schema(self) -> Mapping[str, Any]:  # type: ignore
@@ -146,7 +161,11 @@ class DeclarativeStream(Stream):
         return self._schema_loader.get_json_schema()
 
     def stream_slices(
-        self, *, sync_mode: SyncMode, cursor_field: Optional[List[str]] = None, stream_state: Optional[Mapping[str, Any]] = None
+        self,
+        *,
+        sync_mode: SyncMode,
+        cursor_field: Optional[List[str]] = None,
+        stream_state: Optional[Mapping[str, Any]] = None,
     ) -> Iterable[Optional[StreamSlice]]:
         """
         Override to define the slices for this stream. See the stream slicing section of the docs for more information.
@@ -200,7 +219,9 @@ class DeclarativeStream(Stream):
         cursor = self.get_cursor()
         checkpoint_mode = self._checkpoint_mode
 
-        if isinstance(cursor, (GlobalSubstreamCursor, PerPartitionCursor, PerPartitionWithGlobalCursor)):
+        if isinstance(
+            cursor, (GlobalSubstreamCursor, PerPartitionCursor, PerPartitionWithGlobalCursor)
+        ):
             self.has_multiple_slices = True
             return CursorBasedCheckpointReader(
                 stream_slices=mappings_or_slices,

airbyte_cdk/sources/declarative/decoders/__init__.py

@@ -3,9 +3,9 @@
 #
 
 from airbyte_cdk.sources.declarative.decoders.decoder import Decoder
-from airbyte_cdk.sources.declarative.decoders.json_decoder import JsonDecoder, JsonlDecoder, IterableDecoder
+from airbyte_cdk.sources.declarative.decoders.json_decoder import JsonDecoder, JsonlDecoder, IterableDecoder, GzipJsonDecoder
 from airbyte_cdk.sources.declarative.decoders.noop_decoder import NoopDecoder
 from airbyte_cdk.sources.declarative.decoders.pagination_decoder_decorator import PaginationDecoderDecorator
 from airbyte_cdk.sources.declarative.decoders.xml_decoder import XmlDecoder
 
-__all__ = ["Decoder", "JsonDecoder", "JsonlDecoder", "IterableDecoder", "NoopDecoder", "PaginationDecoderDecorator", "XmlDecoder"]
+__all__ = ["Decoder", "JsonDecoder", "JsonlDecoder", "IterableDecoder", "GzipJsonDecoder", "NoopDecoder", "PaginationDecoderDecorator", "XmlDecoder"]

airbyte_cdk/sources/declarative/decoders/decoder.py

@@ -22,7 +22,9 @@ class Decoder:
         """
 
     @abstractmethod
-    def decode(self, response: requests.Response) -> Generator[MutableMapping[str, Any], None, None]:
+    def decode(
+        self, response: requests.Response
+    ) -> Generator[MutableMapping[str, Any], None, None]:
         """
         Decodes a requests.Response into a Mapping[str, Any] or an array
         :param response: the response to decode