airbyte-cdk 6.12.0rc1__py3-none-any.whl → 6.12.1.dev0__py3-none-any.whl

airbyte_cdk/sources/declarative/declarative_component_schema.yaml

@@ -1897,56 +1897,6 @@ definitions:
       $parameters:
         type: object
         additionalProperties: true
-  ZipfileDecoder:
-    title: Zipfile Decoder
-    description: Decoder for response data that is returned as zipfile(s).
-    type: object
-    additionalProperties: true
-    required:
-      - type
-    properties:
-      type:
-        type: string
-        enum: [ZipfileDecoder]
-      parser:
-        title: Parser
-        description: Parser to parse the decompressed data from the zipfile(s).
-        anyOf:
-          - "$ref": "#/definitions/JsonParser"
-          - "$ref": "#/definitions/CustomParser"
-  JsonParser:
-    title: JsonParser
-    description: Parser used for parsing str, bytes, or bytearray data and returning data in a dictionary format.
-    type: object
-    additionalProperties: true
-    required:
-      - type
-    properties:
-      type:
-        type: string
-        enum: [JsonParser]
-  CustomParser:
-    title: Custom Parser
-    description: Use this to implement custom parser logic.
-    type: object
-    additionalProperties: true
-    required:
-      - type
-      - class_name
-    properties:
-      type:
-        type: string
-        enum: [CustomParser]
-      class_name:
-        title: Class Name
-        description: Fully-qualified name of the class that will be implementing the custom decoding. Has to be a sub class of Parser. The format is `source_<name>.<package>.<class_name>`.
-        type: string
-        additionalProperties: true
-        examples:
-          - "source_rivendell.components.ElvishParser"
-      $parameters:
-        type: object
-        additionalProperties: true
   ListPartitionRouter:
     title: List Partition Router
     description: A Partition router that specifies a list of attributes where each attribute describes a portion of the complete data set for a stream. During a sync, each value is iterated over and can be used as input to outbound API requests.

airbyte_cdk/sources/declarative/decoders/__init__.py

@@ -7,6 +7,5 @@ from airbyte_cdk.sources.declarative.decoders.json_decoder import JsonDecoder, J
 from airbyte_cdk.sources.declarative.decoders.noop_decoder import NoopDecoder
 from airbyte_cdk.sources.declarative.decoders.pagination_decoder_decorator import PaginationDecoderDecorator
 from airbyte_cdk.sources.declarative.decoders.xml_decoder import XmlDecoder
-from airbyte_cdk.sources.declarative.decoders.zipfile_decoder import ZipfileDecoder
 
-__all__ = ["Decoder", "JsonDecoder", "JsonlDecoder", "IterableDecoder", "GzipJsonDecoder", "NoopDecoder", "PaginationDecoderDecorator", "XmlDecoder", "ZipfileDecoder"]
+__all__ = ["Decoder", "JsonDecoder", "JsonlDecoder", "IterableDecoder", "GzipJsonDecoder", "NoopDecoder", "PaginationDecoderDecorator", "XmlDecoder"]

airbyte_cdk/sources/declarative/models/declarative_component_schema.py

@@ -741,27 +741,6 @@ class GzipJsonDecoder(BaseModel):
     parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
 
 
-class JsonParser(BaseModel):
-    class Config:
-        extra = Extra.allow
-
-    type: Literal["JsonParser"]
-
-
-class CustomParser(BaseModel):
-    class Config:
-        extra = Extra.allow
-
-    type: Literal["CustomParser"]
-    class_name: str = Field(
-        ...,
-        description="Fully-qualified name of the class that will be implementing the custom decoding. Has to be a sub class of Parser. The format is `source_<name>.<package>.<class_name>`.",
-        examples=["source_rivendell.components.ElvishParser"],
-        title="Class Name",
-    )
-    parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
-
-
 class MinMaxDatetime(BaseModel):
     type: Literal["MinMaxDatetime"]
     datetime: str = Field(
@@ -1489,18 +1468,6 @@ class SessionTokenRequestApiKeyAuthenticator(BaseModel):
     )
 
 
-class ZipfileDecoder(BaseModel):
-    class Config:
-        extra = Extra.allow
-
-    type: Literal["ZipfileDecoder"]
-    parser: Optional[Union[JsonParser, CustomParser]] = Field(
-        None,
-        description="Parser to parse the decompressed data from the zipfile(s).",
-        title="Parser",
-    )
-
-
 class ListPartitionRouter(BaseModel):
     type: Literal["ListPartitionRouter"]
     cursor_field: str = Field(

airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py

@@ -66,9 +66,7 @@ from airbyte_cdk.sources.declarative.decoders import (
     JsonlDecoder,
     PaginationDecoderDecorator,
     XmlDecoder,
-    ZipfileDecoder,
 )
-from airbyte_cdk.sources.declarative.decoders.parsers import JsonParser
 from airbyte_cdk.sources.declarative.extractors import (
     DpathExtractor,
     RecordFilter,
@@ -226,9 +224,6 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     JsonlDecoder as JsonlDecoderModel,
 )
-from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
-    JsonParser as JsonParserModel,
-)
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     JwtAuthenticator as JwtAuthenticatorModel,
 )
@@ -321,9 +316,6 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     XmlDecoder as XmlDecoderModel,
 )
-from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
-    ZipfileDecoder as ZipfileDecoderModel,
-)
 from airbyte_cdk.sources.declarative.partition_routers import (
     CartesianProductStreamSlicer,
     ListPartitionRouter,
@@ -478,7 +470,6 @@ class ModelToComponentFactory:
             InlineSchemaLoaderModel: self.create_inline_schema_loader,
             JsonDecoderModel: self.create_json_decoder,
             JsonlDecoderModel: self.create_jsonl_decoder,
-            JsonParserModel: self.create_json_parser,
             GzipJsonDecoderModel: self.create_gzipjson_decoder,
             KeysToLowerModel: self.create_keys_to_lower_transformation,
             IterableDecoderModel: self.create_iterable_decoder,
@@ -514,7 +505,6 @@ class ModelToComponentFactory:
             ConfigComponentsResolverModel: self.create_config_components_resolver,
             StreamConfigModel: self.create_stream_config,
             ComponentMappingDefinitionModel: self.create_components_mapping_definition,
-            ZipfileDecoderModel: self.create_zipfile_decoder,
         }
 
         # Needed for the case where we need to perform a second parse on the fields of a custom component
@@ -1692,20 +1682,6 @@ class ModelToComponentFactory:
     ) -> GzipJsonDecoder:
         return GzipJsonDecoder(parameters={}, encoding=model.encoding)
 
-    def create_zipfile_decoder(
-        self, model: ZipfileDecoderModel, config: Config, **kwargs: Any
-    ) -> ZipfileDecoder:
-        parser = (
-            self._create_component_from_model(model=model.parser, config=config)
-            if model.parser
-            else None
-        )
-        return ZipfileDecoder(parameters={}, parser=parser)
-
-    @staticmethod
-    def create_json_parser(model: JsonParserModel, config: Config, **kwargs: Any) -> JsonParser:
-        return JsonParser(parameters={})
-
     @staticmethod
     def create_json_file_schema_loader(
         model: JsonFileSchemaLoaderModel, config: Config, **kwargs: Any

airbyte_cdk/sources/streams/core.py

@@ -223,17 +223,17 @@ class Stream(ABC):
                     record_counter += 1
 
                     checkpoint_interval = self.state_checkpoint_interval
-                    checkpoint = checkpoint_reader.get_checkpoint()
                     if (
                         should_checkpoint
                         and checkpoint_interval
                         and record_counter % checkpoint_interval == 0
-                        and checkpoint is not None
                     ):
-                        airbyte_state_message = self._checkpoint_state(
-                            checkpoint, state_manager=state_manager
-                        )
-                        yield airbyte_state_message
+                        checkpoint = checkpoint_reader.get_checkpoint()
+                        if checkpoint:
+                            airbyte_state_message = self._checkpoint_state(
+                                checkpoint, state_manager=state_manager
+                            )
+                            yield airbyte_state_message
 
                     if internal_config.is_limit_reached(record_counter):
                         break

{airbyte_cdk-6.12.0rc1.dist-info → airbyte_cdk-6.12.1.dev0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 6.12.0rc1
+Version: 6.12.1.dev0
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://airbyte.com
 License: MIT

{airbyte_cdk-6.12.0rc1.dist-info → airbyte_cdk-6.12.1.dev0.dist-info}/RECORD

@@ -66,18 +66,15 @@ airbyte_cdk/sources/declarative/concurrent_declarative_source.py,sha256=PxP4p268
 airbyte_cdk/sources/declarative/datetime/__init__.py,sha256=l9LG7Qm6e5r_qgqfVKnx3mXYtg1I9MmMjomVIPfU4XA,177
 airbyte_cdk/sources/declarative/datetime/datetime_parser.py,sha256=SX9JjdesN1edN2WVUVMzU_ptqp2QB1OnsnjZ4mwcX7w,2579
 airbyte_cdk/sources/declarative/datetime/min_max_datetime.py,sha256=0BHBtDNQZfvwM45-tY5pNlTcKAFSGGNxemoi0Jic-0E,5785
-airbyte_cdk/sources/declarative/declarative_component_schema.yaml,sha256=aW6LC_YuqlqqCiRgBKm7ecNsF_0iPi1NF8sDmiq6wsA,130106
+airbyte_cdk/sources/declarative/declarative_component_schema.yaml,sha256=bmuzrO0q4yeRXXkAMAFdzgtDx2gF8-k8ERq2MqVMgq4,128593
 airbyte_cdk/sources/declarative/declarative_source.py,sha256=nF7wBqFd3AQmEKAm4CnIo29CJoQL562cJGSCeL8U8bA,1531
 airbyte_cdk/sources/declarative/declarative_stream.py,sha256=JRyNeOIpsFu4ztVZsN6sncqUEIqIE-bUkD2TPgbMgk0,10375
-airbyte_cdk/sources/declarative/decoders/__init__.py,sha256=q4XSJFXbVYgXoal910hrCL6t-EQLBtLUq4gvP1eNW9o,773
+airbyte_cdk/sources/declarative/decoders/__init__.py,sha256=hNlhaB5FjNC6IfJyglj5ZJWkYD2nEAukMDmzRz5PC6o,671
 airbyte_cdk/sources/declarative/decoders/decoder.py,sha256=sl-Gt8lXi7yD2Q-sD8je5QS2PbgrgsYjxRLWsay7DMc,826
 airbyte_cdk/sources/declarative/decoders/json_decoder.py,sha256=qdbjeR6RffKaah_iWvMsOcDolYuxJY5DaI3b9AMTZXg,3327
 airbyte_cdk/sources/declarative/decoders/noop_decoder.py,sha256=iZh0yKY_JzgBnJWiubEusf5c0o6Khd-8EWFWT-8EgFo,542
 airbyte_cdk/sources/declarative/decoders/pagination_decoder_decorator.py,sha256=ZVBZhAOl0I0MymXN5CKTC-kIXG4GuUQAEyn0XpUDuSE,1081
-airbyte_cdk/sources/declarative/decoders/parsers/__init__.py,sha256=3qvAJF3O2Ie0qK5ANgsEKR136t7HKk53rR_FyO8ihuY,186
-airbyte_cdk/sources/declarative/decoders/parsers/parsers.py,sha256=Xlz4xmRYAtMFqdvAMwEdTxh-90glKlijMEm6qLOyuAs,1257
 airbyte_cdk/sources/declarative/decoders/xml_decoder.py,sha256=EU-7t-5vIGRHZ14h-f0GUE4V5-eTM9Flux-A8xgI1Rc,3117
-airbyte_cdk/sources/declarative/decoders/zipfile_decoder.py,sha256=IPLCgYpab6og6j7BdvBuhn_7c1ZYyg8yu48DPkvA84c,1752
 airbyte_cdk/sources/declarative/exceptions.py,sha256=kTPUA4I2NV4J6HDz-mKPGMrfuc592akJnOyYx38l_QM,176
 airbyte_cdk/sources/declarative/extractors/__init__.py,sha256=YFuL4D4RuuB8E1DNSbJNIj0_HApOlyECoJ_s8DuJMeI,611
 airbyte_cdk/sources/declarative/extractors/dpath_extractor.py,sha256=wR4Ol4MG2lt5UlqXF5EU_k7qa5cN4_-luu3PJ1PlO3A,3131
@@ -107,12 +104,12 @@ airbyte_cdk/sources/declarative/migrations/__init__.py,sha256=47DEQpj8HBSa-_TImW
 airbyte_cdk/sources/declarative/migrations/legacy_to_per_partition_state_migration.py,sha256=iemy3fKLczcU0-Aor7tx5jcT6DRedKMqyK7kCOp01hg,3924
 airbyte_cdk/sources/declarative/migrations/state_migration.py,sha256=KWPjealMLKSMtajXgkdGgKg7EmTLR-CqqD7UIh0-eDU,794
 airbyte_cdk/sources/declarative/models/__init__.py,sha256=nUFxNCiKeYRVXuZEKA7GD-lTHxsiKcQ8FitZjKhPIvE,100
-airbyte_cdk/sources/declarative/models/declarative_component_schema.py,sha256=94efE5chOvaQaBTRx-Hxu2xyN6mDZ7DuFWJUH5fOgWY,91690
+airbyte_cdk/sources/declarative/models/declarative_component_schema.py,sha256=-XTzt8EY23Qel7KZINQIbQOoBWqdiK6am-p7A615AI0,90753
 airbyte_cdk/sources/declarative/parsers/__init__.py,sha256=ZnqYNxHsKCgO38IwB34RQyRMXTs4GTvlRi3ImKnIioo,61
 airbyte_cdk/sources/declarative/parsers/custom_exceptions.py,sha256=Rir9_z3Kcd5Es0-LChrzk-0qubAsiK_RSEnLmK2OXm8,553
 airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py,sha256=CXwTfD3wSQq3okcqwigpprbHhSURUokh4GK2OmOyKC8,9132
 airbyte_cdk/sources/declarative/parsers/manifest_reference_resolver.py,sha256=IWUOdF03o-aQn0Occo1BJCxU0Pz-QILk5L67nzw2thw,6803
-airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py,sha256=62HI4jMJE7EUJY8GLuGL33mfBmfP7zVSnoLcdyTlwYw,105328
+airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py,sha256=UWUlj6m2WT9SspMtvvhni31FoQBrkqnHGzrwFKem-Qc,104367
 airbyte_cdk/sources/declarative/partition_routers/__init__.py,sha256=1NjaZoGAIefvWwj6wx-LOKIXXWS-UnBlZFnuR7y6uYA,745
 airbyte_cdk/sources/declarative/partition_routers/cartesian_product_stream_slicer.py,sha256=c5cuVFM6NFkuQqG8Z5IwkBuwDrvXZN1CunUOM_L0ezg,6892
 airbyte_cdk/sources/declarative/partition_routers/list_partition_router.py,sha256=t7pRdFWfFWJtQQG19c9PVeMODyO2BknRTakpM5U9N-8,4844
@@ -267,7 +264,7 @@ airbyte_cdk/sources/streams/concurrent/partitions/types.py,sha256=frPVvHtY7vLxpG
 airbyte_cdk/sources/streams/concurrent/state_converters/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 airbyte_cdk/sources/streams/concurrent/state_converters/abstract_stream_state_converter.py,sha256=CXHUMOhndu-LOKgsnNTItv5s5qrKpmJDeHOzlH1nBy8,6819
 airbyte_cdk/sources/streams/concurrent/state_converters/datetime_stream_state_converter.py,sha256=syjdxEoElIOzqVS5Jrm5FOR70jsbBdttEO_3Iz12Jyo,7523
-airbyte_cdk/sources/streams/core.py,sha256=z4Oi5qmJPjs-RdMd5tPWHvHqIjkcxhkVKTvIpfAs2uA,32211
+airbyte_cdk/sources/streams/core.py,sha256=jiYW6w8cjNjzXMd8U8Gt-02fYYU7b0ciXSSSnGvFRak,32219
 airbyte_cdk/sources/streams/http/__init__.py,sha256=NXaNlkzZMkh5kS8S5ujEaKEE6855sk6_HljF_GFjKZI,311
 airbyte_cdk/sources/streams/http/availability_strategy.py,sha256=sovoGFThZr-doMN9vJvTuJBrvkwQVIO0qTQO64pGZPY,2428
 airbyte_cdk/sources/streams/http/error_handlers/__init__.py,sha256=R8OgTcratGH4f6BbYM2Hp8qYyEk7wMYuyda5H9ohGW8,665
@@ -339,8 +336,8 @@ airbyte_cdk/utils/slice_hasher.py,sha256=-pHexlNYoWYPnXNH-M7HEbjmeJe9Zk7SJijdQ7d
 airbyte_cdk/utils/spec_schema_transformations.py,sha256=-5HTuNsnDBAhj-oLeQXwpTGA0HdcjFOf2zTEMUTTg_Y,816
 airbyte_cdk/utils/stream_status_utils.py,sha256=ZmBoiy5HVbUEHAMrUONxZvxnvfV9CesmQJLDTAIWnWw,1171
 airbyte_cdk/utils/traced_exception.py,sha256=C8uIBuCL_E4WnBAOPSxBicD06JAldoN9fGsQDp463OY,6292
-airbyte_cdk-6.12.0rc1.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
-airbyte_cdk-6.12.0rc1.dist-info/METADATA,sha256=Dmtuhf6DiAt2U3kmIQjlVuruZToq8QNPhsqtZvUePAE,5953
-airbyte_cdk-6.12.0rc1.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-airbyte_cdk-6.12.0rc1.dist-info/entry_points.txt,sha256=fj-e3PAQvsxsQzyyq8UkG1k8spunWnD4BAH2AwlR6NM,95
-airbyte_cdk-6.12.0rc1.dist-info/RECORD,,
+airbyte_cdk-6.12.1.dev0.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
+airbyte_cdk-6.12.1.dev0.dist-info/METADATA,sha256=wuEYLVSyehdNYlyYVlLw9n2uOyx1aT_WRmRnDlfHu0s,5955
+airbyte_cdk-6.12.1.dev0.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+airbyte_cdk-6.12.1.dev0.dist-info/entry_points.txt,sha256=fj-e3PAQvsxsQzyyq8UkG1k8spunWnD4BAH2AwlR6NM,95
+airbyte_cdk-6.12.1.dev0.dist-info/RECORD,,

airbyte_cdk/sources/declarative/decoders/parsers/__init__.py

@@ -1,7 +0,0 @@
-#
-# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
-#
-
-from airbyte_cdk.sources.declarative.decoders.parsers.parsers import Parser, JsonParser
-
-__all__ = ["Parser", "JsonParser"]

airbyte_cdk/sources/declarative/decoders/parsers/parsers.py

@@ -1,49 +0,0 @@
-#
-# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
-#
-
-import json
-import logging
-from abc import abstractmethod
-from dataclasses import InitVar, dataclass
-from typing import Any, Generator, Mapping, MutableMapping, Union
-
-logger = logging.getLogger("airbyte")
-
-
-@dataclass
-class Parser:
-    """
-    Parser strategy to convert str, bytes, or bytearray data into MutableMapping[str, Any].
-    """
-
-    @abstractmethod
-    def parse(
-        self, data: Union[str, bytes, bytearray]
-    ) -> Generator[MutableMapping[str, Any], None, None]:
-        pass
-
-
-@dataclass
-class JsonParser(Parser):
-    """
-    Parser strategy for converting JSON-structure str, bytes, or bytearray data into MutableMapping[str, Any].
-    """
-
-    parameters: InitVar[Mapping[str, Any]]
-
-    def parse(
-        self, data: Union[str, bytes, bytearray]
-    ) -> Generator[MutableMapping[str, Any], None, None]:
-        try:
-            body_json = json.loads(data)
-        except json.JSONDecodeError:
-            logger.warning(f"Data cannot be parsed into json: {data=}")
-            yield {}
-
-        if not isinstance(body_json, list):
-            body_json = [body_json]
-        if len(body_json) == 0:
-            yield {}
-        else:
-            yield from body_json

airbyte_cdk/sources/declarative/decoders/zipfile_decoder.py

@@ -1,54 +0,0 @@
-#
-# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
-#
-
-import gzip
-import io
-import logging
-import zipfile
-from dataclasses import InitVar, dataclass
-from typing import Any, Generator, Mapping, MutableMapping, Optional
-
-import requests
-
-from airbyte_cdk.sources.declarative.decoders import Decoder
-from airbyte_cdk.sources.declarative.decoders.parsers import JsonParser, Parser
-
-logger = logging.getLogger("airbyte")
-
-
-@dataclass
-class ZipfileDecoder(Decoder):
-    parameters: InitVar[Mapping[str, Any]]
-    parser: Optional[Parser] = None
-
-    def __post_init__(self, parameters: Mapping[str, Any]) -> None:
-        self._parser = (
-            self.parser(parameters=parameters) if self.parser else JsonParser(parameters=parameters)
-        )
-
-    def is_stream_response(self) -> bool:
-        return False
-
-    def decode(
-        self, response: requests.Response
-    ) -> Generator[MutableMapping[str, Any], None, None]:
-        try:
-            zip_file = zipfile.ZipFile(io.BytesIO(response.content))
-        except zipfile.BadZipFile as e:
-            logger.exception(e)
-            logger.error(
-                f"Received an invalid zip file in response to URL: {response.request.url}. "
-                f"The size of the response body is: {len(response.content)}"
-            )
-            yield {}
-
-        for gzip_filename in zip_file.namelist():
-            with zip_file.open(gzip_filename) as file:
-                try:
-                    for data in gzip.open(file):
-                        yield from self._parser.parse(data)
-                except gzip.BadGzipFile as e:
-                    logger.exception(e)
-                    logger.error(f"Fail to read contents of zipped response: {e}")
-                    yield {}