airbyte-cdk 6.18.0.dev1__py3-none-any.whl → 6.18.0.dev3__py3-none-any.whl

airbyte_cdk/sources/declarative/declarative_component_schema.yaml

@@ -2014,20 +2014,6 @@ definitions:
       $parameters:
         type: object
         additionalProperties: true
-  JsonParser:
-    title: JsonParser
-    description: Parser used for parsing str, bytes, or bytearray data and returning data in a dictionary format.
-    type: object
-    additionalProperties: true
-    required:
-      - type
-    properties:
-      type:
-        type: string
-        enum: [JsonParser]
-      encoding:
-        type: string
-        default: utf-8
   ListPartitionRouter:
     title: List Partition Router
     description: A Partition router that specifies a list of attributes where each attribute describes a portion of the complete data set for a stream. During a sync, each value is iterated over and can be used as input to outbound API requests.
@@ -2873,6 +2859,7 @@ definitions:
       parser:
         anyOf:
           - "$ref": "#/definitions/GzipParser"
+          - "$ref": "#/definitions/JsonParser"
           - "$ref": "#/definitions/JsonLineParser"
           - "$ref": "#/definitions/CsvParser"
   #    PARSERS
@@ -2889,6 +2876,21 @@ definitions:
         anyOf:
           - "$ref": "#/definitions/JsonLineParser"
           - "$ref": "#/definitions/CsvParser"
+          - "$ref": "#/definitions/JsonParser"
+  JsonParser:
+    title: JsonParser
+    description: Parser used for parsing str, bytes, or bytearray data and returning data in a dictionary format.
+    type: object
+    additionalProperties: true
+    required:
+      - type
+    properties:
+      type:
+        type: string
+        enum: [JsonParser]
+      encoding:
+        type: string
+        default: utf-8
   JsonLineParser:
     type: object
     required:

airbyte_cdk/sources/declarative/decoders/composite_raw_decoder.py

@@ -50,25 +50,40 @@ class JsonParser(Parser):
     encoding: str = "utf-8"
 
     def parse(self, data: BufferedIOBase) -> Generator[MutableMapping[str, Any], None, None]:
+        """
+        Attempts to deserialize data using orjson library. As an extra layer of safety we fallback on the json library to deserialize the data.
+        """
         raw_data = data.read()
-        try:
-            body_json = orjson.loads(raw_data.decode(self.encoding))
-        except Exception:
-            try:
-                body_json = json.loads(raw_data.decode(self.encoding))
-            except Exception as exc:
-                raise AirbyteTracedException(
-                    message="Response JSON data failed to be parsed. See logs for more inforation.",
-                    internal_message=f"Response JSON data faild to be parsed: {exc=}, {raw_data=}",
-                    failure_type=FailureType.system_error,
-                    exception=exc,
-                )
+        body_json = self._parse_orjson(raw_data) or self._parse_json(raw_data)
+
+        if body_json is None:
+            raise AirbyteTracedException(
+                message="Response JSON data failed to be parsed. See logs for more information.",
+                internal_message=f"Response JSON data failed to be parsed.",
+                failure_type=FailureType.system_error,
+            )
 
         if isinstance(body_json, list):
             yield from body_json
         else:
             yield from [body_json]
 
+    def _parse_orjson(self, raw_data: bytes) -> Optional[Any]:
+        try:
+            return orjson.loads(raw_data.decode(self.encoding))
+        except Exception as exc:
+            logger.debug(
+                f"Failed to parse JSON data using orjson library. Falling back to json library. {exc}"
+            )
+            return None
+
+    def _parse_json(self, raw_data: bytes) -> Optional[Any]:
+        try:
+            return json.loads(raw_data.decode(self.encoding))
+        except Exception as exc:
+            logger.error(f"Failed to parse JSON data using json library. {exc}")
+            return None
+
 
 @dataclass
 class JsonLineParser(Parser):

airbyte_cdk/sources/declarative/models/declarative_component_schema.py

@@ -805,14 +805,6 @@ class GzipJsonDecoder(BaseModel):
     parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
 
 
-class JsonParser(BaseModel):
-    class Config:
-        extra = Extra.allow
-
-    type: Literal["JsonParser"]
-    encoding: Optional[str] = "utf-8"
-
-
 class MinMaxDatetime(BaseModel):
     type: Literal["MinMaxDatetime"]
     datetime: str = Field(
@@ -1181,6 +1173,14 @@ class LegacySessionTokenAuthenticator(BaseModel):
     parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
 
 
+class JsonParser(BaseModel):
+    class Config:
+        extra = Extra.allow
+
+    type: Literal["JsonParser"]
+    encoding: Optional[str] = "utf-8"
+
+
 class JsonLineParser(BaseModel):
     type: Literal["JsonLineParser"]
     encoding: Optional[str] = "utf-8"
@@ -1579,7 +1579,7 @@ class RecordSelector(BaseModel):
 
 class GzipParser(BaseModel):
     type: Literal["GzipParser"]
-    inner_parser: Union[JsonLineParser, CsvParser]
+    inner_parser: Union[JsonLineParser, CsvParser, JsonParser]
 
 
 class Spec(BaseModel):
@@ -1614,7 +1614,7 @@ class CompositeErrorHandler(BaseModel):
 
 class CompositeRawDecoder(BaseModel):
     type: Literal["CompositeRawDecoder"]
-    parser: Union[GzipParser, JsonLineParser, CsvParser]
+    parser: Union[GzipParser, JsonParser, JsonLineParser, CsvParser]
 
 
 class DeclarativeSource1(BaseModel):

airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py

@@ -73,6 +73,7 @@ from airbyte_cdk.sources.declarative.decoders.composite_raw_decoder import (
     GzipParser,
     JsonLineParser,
     JsonParser,
+    Parser,
 )
 from airbyte_cdk.sources.declarative.extractors import (
     DpathExtractor,
@@ -1037,17 +1038,17 @@ class ModelToComponentFactory:
         self, model: CursorPaginationModel, config: Config, decoder: Decoder, **kwargs: Any
     ) -> CursorPaginationStrategy:
         if isinstance(decoder, PaginationDecoderDecorator):
-            if not isinstance(decoder.decoder, (JsonDecoder, XmlDecoder)):
-                raise ValueError(
-                    f"Provided decoder of {type(decoder.decoder)=} is not supported. Please set JsonDecoder or XmlDecoder instead."
-                )
+            inner_decoder = decoder.decoder
+        else:
+            inner_decoder = decoder
+            decoder = PaginationDecoderDecorator(decoder=decoder)
+
+        if self._is_supported_decoder_for_pagination(inner_decoder):
             decoder_to_use = decoder
         else:
-            if not isinstance(decoder, (JsonDecoder, XmlDecoder)):
-                raise ValueError(
-                    f"Provided decoder of {type(decoder)=} is not supported. Please set JsonDecoder or XmlDecoder instead."
-                )
-            decoder_to_use = PaginationDecoderDecorator(decoder=decoder)
+            raise ValueError(
+                self._UNSUPPORTED_DECODER_ERROR.format(decoder_type=type(inner_decoder))
+            )
 
         return CursorPaginationStrategy(
             cursor_value=model.cursor_value,
@@ -1520,11 +1521,10 @@ class ModelToComponentFactory:
         cursor_used_for_stop_condition: Optional[DeclarativeCursor] = None,
     ) -> Union[DefaultPaginator, PaginatorTestReadDecorator]:
         if decoder:
-            if not isinstance(decoder, (JsonDecoder, XmlDecoder)):
-                raise ValueError(
-                    f"Provided decoder of {type(decoder)=} is not supported. Please set JsonDecoder or XmlDecoder instead."
-                )
-            decoder_to_use = PaginationDecoderDecorator(decoder=decoder)
+            if self._is_supported_decoder_for_pagination(decoder):
+                decoder_to_use = PaginationDecoderDecorator(decoder=decoder)
+            else:
+                raise ValueError(self._UNSUPPORTED_DECODER_ERROR.format(decoder_type=type(decoder)))
         else:
             decoder_to_use = PaginationDecoderDecorator(decoder=JsonDecoder(parameters={}))
         page_size_option = (
@@ -1755,7 +1755,7 @@ class ModelToComponentFactory:
 
     @staticmethod
     def create_json_parser(model: JsonParserModel, config: Config, **kwargs: Any) -> JsonParser:
-        encoding = model.encoding or "utf-8"
+        encoding = model.encoding if model.encoding else "utf-8"
         return JsonParser(encoding=encoding)
 
     @staticmethod
@@ -1937,22 +1937,22 @@ class ModelToComponentFactory:
             message_repository=self._message_repository,
         )
 
-    @staticmethod
     def create_offset_increment(
-        model: OffsetIncrementModel, config: Config, decoder: Decoder, **kwargs: Any
+        self, model: OffsetIncrementModel, config: Config, decoder: Decoder, **kwargs: Any
     ) -> OffsetIncrement:
         if isinstance(decoder, PaginationDecoderDecorator):
-            if not isinstance(decoder.decoder, (JsonDecoder, XmlDecoder)):
-                raise ValueError(
-                    f"Provided decoder of {type(decoder.decoder)=} is not supported. Please set JsonDecoder or XmlDecoder instead."
-                )
+            inner_decoder = decoder.decoder
+        else:
+            inner_decoder = decoder
+            decoder = PaginationDecoderDecorator(decoder=decoder)
+
+        if self._is_supported_decoder_for_pagination(inner_decoder):
             decoder_to_use = decoder
         else:
-            if not isinstance(decoder, (JsonDecoder, XmlDecoder)):
-                raise ValueError(
-                    f"Provided decoder of {type(decoder)=} is not supported. Please set JsonDecoder or XmlDecoder instead."
-                )
-            decoder_to_use = PaginationDecoderDecorator(decoder=decoder)
+            raise ValueError(
+                self._UNSUPPORTED_DECODER_ERROR.format(decoder_type=type(inner_decoder))
+            )
+
         return OffsetIncrement(
             page_size=model.page_size,
             config=config,
@@ -2541,3 +2541,25 @@ class ModelToComponentFactory:
             components_mapping=components_mapping,
             parameters=model.parameters or {},
         )
+
+    _UNSUPPORTED_DECODER_ERROR = (
+        "Specified decoder of {decoder_type} is not supported for pagination."
+        "Please set as `JsonDecoder`, `XmlDecoder`, or a `CompositeRawDecoder` with an inner_parser of `JsonParser` or `GzipParser` instead."
+        "If using `GzipParser`, please ensure that the lowest level inner_parser is a `JsonParser`."
+    )
+
+    def _is_supported_decoder_for_pagination(self, decoder: Decoder) -> bool:
+        if isinstance(decoder, (JsonDecoder, XmlDecoder)):
+            return True
+        elif isinstance(decoder, CompositeRawDecoder):
+            return self._is_supported_parser_for_pagination(decoder.parser)
+        else:
+            return False
+
+    def _is_supported_parser_for_pagination(self, parser: Parser) -> bool:
+        if isinstance(parser, JsonParser):
+            return True
+        elif isinstance(parser, GzipParser):
+            return isinstance(parser.inner_parser, JsonParser)
+        else:
+            return False

{airbyte_cdk-6.18.0.dev1.dist-info → airbyte_cdk-6.18.0.dev3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: airbyte-cdk
-Version: 6.18.0.dev1
+Version: 6.18.0.dev3
 Summary: A framework for writing Airbyte Connectors.
 License: MIT
 Keywords: airbyte,connector-development-kit,cdk

{airbyte_cdk-6.18.0.dev1.dist-info → airbyte_cdk-6.18.0.dev3.dist-info}/RECORD

@@ -66,11 +66,11 @@ airbyte_cdk/sources/declarative/concurrent_declarative_source.py,sha256=tSTCSmyM
 airbyte_cdk/sources/declarative/datetime/__init__.py,sha256=l9LG7Qm6e5r_qgqfVKnx3mXYtg1I9MmMjomVIPfU4XA,177
 airbyte_cdk/sources/declarative/datetime/datetime_parser.py,sha256=SX9JjdesN1edN2WVUVMzU_ptqp2QB1OnsnjZ4mwcX7w,2579
 airbyte_cdk/sources/declarative/datetime/min_max_datetime.py,sha256=0BHBtDNQZfvwM45-tY5pNlTcKAFSGGNxemoi0Jic-0E,5785
-airbyte_cdk/sources/declarative/declarative_component_schema.yaml,sha256=QDqDyKmkYDDW3fXA8ImE61p4v_sBNQnqnV-uX_qNHNM,133531
+airbyte_cdk/sources/declarative/declarative_component_schema.yaml,sha256=fCvq1sFwE4h7sP9sjZHoggsw8K3pvOEMvnf_es3fizA,133625
 airbyte_cdk/sources/declarative/declarative_source.py,sha256=nF7wBqFd3AQmEKAm4CnIo29CJoQL562cJGSCeL8U8bA,1531
 airbyte_cdk/sources/declarative/declarative_stream.py,sha256=JRyNeOIpsFu4ztVZsN6sncqUEIqIE-bUkD2TPgbMgk0,10375
 airbyte_cdk/sources/declarative/decoders/__init__.py,sha256=edGj4fGxznBk4xzRQyCA1rGfbpqe7z-RE0K3kQQWbgA,858
-airbyte_cdk/sources/declarative/decoders/composite_raw_decoder.py,sha256=rSvqdGsVgBT3ZfY_bthjZl_OmxY84iKz8g9GQIWyq8k,3766
+airbyte_cdk/sources/declarative/decoders/composite_raw_decoder.py,sha256=kQfUVMVhChKe5OngwIQrs0F9KGnRUN-CKVFakCU23DQ,4354
 airbyte_cdk/sources/declarative/decoders/decoder.py,sha256=sl-Gt8lXi7yD2Q-sD8je5QS2PbgrgsYjxRLWsay7DMc,826
 airbyte_cdk/sources/declarative/decoders/json_decoder.py,sha256=qdbjeR6RffKaah_iWvMsOcDolYuxJY5DaI3b9AMTZXg,3327
 airbyte_cdk/sources/declarative/decoders/noop_decoder.py,sha256=iZh0yKY_JzgBnJWiubEusf5c0o6Khd-8EWFWT-8EgFo,542
@@ -106,12 +106,12 @@ airbyte_cdk/sources/declarative/migrations/__init__.py,sha256=47DEQpj8HBSa-_TImW
 airbyte_cdk/sources/declarative/migrations/legacy_to_per_partition_state_migration.py,sha256=iemy3fKLczcU0-Aor7tx5jcT6DRedKMqyK7kCOp01hg,3924
 airbyte_cdk/sources/declarative/migrations/state_migration.py,sha256=KWPjealMLKSMtajXgkdGgKg7EmTLR-CqqD7UIh0-eDU,794
 airbyte_cdk/sources/declarative/models/__init__.py,sha256=nUFxNCiKeYRVXuZEKA7GD-lTHxsiKcQ8FitZjKhPIvE,100
-airbyte_cdk/sources/declarative/models/declarative_component_schema.py,sha256=SpMwCe-6NZTxICSFIXzwlAnAwNLlC8xS12ncEC1NcbA,93536
+airbyte_cdk/sources/declarative/models/declarative_component_schema.py,sha256=_aQM6pF-a4Zk7kQ-_GiVaRi30bpyTQDg4G7KZD83yuI,93560
 airbyte_cdk/sources/declarative/parsers/__init__.py,sha256=ZnqYNxHsKCgO38IwB34RQyRMXTs4GTvlRi3ImKnIioo,61
 airbyte_cdk/sources/declarative/parsers/custom_exceptions.py,sha256=Rir9_z3Kcd5Es0-LChrzk-0qubAsiK_RSEnLmK2OXm8,553
 airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py,sha256=CXwTfD3wSQq3okcqwigpprbHhSURUokh4GK2OmOyKC8,9132
 airbyte_cdk/sources/declarative/parsers/manifest_reference_resolver.py,sha256=IWUOdF03o-aQn0Occo1BJCxU0Pz-QILk5L67nzw2thw,6803
-airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py,sha256=lmSh2Yp-lgRTbbSw3m6UH8L2nTRjt0w3aiISWHRG6IM,109739
+airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py,sha256=PEtlR6K9zCYDEXfba9QtYeUnC-KOVdaHD-9QSq_UNKY,110269
 airbyte_cdk/sources/declarative/partition_routers/__init__.py,sha256=HJ-Syp3p7RpyR_OK0X_a2kSyISfu3W-PKrRI16iY0a8,957
 airbyte_cdk/sources/declarative/partition_routers/async_job_partition_router.py,sha256=n82J15S8bjeMZ5uROu--P3hnbQoxkY5v7RPHYx7g7ro,2929
 airbyte_cdk/sources/declarative/partition_routers/cartesian_product_stream_slicer.py,sha256=c5cuVFM6NFkuQqG8Z5IwkBuwDrvXZN1CunUOM_L0ezg,6892
@@ -342,8 +342,8 @@ airbyte_cdk/utils/slice_hasher.py,sha256=-pHexlNYoWYPnXNH-M7HEbjmeJe9Zk7SJijdQ7d
 airbyte_cdk/utils/spec_schema_transformations.py,sha256=-5HTuNsnDBAhj-oLeQXwpTGA0HdcjFOf2zTEMUTTg_Y,816
 airbyte_cdk/utils/stream_status_utils.py,sha256=ZmBoiy5HVbUEHAMrUONxZvxnvfV9CesmQJLDTAIWnWw,1171
 airbyte_cdk/utils/traced_exception.py,sha256=C8uIBuCL_E4WnBAOPSxBicD06JAldoN9fGsQDp463OY,6292
-airbyte_cdk-6.18.0.dev1.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
-airbyte_cdk-6.18.0.dev1.dist-info/METADATA,sha256=ALXOgvI3pTcF2tNmvbQ9S8fG424n229th_tx1u2uSCo,6005
-airbyte_cdk-6.18.0.dev1.dist-info/WHEEL,sha256=IYZQI976HJqqOpQU6PHkJ8fb3tMNBFjg-Cn-pwAbaFM,88
-airbyte_cdk-6.18.0.dev1.dist-info/entry_points.txt,sha256=fj-e3PAQvsxsQzyyq8UkG1k8spunWnD4BAH2AwlR6NM,95
-airbyte_cdk-6.18.0.dev1.dist-info/RECORD,,
+airbyte_cdk-6.18.0.dev3.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
+airbyte_cdk-6.18.0.dev3.dist-info/METADATA,sha256=1N26t4q8cvzW-7escwWwzk2BpIgCDOhfvCsLtu4-270,6005
+airbyte_cdk-6.18.0.dev3.dist-info/WHEEL,sha256=IYZQI976HJqqOpQU6PHkJ8fb3tMNBFjg-Cn-pwAbaFM,88
+airbyte_cdk-6.18.0.dev3.dist-info/entry_points.txt,sha256=fj-e3PAQvsxsQzyyq8UkG1k8spunWnD4BAH2AwlR6NM,95
+airbyte_cdk-6.18.0.dev3.dist-info/RECORD,,