airbyte-cdk 6.6.0__tar.gz → 6.6.0rc1__tar.gz

{airbyte_cdk-6.6.0 → airbyte_cdk-6.6.0rc1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 6.6.0
+Version: 6.6.0rc1
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://airbyte.com
 License: MIT
@@ -31,11 +31,11 @@ Requires-Dist: avro (>=1.11.2,<1.12.0) ; extra == "file-based"
 Requires-Dist: backoff
 Requires-Dist: cachetools
 Requires-Dist: cohere (==4.21) ; extra == "vector-db-based"
-Requires-Dist: cryptography (>=42.0.5,<44.0.0)
+Requires-Dist: cryptography (>=42.0.5,<43.0.0)
 Requires-Dist: dpath (>=2.1.6,<3.0.0)
 Requires-Dist: dunamai (>=1.22.0,<2.0.0)
 Requires-Dist: fastavro (>=1.8.0,<1.9.0) ; extra == "file-based"
-Requires-Dist: genson (==1.3.0)
+Requires-Dist: genson (==1.2.2)
 Requires-Dist: isodate (>=0.6.1,<0.7.0)
 Requires-Dist: jsonref (>=0.2,<0.3)
 Requires-Dist: jsonschema (>=3.2.0,<3.3.0)
@@ -66,10 +66,10 @@ Requires-Dist: requests_cache
 Requires-Dist: serpyco-rs (>=1.10.2,<2.0.0)
 Requires-Dist: sphinx-rtd-theme (>=1.0,<1.1) ; extra == "sphinx-docs"
 Requires-Dist: sqlalchemy (>=2.0,<3.0,!=2.0.36) ; extra == "sql"
-Requires-Dist: tiktoken (==0.8.0) ; extra == "vector-db-based"
+Requires-Dist: tiktoken (==0.4.0) ; extra == "vector-db-based"
 Requires-Dist: unstructured.pytesseract (>=0.3.12) ; extra == "file-based"
 Requires-Dist: unstructured[docx,pptx] (==0.10.27) ; extra == "file-based"
-Requires-Dist: wcmatch (==10.0)
+Requires-Dist: wcmatch (==8.4)
 Requires-Dist: xmltodict (>=0.13.0,<0.14.0)
 Project-URL: Documentation, https://docs.airbyte.io/
 Project-URL: Repository, https://github.com/airbytehq/airbyte-python-cdk

{airbyte_cdk-6.6.0 → airbyte_cdk-6.6.0rc1}/airbyte_cdk/__init__.py

@@ -282,18 +282,12 @@ __all__ = [
     "StreamSlice",
 ]
 
-__version__: str
+__version__ = _dunamai.get_version(
+    "airbyte-cdk",
+    third_choice=_dunamai.Version.from_any_vcs,
+).serialize()
 """Version generated by poetry dynamic versioning during publish.
 
 When running in development, dunamai will calculate a new prerelease version
 from existing git release tag info.
 """
-
-try:
-    __version__ = _dunamai.get_version(
-        "airbyte-cdk",
-        third_choice=_dunamai.Version.from_any_vcs,
-        fallback=_dunamai.Version("0.0.0+dev"),
-    ).serialize()
-except:
-    __version__ = "0.0.0+dev"

{airbyte_cdk-6.6.0 → airbyte_cdk-6.6.0rc1}/airbyte_cdk/sources/declarative/concurrent_declarative_source.py

@@ -16,9 +16,6 @@ from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager
 from airbyte_cdk.sources.declarative.concurrency_level import ConcurrencyLevel
 from airbyte_cdk.sources.declarative.declarative_stream import DeclarativeStream
 from airbyte_cdk.sources.declarative.extractors import RecordSelector
-from airbyte_cdk.sources.declarative.extractors.record_filter import (
-    ClientSideIncrementalRecordFilterDecorator,
-)
 from airbyte_cdk.sources.declarative.incremental.datetime_based_cursor import DatetimeBasedCursor
 from airbyte_cdk.sources.declarative.interpolation import InterpolatedString
 from airbyte_cdk.sources.declarative.manifest_declarative_source import ManifestDeclarativeSource
@@ -294,9 +291,6 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
             if isinstance(record_selector, RecordSelector):
                 if (
                     record_selector.record_filter
-                    and not isinstance(
-                        record_selector.record_filter, ClientSideIncrementalRecordFilterDecorator
-                    )
                     and "stream_state" in record_selector.record_filter.condition
                 ):
                     self.logger.warning(

{airbyte_cdk-6.6.0 → airbyte_cdk-6.6.0rc1}/airbyte_cdk/sources/declarative/declarative_component_schema.yaml

@@ -1750,45 +1750,6 @@ definitions:
       type:
         type: string
         enum: [XmlDecoder]
-  CustomDecoder:
-    title: Custom Decoder
-    description: Use this to implement custom decoder logic.
-    type: object
-    additionalProperties: true
-    required:
-      - type
-      - class_name
-    properties:
-      type:
-        type: string
-        enum: [CustomDecoder]
-      class_name:
-        title: Class Name
-        description: Fully-qualified name of the class that will be implementing the custom decoding. Has to be a sub class of Decoder. The format is `source_<name>.<package>.<class_name>`.
-        type: string
-        additionalProperties: true
-        examples:
-          - "source_amazon_ads.components.GzipJsonlDecoder"
-      $parameters:
-        type: object
-        additionalProperties: true
-  GzipJsonDecoder:
-    title: GzipJson Decoder
-    description: Use this if the response is Gzip compressed Json.
-    type: object
-    additionalProperties: true
-    required:
-      - type
-    properties:
-      type:
-        type: string
-        enum: [GzipJsonDecoder]
-      encoding:
-        type: string
-        default: utf-8
-      $parameters:
-        type: object
-        additionalProperties: true
   ListPartitionRouter:
     title: List Partition Router
     description: A Partition router that specifies a list of attributes where each attribute describes a portion of the complete data set for a stream. During a sync, each value is iterated over and can be used as input to outbound API requests.
@@ -2443,12 +2404,10 @@ definitions:
         title: Decoder
         description: Component decoding the response so records can be extracted.
         anyOf:
-          - "$ref": "#/definitions/CustomDecoder"
           - "$ref": "#/definitions/JsonDecoder"
           - "$ref": "#/definitions/JsonlDecoder"
           - "$ref": "#/definitions/IterableDecoder"
           - "$ref": "#/definitions/XmlDecoder"
-          - "$ref": "#/definitions/GzipJsonDecoder"
       $parameters:
         type: object
         additionalProperties: true
@@ -2561,12 +2520,10 @@ definitions:
         title: Decoder
         description: Component decoding the response so records can be extracted.
         anyOf:
-          - "$ref": "#/definitions/CustomDecoder"
           - "$ref": "#/definitions/JsonDecoder"
           - "$ref": "#/definitions/JsonlDecoder"
           - "$ref": "#/definitions/IterableDecoder"
           - "$ref": "#/definitions/XmlDecoder"
-          - "$ref": "#/definitions/GzipJsonDecoder"
       $parameters:
         type: object
         additionalProperties: true

{airbyte_cdk-6.6.0 → airbyte_cdk-6.6.0rc1}/airbyte_cdk/sources/declarative/decoders/__init__.py

@@ -3,9 +3,9 @@
 #
 
 from airbyte_cdk.sources.declarative.decoders.decoder import Decoder
-from airbyte_cdk.sources.declarative.decoders.json_decoder import JsonDecoder, JsonlDecoder, IterableDecoder, GzipJsonDecoder
+from airbyte_cdk.sources.declarative.decoders.json_decoder import JsonDecoder, JsonlDecoder, IterableDecoder
 from airbyte_cdk.sources.declarative.decoders.noop_decoder import NoopDecoder
 from airbyte_cdk.sources.declarative.decoders.pagination_decoder_decorator import PaginationDecoderDecorator
 from airbyte_cdk.sources.declarative.decoders.xml_decoder import XmlDecoder
 
-__all__ = ["Decoder", "JsonDecoder", "JsonlDecoder", "IterableDecoder", "GzipJsonDecoder", "NoopDecoder", "PaginationDecoderDecorator", "XmlDecoder"]
+__all__ = ["Decoder", "JsonDecoder", "JsonlDecoder", "IterableDecoder", "NoopDecoder", "PaginationDecoderDecorator", "XmlDecoder"]

{airbyte_cdk-6.6.0 → airbyte_cdk-6.6.0rc1}/airbyte_cdk/sources/declarative/decoders/json_decoder.py

@@ -1,15 +1,14 @@
 #
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
-import codecs
+
 import logging
 from dataclasses import InitVar, dataclass
-from gzip import decompress
-from typing import Any, Generator, Mapping, MutableMapping, List, Optional
+from typing import Any, Generator, Mapping
 
 import requests
 from airbyte_cdk.sources.declarative.decoders.decoder import Decoder
-import orjson
+from orjson import orjson
 
 logger = logging.getLogger("airbyte")
 
@@ -25,32 +24,24 @@ class JsonDecoder(Decoder):
     def is_stream_response(self) -> bool:
         return False
 
-    def decode(
-        self, response: requests.Response
-    ) -> Generator[MutableMapping[str, Any], None, None]:
+    def decode(self, response: requests.Response) -> Generator[Mapping[str, Any], None, None]:
         """
         Given the response is an empty string or an emtpy list, the function will return a generator with an empty mapping.
         """
         try:
             body_json = response.json()
-            yield from self.parse_body_json(body_json)
+            if not isinstance(body_json, list):
+                body_json = [body_json]
+            if len(body_json) == 0:
+                yield {}
+            else:
+                yield from body_json
         except requests.exceptions.JSONDecodeError:
             logger.warning(
                 f"Response cannot be parsed into json: {response.status_code=}, {response.text=}"
             )
             yield {}
 
-    @staticmethod
-    def parse_body_json(
-        body_json: MutableMapping[str, Any] | List[MutableMapping[str, Any]],
-    ) -> Generator[MutableMapping[str, Any], None, None]:
-        if not isinstance(body_json, list):
-            body_json = [body_json]
-        if len(body_json) == 0:
-            yield {}
-        else:
-            yield from body_json
-
 
 @dataclass
 class IterableDecoder(Decoder):
@@ -63,9 +54,7 @@ class IterableDecoder(Decoder):
     def is_stream_response(self) -> bool:
         return True
 
-    def decode(
-        self, response: requests.Response
-    ) -> Generator[MutableMapping[str, Any], None, None]:
+    def decode(self, response: requests.Response) -> Generator[Mapping[str, Any], None, None]:
         for line in response.iter_lines():
             yield {"record": line.decode()}
 
@@ -81,30 +70,8 @@ class JsonlDecoder(Decoder):
     def is_stream_response(self) -> bool:
         return True
 
-    def decode(
-        self, response: requests.Response
-    ) -> Generator[MutableMapping[str, Any], None, None]:
+    def decode(self, response: requests.Response) -> Generator[Mapping[str, Any], None, None]:
         # TODO???: set delimiter? usually it is `\n` but maybe it would be useful to set optional?
         #  https://github.com/airbytehq/airbyte-internal-issues/issues/8436
         for record in response.iter_lines():
             yield orjson.loads(record)
-
-
-@dataclass
-class GzipJsonDecoder(JsonDecoder):
-    encoding: Optional[str]
-
-    def __post_init__(self, parameters: Mapping[str, Any]) -> None:
-        if self.encoding:
-            try:
-                codecs.lookup(self.encoding)
-            except LookupError:
-                raise ValueError(
-                    f"Invalid encoding '{self.encoding}'. Please check provided encoding"
-                )
-
-    def decode(
-        self, response: requests.Response
-    ) -> Generator[MutableMapping[str, Any], None, None]:
-        raw_string = decompress(response.content).decode(encoding=self.encoding or "utf-8")
-        yield from self.parse_body_json(orjson.loads(raw_string))

{airbyte_cdk-6.6.0 → airbyte_cdk-6.6.0rc1}/airbyte_cdk/sources/declarative/manifest_declarative_source.py

@@ -256,10 +256,7 @@ class ManifestDeclarativeSource(DeclarativeSource):
             manifest_version, "manifest"
         )
 
-        if cdk_version.startswith("0.0.0"):
-            # Skipping version compatibility check on unreleased dev branch
-            pass
-        elif cdk_major < manifest_major or (
+        if cdk_major < manifest_major or (
             cdk_major == manifest_major and cdk_minor < manifest_minor
         ):
             raise ValidationError(

{airbyte_cdk-6.6.0 → airbyte_cdk-6.6.0rc1}/airbyte_cdk/sources/declarative/migrations/legacy_to_per_partition_state_migration.py

@@ -4,11 +4,7 @@ from typing import Any, Mapping
 
 from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
 from airbyte_cdk.sources.declarative.migrations.state_migration import StateMigration
-from airbyte_cdk.sources.declarative.models import (
-    DatetimeBasedCursor,
-    SubstreamPartitionRouter,
-    CustomIncrementalSync,
-)
+from airbyte_cdk.sources.declarative.models import DatetimeBasedCursor, SubstreamPartitionRouter
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import ParentStreamConfig
 
 
@@ -36,7 +32,7 @@ class LegacyToPerPartitionStateMigration(StateMigration):
     def __init__(
         self,
         partition_router: SubstreamPartitionRouter,
-        cursor: CustomIncrementalSync | DatetimeBasedCursor,
+        cursor: DatetimeBasedCursor,
         config: Mapping[str, Any],
         parameters: Mapping[str, Any],
     ):
@@ -68,7 +64,7 @@ class LegacyToPerPartitionStateMigration(StateMigration):
             return False
 
         # There is exactly one parent stream
-        number_of_parent_streams = len(self._partition_router.parent_stream_configs)  # type: ignore # custom partition will introduce this attribute if needed
+        number_of_parent_streams = len(self._partition_router.parent_stream_configs)
         if number_of_parent_streams != 1:
             # There should be exactly one parent stream
             return False

{airbyte_cdk-6.6.0 → airbyte_cdk-6.6.0rc1}/airbyte_cdk/sources/declarative/models/declarative_component_schema.py

@@ -4,9 +4,10 @@
 from __future__ import annotations
 
 from enum import Enum
-from typing import Any, Dict, List, Literal, Optional, Union
+from typing import Any, Dict, List, Optional, Union
 
 from pydantic.v1 import BaseModel, Extra, Field
+from typing_extensions import Literal
 
 
 class AuthFlowType(Enum):
@@ -631,7 +632,6 @@ class HttpResponseFilter(BaseModel):
         description="Match the response if its HTTP code is included in this list.",
         examples=[[420, 429], [500]],
         title="HTTP Codes",
-        unique_items=True,
     )
     predicate: Optional[str] = Field(
         None,
@@ -687,29 +687,6 @@ class XmlDecoder(BaseModel):
     type: Literal["XmlDecoder"]
 
 
-class CustomDecoder(BaseModel):
-    class Config:
-        extra = Extra.allow
-
-    type: Literal["CustomDecoder"]
-    class_name: str = Field(
-        ...,
-        description="Fully-qualified name of the class that will be implementing the custom decoding. Has to be a sub class of Decoder. The format is `source_<name>.<package>.<class_name>`.",
-        examples=["source_amazon_ads.components.GzipJsonlDecoder"],
-        title="Class Name",
-    )
-    parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
-
-
-class GzipJsonDecoder(BaseModel):
-    class Config:
-        extra = Extra.allow
-
-    type: Literal["GzipJsonDecoder"]
-    encoding: Optional[str] = "utf-8"
-    parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
-
-
 class MinMaxDatetime(BaseModel):
     type: Literal["MinMaxDatetime"]
     datetime: str = Field(
@@ -1643,16 +1620,7 @@ class SimpleRetriever(BaseModel):
         description="PartitionRouter component that describes how to partition the stream, enabling incremental syncs and checkpointing.",
         title="Partition Router",
     )
-    decoder: Optional[
-        Union[
-            CustomDecoder,
-            JsonDecoder,
-            JsonlDecoder,
-            IterableDecoder,
-            XmlDecoder,
-            GzipJsonDecoder,
-        ]
-    ] = Field(
+    decoder: Optional[Union[JsonDecoder, JsonlDecoder, IterableDecoder, XmlDecoder]] = Field(
         None,
         description="Component decoding the response so records can be extracted.",
         title="Decoder",
@@ -1712,16 +1680,7 @@ class AsyncRetriever(BaseModel):
         description="PartitionRouter component that describes how to partition the stream, enabling incremental syncs and checkpointing.",
         title="Partition Router",
     )
-    decoder: Optional[
-        Union[
-            CustomDecoder,
-            JsonDecoder,
-            JsonlDecoder,
-            IterableDecoder,
-            XmlDecoder,
-            GzipJsonDecoder,
-        ]
-    ] = Field(
+    decoder: Optional[Union[JsonDecoder, JsonlDecoder, IterableDecoder, XmlDecoder]] = Field(
         None,
         description="Component decoding the response so records can be extracted.",
         title="Decoder",

{airbyte_cdk-6.6.0 → airbyte_cdk-6.6.0rc1}/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py

@@ -58,7 +58,6 @@ from airbyte_cdk.sources.declarative.datetime import MinMaxDatetime
 from airbyte_cdk.sources.declarative.declarative_stream import DeclarativeStream
 from airbyte_cdk.sources.declarative.decoders import (
     Decoder,
-    GzipJsonDecoder,
     IterableDecoder,
     JsonDecoder,
     JsonlDecoder,
@@ -135,9 +134,6 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     CustomBackoffStrategy as CustomBackoffStrategyModel,
 )
-from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
-    CustomDecoder as CustomDecoderModel,
-)
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     CustomErrorHandler as CustomErrorHandlerModel,
 )
@@ -186,9 +182,6 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     ExponentialBackoffStrategy as ExponentialBackoffStrategyModel,
 )
-from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
-    GzipJsonDecoder as GzipJsonDecoderModel,
-)
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     HttpRequester as HttpRequesterModel,
 )
@@ -409,7 +402,6 @@ class ModelToComponentFactory:
             CursorPaginationModel: self.create_cursor_pagination,
             CustomAuthenticatorModel: self.create_custom_component,
             CustomBackoffStrategyModel: self.create_custom_component,
-            CustomDecoderModel: self.create_custom_component,
             CustomErrorHandlerModel: self.create_custom_component,
             CustomIncrementalSyncModel: self.create_custom_component,
             CustomRecordExtractorModel: self.create_custom_component,
@@ -433,7 +425,6 @@ class ModelToComponentFactory:
             InlineSchemaLoaderModel: self.create_inline_schema_loader,
             JsonDecoderModel: self.create_json_decoder,
             JsonlDecoderModel: self.create_jsonl_decoder,
-            GzipJsonDecoderModel: self.create_gzipjson_decoder,
             KeysToLowerModel: self.create_keys_to_lower_transformation,
             IterableDecoderModel: self.create_iterable_decoder,
             XmlDecoderModel: self.create_xml_decoder,
@@ -628,16 +619,11 @@ class ModelToComponentFactory:
                 "LegacyToPerPartitionStateMigrations can only be applied with a parent stream configuration."
             )
 
-        if not hasattr(declarative_stream, "incremental_sync"):
-            raise ValueError(
-                "LegacyToPerPartitionStateMigrations can only be applied with an incremental_sync configuration."
-            )
-
         return LegacyToPerPartitionStateMigration(
-            partition_router,  # type: ignore # was already checked above
-            declarative_stream.incremental_sync,  # type: ignore # was already checked. Migration can be applied only to incremental streams.
+            declarative_stream.retriever.partition_router,
+            declarative_stream.incremental_sync,
             config,
-            declarative_stream.parameters,  # type: ignore # different type is expected here Mapping[str, Any], got Dict[str, Any]
+            declarative_stream.parameters,
         )  # type: ignore # The retriever type was already checked
 
     def create_session_token_authenticator(
@@ -1562,12 +1548,6 @@ class ModelToComponentFactory:
     def create_xml_decoder(model: XmlDecoderModel, config: Config, **kwargs: Any) -> XmlDecoder:
         return XmlDecoder(parameters={})
 
-    @staticmethod
-    def create_gzipjson_decoder(
-        model: GzipJsonDecoderModel, config: Config, **kwargs: Any
-    ) -> GzipJsonDecoder:
-        return GzipJsonDecoder(parameters={}, encoding=model.encoding)
-
     @staticmethod
     def create_json_file_schema_loader(
         model: JsonFileSchemaLoaderModel, config: Config, **kwargs: Any

{airbyte_cdk-6.6.0 → airbyte_cdk-6.6.0rc1}/pyproject.toml

@@ -2,7 +2,6 @@
 requires = ["poetry-core>=1.0.0", "poetry-dynamic-versioning>=1.0.0,<2.0.0"]
 build-backend = "poetry_dynamic_versioning.backend"
 
-
 [tool.poetry]
 name = "airbyte-cdk"
 description = "A framework for writing Airbyte Connectors."
@@ -23,7 +22,7 @@ classifiers = [
 keywords = ["airbyte", "connector-development-kit", "cdk"]
 
 # Python CDK uses dynamic versioning: https://github.com/mtkennerly/poetry-dynamic-versioning
-version = "6.6.0"  # Version will be calculated dynamically.
+version = "6.6.0rc1"  # Version will be calculated dynamically.
 
 [tool.poetry-dynamic-versioning]
 enable = false
@@ -36,7 +35,7 @@ cachetools = "*"
 Deprecated = "~1.2"
 dpath = "^2.1.6"
 dunamai = "^1.22.0"
-genson = "1.3.0"
+genson = "1.2.2"
 isodate = "~0.6.1"
 Jinja2 = "~3.1.2"
 jsonref = "~0.2"
@@ -51,7 +50,7 @@ python-ulid = "^3.0.0"
 PyYAML = "^6.0.1"
 requests = "*"
 requests_cache = "*"
-wcmatch = "10.0"
+wcmatch = "8.4"
 # Extras depedencies
 avro = { version = "~1.11.2", optional = true }
 cohere = { version = "4.21", optional = true }
@@ -68,7 +67,7 @@ python-calamine = { version = "0.2.3", optional = true }
 python-snappy = { version = "0.7.3", optional = true }
 Sphinx = { version = "~4.2", optional = true }
 sphinx-rtd-theme = { version = "~1.0", optional = true }
-tiktoken = { version = "0.8.0", optional = true }
+tiktoken = { version = "0.4.0", optional = true }
 nltk = { version = "3.8.1", optional = true }
 # This will ensure that even when you run poetry install or pip install, the compatible version of numpy will always be chosen.
 # airbyte-ci will try to install latest version when --use-local-cdk is used, resulting in the conflict.
@@ -76,7 +75,7 @@ numpy = "<2"
 unstructured = { version = "0.10.27", extras = ["docx", "pptx"], optional = true }
 "unstructured.pytesseract" = { version = ">=0.3.12", optional = true }
 pyjwt = "^2.8.0"
-cryptography = ">=42.0.5,<44.0.0"
+cryptography = "^42.0.5"
 pytz = "2024.1"
 orjson = "^3.10.7"
 serpyco-rs = "^1.10.2"