airbyte-cdk 6.7.0rc3__py3-none-any.whl → 6.7.1.dev0__py3-none-any.whl

airbyte_cdk/cli/source_declarative_manifest/_run.py

@@ -25,6 +25,8 @@ from datetime import datetime
 from pathlib import Path
 from typing import Any, cast
 
+from orjson import orjson
+
 from airbyte_cdk.entrypoint import AirbyteEntrypoint, launch
 from airbyte_cdk.models import (
     AirbyteErrorTraceMessage,
@@ -42,7 +44,6 @@ from airbyte_cdk.sources.declarative.concurrent_declarative_source import (
 )
 from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource
 from airbyte_cdk.sources.source import TState
-from orjson import orjson
 
 
 class SourceLocalYaml(YamlDeclarativeSource):

airbyte_cdk/config_observation.py

@@ -10,6 +10,8 @@ import time
 from copy import copy
 from typing import Any, List, MutableMapping
 
+from orjson import orjson
+
 from airbyte_cdk.models import (
     AirbyteControlConnectorConfigMessage,
     AirbyteControlMessage,
@@ -18,7 +20,6 @@ from airbyte_cdk.models import (
     OrchestratorType,
     Type,
 )
-from orjson import orjson
 
 
 class ObservedDict(dict):  # type: ignore # disallow_any_generics is set to True, and dict is equivalent to dict[Any]

airbyte_cdk/connector.py

@@ -11,6 +11,7 @@ from abc import ABC, abstractmethod
 from typing import Any, Generic, Mapping, Optional, Protocol, TypeVar
 
 import yaml
+
 from airbyte_cdk.models import (
     AirbyteConnectionStatus,
     ConnectorSpecification,

airbyte_cdk/connector_builder/connector_builder_handler.py

@@ -12,8 +12,8 @@ from airbyte_cdk.models import (
     AirbyteRecordMessage,
     AirbyteStateMessage,
     ConfiguredAirbyteCatalog,
+    Type,
 )
-from airbyte_cdk.models import Type
 from airbyte_cdk.models import Type as MessageType
 from airbyte_cdk.sources.declarative.declarative_source import DeclarativeSource
 from airbyte_cdk.sources.declarative.manifest_declarative_source import ManifestDeclarativeSource

airbyte_cdk/connector_builder/main.py

@@ -6,6 +6,8 @@
 import sys
 from typing import Any, List, Mapping, Optional, Tuple
 
+from orjson import orjson
+
 from airbyte_cdk.connector import BaseConnector
 from airbyte_cdk.connector_builder.connector_builder_handler import (
     TestReadLimits,
@@ -25,7 +27,6 @@ from airbyte_cdk.models import (
 from airbyte_cdk.sources.declarative.manifest_declarative_source import ManifestDeclarativeSource
 from airbyte_cdk.sources.source import Source
 from airbyte_cdk.utils.traced_exception import AirbyteTracedException
-from orjson import orjson
 
 
 def get_config_and_catalog_from_args(

airbyte_cdk/destinations/destination.py

@@ -9,6 +9,8 @@ import sys
 from abc import ABC, abstractmethod
 from typing import Any, Iterable, List, Mapping
 
+from orjson import orjson
+
 from airbyte_cdk.connector import Connector
 from airbyte_cdk.exception_handler import init_uncaught_exception_handler
 from airbyte_cdk.models import (
@@ -20,7 +22,6 @@ from airbyte_cdk.models import (
 )
 from airbyte_cdk.sources.utils.schema_helpers import check_config_against_spec_or_exit
 from airbyte_cdk.utils.traced_exception import AirbyteTracedException
-from orjson import orjson
 
 logger = logging.getLogger("airbyte")
 

airbyte_cdk/destinations/vector_db_based/config.py

@@ -5,9 +5,10 @@
 from typing import Any, Dict, List, Literal, Optional, Union
 
 import dpath
+from pydantic.v1 import BaseModel, Field
+
 from airbyte_cdk.utils.oneof_option_config import OneOfOptionConfig
 from airbyte_cdk.utils.spec_schema_transformations import resolve_refs
-from pydantic.v1 import BaseModel, Field
 
 
 class SeparatorSplitterConfigModel(BaseModel):

airbyte_cdk/destinations/vector_db_based/document_processor.py

@@ -8,6 +8,10 @@ from dataclasses import dataclass
 from typing import Any, Dict, List, Mapping, Optional, Tuple
 
 import dpath
+from langchain.text_splitter import Language, RecursiveCharacterTextSplitter
+from langchain.utils import stringify_dict
+from langchain_core.documents.base import Document
+
 from airbyte_cdk.destinations.vector_db_based.config import (
     ProcessingConfigModel,
     SeparatorSplitterConfigModel,
@@ -21,9 +25,6 @@ from airbyte_cdk.models import (
     DestinationSyncMode,
 )
 from airbyte_cdk.utils.traced_exception import AirbyteTracedException, FailureType
-from langchain.text_splitter import Language, RecursiveCharacterTextSplitter
-from langchain.utils import stringify_dict
-from langchain_core.documents.base import Document
 
 METADATA_STREAM_FIELD = "_ab_stream"
 METADATA_RECORD_ID_FIELD = "_ab_record_id"

airbyte_cdk/destinations/vector_db_based/embedder.py

@@ -7,6 +7,11 @@ from abc import ABC, abstractmethod
 from dataclasses import dataclass
 from typing import List, Optional, Union, cast
 
+from langchain.embeddings.cohere import CohereEmbeddings
+from langchain.embeddings.fake import FakeEmbeddings
+from langchain.embeddings.localai import LocalAIEmbeddings
+from langchain.embeddings.openai import OpenAIEmbeddings
+
 from airbyte_cdk.destinations.vector_db_based.config import (
     AzureOpenAIEmbeddingConfigModel,
     CohereEmbeddingConfigModel,
@@ -19,10 +24,6 @@ from airbyte_cdk.destinations.vector_db_based.config import (
 from airbyte_cdk.destinations.vector_db_based.utils import create_chunks, format_exception
 from airbyte_cdk.models import AirbyteRecordMessage
 from airbyte_cdk.utils.traced_exception import AirbyteTracedException, FailureType
-from langchain.embeddings.cohere import CohereEmbeddings
-from langchain.embeddings.fake import FakeEmbeddings
-from langchain.embeddings.localai import LocalAIEmbeddings
-from langchain.embeddings.openai import OpenAIEmbeddings
 
 
 @dataclass

airbyte_cdk/entrypoint.py

@@ -16,6 +16,9 @@ from typing import Any, DefaultDict, Iterable, List, Mapping, Optional
 from urllib.parse import urlparse
 
 import requests
+from orjson import orjson
+from requests import PreparedRequest, Response, Session
+
 from airbyte_cdk.connector import TConfig
 from airbyte_cdk.exception_handler import init_uncaught_exception_handler
 from airbyte_cdk.logger import init_logger
@@ -38,8 +41,6 @@ from airbyte_cdk.utils import is_cloud_environment, message_utils
 from airbyte_cdk.utils.airbyte_secrets_utils import get_secrets, update_secrets
 from airbyte_cdk.utils.constants import ENV_REQUEST_CACHE_PATH
 from airbyte_cdk.utils.traced_exception import AirbyteTracedException
-from orjson import orjson
-from requests import PreparedRequest, Response, Session
 
 logger = init_logger("airbyte")
 

airbyte_cdk/logger.py

@@ -7,6 +7,8 @@ import logging
 import logging.config
 from typing import Any, Callable, Mapping, Optional, Tuple
 
+from orjson import orjson
+
 from airbyte_cdk.models import (
     AirbyteLogMessage,
     AirbyteMessage,
@@ -15,7 +17,6 @@ from airbyte_cdk.models import (
     Type,
 )
 from airbyte_cdk.utils.airbyte_secrets_utils import filter_secrets
-from orjson import orjson
 
 LOGGING_CONFIG = {
     "version": 1,

airbyte_cdk/models/__init__.py

@@ -39,7 +39,9 @@ from .airbyte_protocol import (
     FailureType,
     Level,
     OAuthConfigSpecification,
+    OauthConnectorInputSpecification,
     OrchestratorType,
+    State,
     Status,
     StreamDescriptor,
     SyncMode,

airbyte_cdk/models/airbyte_protocol.py

@@ -5,10 +5,11 @@
 from dataclasses import InitVar, dataclass
 from typing import Annotated, Any, Dict, List, Mapping, Optional, Union
 
-from airbyte_cdk.models.file_transfer_record_message import AirbyteFileTransferRecordMessage
 from airbyte_protocol_dataclasses.models import *  # noqa: F403  # Allow '*'
 from serpyco_rs.metadata import Alias
 
+from airbyte_cdk.models.file_transfer_record_message import AirbyteFileTransferRecordMessage
+
 # ruff: noqa: F405  # ignore fuzzy import issues with 'import *'
 
 

airbyte_cdk/sources/concurrent_source/concurrent_read_processor.py

@@ -17,8 +17,8 @@ from airbyte_cdk.sources.streams.concurrent.abstract_stream import AbstractStrea
 from airbyte_cdk.sources.streams.concurrent.partition_enqueuer import PartitionEnqueuer
 from airbyte_cdk.sources.streams.concurrent.partition_reader import PartitionReader
 from airbyte_cdk.sources.streams.concurrent.partitions.partition import Partition
-from airbyte_cdk.sources.streams.concurrent.partitions.record import Record
 from airbyte_cdk.sources.streams.concurrent.partitions.types import PartitionCompleteSentinel
+from airbyte_cdk.sources.types import Record
 from airbyte_cdk.sources.utils.record_helper import stream_data_to_airbyte_message
 from airbyte_cdk.sources.utils.slice_logger import SliceLogger
 from airbyte_cdk.utils import AirbyteTracedException
@@ -147,11 +147,11 @@ class ConcurrentReadProcessor:
         # AbstractStreams are expected to return data as they are expected.
         # Any transformation on the data should be done before reaching this point
         message = stream_data_to_airbyte_message(
-            stream_name=record.partition.stream_name(),
+            stream_name=record.stream_name,
             data_or_message=record.data,
             is_file_transfer_message=record.is_file_transfer_message,
         )
-        stream = self._stream_name_to_instance[record.partition.stream_name()]
+        stream = self._stream_name_to_instance[record.stream_name]
 
         if message.type == MessageType.RECORD:
             if self._record_counter[stream.name] == 0:

airbyte_cdk/sources/concurrent_source/concurrent_source.py

@@ -18,11 +18,11 @@ from airbyte_cdk.sources.streams.concurrent.abstract_stream import AbstractStrea
 from airbyte_cdk.sources.streams.concurrent.partition_enqueuer import PartitionEnqueuer
 from airbyte_cdk.sources.streams.concurrent.partition_reader import PartitionReader
 from airbyte_cdk.sources.streams.concurrent.partitions.partition import Partition
-from airbyte_cdk.sources.streams.concurrent.partitions.record import Record
 from airbyte_cdk.sources.streams.concurrent.partitions.types import (
     PartitionCompleteSentinel,
     QueueItem,
 )
+from airbyte_cdk.sources.types import Record
 from airbyte_cdk.sources.utils.slice_logger import DebugSliceLogger, SliceLogger
 
 

airbyte_cdk/sources/config.py

@@ -4,9 +4,10 @@
 
 from typing import Any, Dict
 
-from airbyte_cdk.sources.utils.schema_helpers import expand_refs, rename_key
 from pydantic.v1 import BaseModel
 
+from airbyte_cdk.sources.utils.schema_helpers import expand_refs, rename_key
+
 
 class BaseConfig(BaseModel):
     """Base class for connector spec, adds the following behaviour:

airbyte_cdk/sources/declarative/auth/jwt.py

@@ -8,6 +8,7 @@ from datetime import datetime
 from typing import Any, Mapping, Optional, Union
 
 import jwt
+
 from airbyte_cdk.sources.declarative.auth.declarative_authenticator import DeclarativeAuthenticator
 from airbyte_cdk.sources.declarative.interpolation.interpolated_boolean import InterpolatedBoolean
 from airbyte_cdk.sources.declarative.interpolation.interpolated_mapping import InterpolatedMapping

airbyte_cdk/sources/declarative/auth/oauth.py

@@ -6,6 +6,7 @@ from dataclasses import InitVar, dataclass, field
 from typing import Any, List, Mapping, Optional, Union
 
 import pendulum
+
 from airbyte_cdk.sources.declarative.auth.declarative_authenticator import DeclarativeAuthenticator
 from airbyte_cdk.sources.declarative.interpolation.interpolated_mapping import InterpolatedMapping
 from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString

airbyte_cdk/sources/declarative/auth/selective_authenticator.py

@@ -6,6 +6,7 @@ from dataclasses import dataclass
 from typing import Any, List, Mapping
 
 import dpath
+
 from airbyte_cdk.sources.declarative.auth.declarative_authenticator import DeclarativeAuthenticator
 
 

airbyte_cdk/sources/declarative/auth/token.py

@@ -8,6 +8,8 @@ from dataclasses import InitVar, dataclass
 from typing import Any, Mapping, Union
 
 import requests
+from cachetools import TTLCache, cached
+
 from airbyte_cdk.sources.declarative.auth.declarative_authenticator import DeclarativeAuthenticator
 from airbyte_cdk.sources.declarative.auth.token_provider import TokenProvider
 from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
@@ -16,7 +18,6 @@ from airbyte_cdk.sources.declarative.requesters.request_option import (
     RequestOptionType,
 )
 from airbyte_cdk.sources.types import Config
-from cachetools import TTLCache, cached
 
 
 @dataclass

airbyte_cdk/sources/declarative/auth/token_provider.py

@@ -10,6 +10,9 @@ from typing import Any, List, Mapping, Optional, Union
 
 import dpath
 import pendulum
+from isodate import Duration
+from pendulum import DateTime
+
 from airbyte_cdk.sources.declarative.decoders.decoder import Decoder
 from airbyte_cdk.sources.declarative.decoders.json_decoder import JsonDecoder
 from airbyte_cdk.sources.declarative.exceptions import ReadException
@@ -18,8 +21,6 @@ from airbyte_cdk.sources.declarative.requesters.requester import Requester
 from airbyte_cdk.sources.http_logger import format_http_message
 from airbyte_cdk.sources.message import MessageRepository, NoopMessageRepository
 from airbyte_cdk.sources.types import Config
-from isodate import Duration
-from pendulum import DateTime
 
 
 class TokenProvider:

airbyte_cdk/sources/declarative/concurrent_declarative_source.py

@@ -3,7 +3,7 @@
 #
 
 import logging
-from typing import Any, Generic, Iterator, List, Mapping, Optional, Tuple, Union, Callable
+from typing import Any, Callable, Generic, Iterator, List, Mapping, Optional, Tuple, Union
 
 from airbyte_cdk.models import (
     AirbyteCatalog,
@@ -27,14 +27,16 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import
 )
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     DatetimeBasedCursor as DatetimeBasedCursorModel,
+)
+from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     DeclarativeStream as DeclarativeStreamModel,
 )
 from airbyte_cdk.sources.declarative.parsers.model_to_component_factory import (
-    ModelToComponentFactory,
     ComponentDefinition,
+    ModelToComponentFactory,
 )
 from airbyte_cdk.sources.declarative.requesters import HttpRequester
-from airbyte_cdk.sources.declarative.retrievers import SimpleRetriever, Retriever
+from airbyte_cdk.sources.declarative.retrievers import Retriever, SimpleRetriever
 from airbyte_cdk.sources.declarative.stream_slicers.declarative_partition_generator import (
     DeclarativePartitionFactory,
     StreamSlicerPartitionGenerator,
@@ -42,15 +44,15 @@ from airbyte_cdk.sources.declarative.stream_slicers.declarative_partition_genera
 from airbyte_cdk.sources.declarative.transformations.add_fields import AddFields
 from airbyte_cdk.sources.declarative.types import ConnectionDefinition
 from airbyte_cdk.sources.source import TState
-from airbyte_cdk.sources.types import Config, StreamState
 from airbyte_cdk.sources.streams import Stream
 from airbyte_cdk.sources.streams.concurrent.abstract_stream import AbstractStream
 from airbyte_cdk.sources.streams.concurrent.availability_strategy import (
     AlwaysAvailableAvailabilityStrategy,
 )
-from airbyte_cdk.sources.streams.concurrent.cursor import Cursor, FinalStateCursor
+from airbyte_cdk.sources.streams.concurrent.cursor import FinalStateCursor
 from airbyte_cdk.sources.streams.concurrent.default_stream import DefaultStream
 from airbyte_cdk.sources.streams.concurrent.helpers import get_primary_key_from_stream
+from airbyte_cdk.sources.types import Config, StreamState
 
 
 class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
@@ -68,6 +70,15 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
         component_factory: Optional[ModelToComponentFactory] = None,
         **kwargs: Any,
     ) -> None:
+        # To reduce the complexity of the concurrent framework, we are not enabling RFR with synthetic
+        # cursors. We do this by no longer automatically instantiating RFR cursors when converting
+        # the declarative models into runtime components. Concurrent sources will continue to checkpoint
+        # incremental streams running in full refresh.
+        component_factory = component_factory or ModelToComponentFactory(
+            emit_connector_builder_messages=emit_connector_builder_messages,
+            disable_resumable_full_refresh=True,
+        )
+
         super().__init__(
             source_config=source_config,
             debug=debug,
@@ -190,48 +201,46 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
             # these legacy Python streams the way we do low-code streams to determine if they are concurrent compatible,
             # so we need to treat them as synchronous
             if isinstance(declarative_stream, DeclarativeStream):
-                datetime_based_cursor_component_definition = name_to_stream_mapping[
+                incremental_sync_component_definition = name_to_stream_mapping[
                     declarative_stream.name
                 ].get("incremental_sync")
 
-                is_without_partition_router_nor_cursor = not bool(
-                    datetime_based_cursor_component_definition
-                ) and not (
+                partition_router_component_definition = (
                     name_to_stream_mapping[declarative_stream.name]
-                    .get("retriever", {})
+                    .get("retriever")
                     .get("partition_router")
                 )
-                is_datetime_incremental_without_partition_routing = (
-                    self._is_datetime_incremental_without_partition_routing(
-                        datetime_based_cursor_component_definition, declarative_stream
-                    )
+
+                is_substream_without_incremental = (
+                    partition_router_component_definition
+                    and not incremental_sync_component_definition
                 )
+
                 if (
-                    is_without_partition_router_nor_cursor
-                    or is_datetime_incremental_without_partition_routing
+                    incremental_sync_component_definition
+                    and incremental_sync_component_definition.get("type", "")
+                    == DatetimeBasedCursorModel.__name__
+                    and self._stream_supports_concurrent_partition_processing(
+                        declarative_stream=declarative_stream
+                    )
+                    and hasattr(declarative_stream.retriever, "stream_slicer")
+                    and isinstance(declarative_stream.retriever.stream_slicer, DatetimeBasedCursor)
                 ):
                     stream_state = state_manager.get_stream_state(
                         stream_name=declarative_stream.name, namespace=declarative_stream.namespace
                     )
 
-                    if is_datetime_incremental_without_partition_routing:
-                        cursor: Cursor = (
-                            self._constructor.create_concurrent_cursor_from_datetime_based_cursor(
-                                state_manager=state_manager,
-                                model_type=DatetimeBasedCursorModel,
-                                component_definition=datetime_based_cursor_component_definition,
-                                stream_name=declarative_stream.name,
-                                stream_namespace=declarative_stream.namespace,
-                                config=config or {},
-                                stream_state=stream_state,
-                            )
-                        )
-                    else:
-                        cursor = FinalStateCursor(
-                            declarative_stream.name,
-                            declarative_stream.namespace,
-                            self.message_repository,
+                    cursor, connector_state_converter = (
+                        self._constructor.create_concurrent_cursor_from_datetime_based_cursor(
+                            state_manager=state_manager,
+                            model_type=DatetimeBasedCursorModel,
+                            component_definition=incremental_sync_component_definition,
+                            stream_name=declarative_stream.name,
+                            stream_namespace=declarative_stream.namespace,
+                            config=config or {},
+                            stream_state=stream_state,
                         )
+                    )
 
                     partition_generator = StreamSlicerPartitionGenerator(
                         DeclarativePartitionFactory(
@@ -254,13 +263,46 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
                             json_schema=declarative_stream.get_json_schema(),
                             availability_strategy=AlwaysAvailableAvailabilityStrategy(),
                             primary_key=get_primary_key_from_stream(declarative_stream.primary_key),
-                            cursor_field=cursor.cursor_field.cursor_field_key
-                            if hasattr(cursor, "cursor_field")
-                            else None,
+                            cursor_field=cursor.cursor_field.cursor_field_key,
                             logger=self.logger,
                             cursor=cursor,
                         )
                     )
+                elif is_substream_without_incremental and hasattr(
+                    declarative_stream.retriever, "stream_slicer"
+                ):
+                    partition_generator = StreamSlicerPartitionGenerator(
+                        DeclarativePartitionFactory(
+                            declarative_stream.name,
+                            declarative_stream.get_json_schema(),
+                            self._retriever_factory(
+                                name_to_stream_mapping[declarative_stream.name],
+                                config,
+                                {},
+                            ),
+                            self.message_repository,
+                        ),
+                        declarative_stream.retriever.stream_slicer,
+                    )
+
+                    final_state_cursor = FinalStateCursor(
+                        stream_name=declarative_stream.name,
+                        stream_namespace=declarative_stream.namespace,
+                        message_repository=self.message_repository,
+                    )
+
+                    concurrent_streams.append(
+                        DefaultStream(
+                            partition_generator=partition_generator,
+                            name=declarative_stream.name,
+                            json_schema=declarative_stream.get_json_schema(),
+                            availability_strategy=AlwaysAvailableAvailabilityStrategy(),
+                            primary_key=get_primary_key_from_stream(declarative_stream.primary_key),
+                            cursor_field=None,
+                            logger=self.logger,
+                            cursor=final_state_cursor,
+                        )
+                    )
                 else:
                     synchronous_streams.append(declarative_stream)
             else:
@@ -268,22 +310,6 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
 
         return concurrent_streams, synchronous_streams
 
-    def _is_datetime_incremental_without_partition_routing(
-        self,
-        datetime_based_cursor_component_definition: Mapping[str, Any],
-        declarative_stream: DeclarativeStream,
-    ) -> bool:
-        return (
-            bool(datetime_based_cursor_component_definition)
-            and datetime_based_cursor_component_definition.get("type", "")
-            == DatetimeBasedCursorModel.__name__
-            and self._stream_supports_concurrent_partition_processing(
-                declarative_stream=declarative_stream
-            )
-            and hasattr(declarative_stream.retriever, "stream_slicer")
-            and isinstance(declarative_stream.retriever.stream_slicer, DatetimeBasedCursor)
-        )
-
     def _stream_supports_concurrent_partition_processing(
         self, declarative_stream: DeclarativeStream
     ) -> bool: