airbyte-cdk 0.67.1__tar.gz → 0.67.3__tar.gz

{airbyte-cdk-0.67.1 → airbyte-cdk-0.67.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 0.67.1
+Version: 0.67.3
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://github.com/airbytehq/airbyte
 Author: Airbyte
@@ -48,7 +48,7 @@ Requires-Dist: pytest-mock; extra == "dev"
 Requires-Dist: requests-mock; extra == "dev"
 Requires-Dist: pytest-httpserver; extra == "dev"
 Requires-Dist: pandas==2.0.3; extra == "dev"
-Requires-Dist: pyarrow==12.0.1; extra == "dev"
+Requires-Dist: pyarrow~=15.0.0; extra == "dev"
 Requires-Dist: langchain==0.0.271; extra == "dev"
 Requires-Dist: openai[embeddings]==0.27.9; extra == "dev"
 Requires-Dist: cohere==4.21; extra == "dev"
@@ -66,7 +66,7 @@ Requires-Dist: sphinx-rtd-theme~=1.0; extra == "sphinx-docs"
 Provides-Extra: file-based
 Requires-Dist: avro~=1.11.2; extra == "file-based"
 Requires-Dist: fastavro~=1.8.0; extra == "file-based"
-Requires-Dist: pyarrow==12.0.1; extra == "file-based"
+Requires-Dist: pyarrow~=15.0.0; extra == "file-based"
 Requires-Dist: unstructured==0.10.27; extra == "file-based"
 Requires-Dist: unstructured[docx,pptx]==0.10.27; extra == "file-based"
 Requires-Dist: pdf2image==1.16.3; extra == "file-based"

{airbyte-cdk-0.67.1 → airbyte-cdk-0.67.3}/airbyte_cdk/sources/abstract_source.py

@@ -23,7 +23,7 @@ from airbyte_cdk.models import Type as MessageType
 from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager
 from airbyte_cdk.sources.message import InMemoryMessageRepository, MessageRepository
 from airbyte_cdk.sources.source import Source
-from airbyte_cdk.sources.streams import Stream
+from airbyte_cdk.sources.streams import FULL_REFRESH_SENTINEL_STATE_KEY, Stream
 from airbyte_cdk.sources.streams.core import StreamData
 from airbyte_cdk.sources.streams.http.http import HttpStream
 from airbyte_cdk.sources.utils.record_helper import stream_data_to_airbyte_message
@@ -181,10 +181,6 @@ class AbstractSource(Source, ABC):
     def raise_exception_on_missing_stream(self) -> bool:
         return True
 
-    @property
-    def per_stream_state_enabled(self) -> bool:
-        return True
-
     def _read_stream(
         self,
         logger: logging.Logger,
@@ -206,22 +202,32 @@ class AbstractSource(Source, ABC):
         )
         stream_instance.log_stream_sync_configuration()
 
-        use_incremental = configured_stream.sync_mode == SyncMode.incremental and stream_instance.supports_incremental
-        if use_incremental:
-            record_iterator = self._read_incremental(
-                logger,
-                stream_instance,
-                configured_stream,
-                state_manager,
-                internal_config,
-            )
-        else:
-            record_iterator = self._read_full_refresh(logger, stream_instance, configured_stream, internal_config)
+        stream_name = configured_stream.stream.name
+        # The platform always passes stream state regardless of sync mode. We shouldn't need to consider this case within the
+        # connector, but right now we need to prevent accidental usage of the previous stream state
+        stream_state = (
+            state_manager.get_stream_state(stream_name, stream_instance.namespace)
+            if configured_stream.sync_mode == SyncMode.incremental
+            else {}
+        )
+
+        if stream_state and "state" in dir(stream_instance) and not self._stream_state_is_full_refresh(stream_state):
+            stream_instance.state = stream_state  # type: ignore # we check that state in the dir(stream_instance)
+            logger.info(f"Setting state of {self.name} stream to {stream_state}")
+
+        record_iterator = stream_instance.read(
+            configured_stream,
+            logger,
+            self._slice_logger,
+            stream_state,
+            state_manager,
+            internal_config,
+        )
 
         record_counter = 0
-        stream_name = configured_stream.stream.name
         logger.info(f"Syncing stream: {stream_name} ")
-        for record in record_iterator:
+        for record_data_or_message in record_iterator:
+            record = self._get_message(record_data_or_message, stream_instance)
             if record.type == MessageType.RECORD:
                 record_counter += 1
                 if record_counter == 1:
@@ -233,62 +239,11 @@ class AbstractSource(Source, ABC):
 
         logger.info(f"Read {record_counter} records from {stream_name} stream")
 
-    def _read_incremental(
-        self,
-        logger: logging.Logger,
-        stream_instance: Stream,
-        configured_stream: ConfiguredAirbyteStream,
-        state_manager: ConnectorStateManager,
-        internal_config: InternalConfig,
-    ) -> Iterator[AirbyteMessage]:
-        """Read stream using incremental algorithm
-
-        :param logger:
-        :param stream_instance:
-        :param configured_stream:
-        :param state_manager:
-        :param internal_config:
-        :return:
-        """
-        stream_name = configured_stream.stream.name
-        stream_state = state_manager.get_stream_state(stream_name, stream_instance.namespace)
-
-        if stream_state and "state" in dir(stream_instance):
-            stream_instance.state = stream_state  # type: ignore # we check that state in the dir(stream_instance)
-            logger.info(f"Setting state of {self.name} stream to {stream_state}")
-
-        for record_data_or_message in stream_instance.read_incremental(
-            configured_stream.cursor_field,
-            logger,
-            self._slice_logger,
-            stream_state,
-            state_manager,
-            self.per_stream_state_enabled,
-            internal_config,
-        ):
-            yield self._get_message(record_data_or_message, stream_instance)
-
     def _emit_queued_messages(self) -> Iterable[AirbyteMessage]:
         if self.message_repository:
             yield from self.message_repository.consume_queue()
         return
 
-    def _read_full_refresh(
-        self,
-        logger: logging.Logger,
-        stream_instance: Stream,
-        configured_stream: ConfiguredAirbyteStream,
-        internal_config: InternalConfig,
-    ) -> Iterator[AirbyteMessage]:
-        total_records_counter = 0
-        for record_data_or_message in stream_instance.read_full_refresh(configured_stream.cursor_field, logger, self._slice_logger):
-            message = self._get_message(record_data_or_message, stream_instance)
-            yield message
-            if message.type == MessageType.RECORD:
-                total_records_counter += 1
-                if internal_config.is_limit_reached(total_records_counter):
-                    return
-
     def _get_message(self, record_data_or_message: Union[StreamData, AirbyteMessage], stream: Stream) -> AirbyteMessage:
         """
         Converts the input to an AirbyteMessage if it is a StreamData. Returns the input as is if it is already an AirbyteMessage
@@ -317,3 +272,9 @@ class AbstractSource(Source, ABC):
     def _generate_failed_streams_error_message(stream_failures: Mapping[str, AirbyteTracedException]) -> str:
         failures = ", ".join([f"{stream}: {filter_secrets(exception.__repr__())}" for stream, exception in stream_failures.items()])
         return f"During the sync, the following streams did not sync successfully: {failures}"
+
+    @staticmethod
+    def _stream_state_is_full_refresh(stream_state: Mapping[str, Any]) -> bool:
+        # For full refresh syncs that don't have a suitable cursor value, we emit a state that contains a sentinel key.
+        # This key is never used by a connector and is needed during a read to skip assigning the incoming state.
+        return FULL_REFRESH_SENTINEL_STATE_KEY in stream_state

{airbyte-cdk-0.67.1 → airbyte-cdk-0.67.3}/airbyte_cdk/sources/connector_state_manager.py

@@ -77,7 +77,7 @@ class ConnectorStateManager:
         stream_descriptor = HashableStreamDescriptor(name=stream_name, namespace=namespace)
         self.per_stream_states[stream_descriptor] = AirbyteStateBlob.parse_obj(value)
 
-    def create_state_message(self, stream_name: str, namespace: Optional[str], send_per_stream_state: bool) -> AirbyteMessage:
+    def create_state_message(self, stream_name: str, namespace: Optional[str]) -> AirbyteMessage:
         """
         Generates an AirbyteMessage using the current per-stream state of a specified stream in either the per-stream or legacy format
         :param stream_name: The name of the stream for the message that is being created
@@ -85,25 +85,18 @@ class ConnectorStateManager:
         :param send_per_stream_state: Decides which state format the message should be generated as
         :return: The Airbyte state message to be emitted by the connector during a sync
         """
-        if send_per_stream_state:
-            hashable_descriptor = HashableStreamDescriptor(name=stream_name, namespace=namespace)
-            stream_state = self.per_stream_states.get(hashable_descriptor) or AirbyteStateBlob()
-
-            # According to the Airbyte protocol, the StreamDescriptor namespace field is not required. However, the platform will throw
-            # a validation error if it receives namespace=null. That is why if namespace is None, the field should be omitted instead.
-            stream_descriptor = (
-                StreamDescriptor(name=stream_name) if namespace is None else StreamDescriptor(name=stream_name, namespace=namespace)
-            )
-
-            return AirbyteMessage(
-                type=MessageType.STATE,
-                state=AirbyteStateMessage(
-                    type=AirbyteStateType.STREAM,
-                    stream=AirbyteStreamState(stream_descriptor=stream_descriptor, stream_state=stream_state),
-                    data=dict(self._get_legacy_state()),
+        hashable_descriptor = HashableStreamDescriptor(name=stream_name, namespace=namespace)
+        stream_state = self.per_stream_states.get(hashable_descriptor) or AirbyteStateBlob()
+
+        return AirbyteMessage(
+            type=MessageType.STATE,
+            state=AirbyteStateMessage(
+                type=AirbyteStateType.STREAM,
+                stream=AirbyteStreamState(
+                    stream_descriptor=StreamDescriptor(name=stream_name, namespace=namespace), stream_state=stream_state
                 ),
-            )
-        return AirbyteMessage(type=MessageType.STATE, state=AirbyteStateMessage(data=dict(self._get_legacy_state())))
+            ),
+        )
 
     @classmethod
     def _extract_from_state_message(
@@ -176,13 +169,6 @@ class ConnectorStateManager:
             streams[stream_descriptor] = AirbyteStateBlob.parse_obj(state_value or {})
         return streams
 
-    def _get_legacy_state(self) -> Mapping[str, Any]:
-        """
-        Using the current per-stream state, creates a mapping of all the stream states for the connector being synced
-        :return: A deep copy of the mapping of stream name to stream state value
-        """
-        return {descriptor.name: state.dict() if state else {} for descriptor, state in self.per_stream_states.items()}
-
     @staticmethod
     def _is_legacy_dict_state(state: Union[List[AirbyteStateMessage], MutableMapping[str, Any]]) -> bool:
         return isinstance(state, dict)

{airbyte-cdk-0.67.1 → airbyte-cdk-0.67.3}/airbyte_cdk/sources/file_based/file_types/parquet_parser.py

@@ -5,7 +5,7 @@
 import json
 import logging
 import os
-from typing import Any, Dict, Iterable, List, Mapping, Optional, Tuple
+from typing import Any, Dict, Iterable, List, Mapping, Optional, Tuple, Union
 from urllib.parse import unquote
 
 import pyarrow as pa
@@ -16,7 +16,7 @@ from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFile
 from airbyte_cdk.sources.file_based.file_types.file_type_parser import FileTypeParser
 from airbyte_cdk.sources.file_based.remote_file import RemoteFile
 from airbyte_cdk.sources.file_based.schema_helpers import SchemaType
-from pyarrow import Scalar
+from pyarrow import DictionaryArray, Scalar
 
 
 class ParquetParser(FileTypeParser):
@@ -95,10 +95,23 @@ class ParquetParser(FileTypeParser):
         return FileReadMode.READ_BINARY
 
     @staticmethod
-    def _to_output_value(parquet_value: Scalar, parquet_format: ParquetFormat) -> Any:
+    def _to_output_value(parquet_value: Union[Scalar, DictionaryArray], parquet_format: ParquetFormat) -> Any:
+        """
+        Convert an entry in a pyarrow table to a value that can be output by the source.
+        """
+        if isinstance(parquet_value, DictionaryArray):
+            return ParquetParser._dictionary_array_to_python_value(parquet_value)
+        else:
+            return ParquetParser._scalar_to_python_value(parquet_value, parquet_format)
+
+    @staticmethod
+    def _scalar_to_python_value(parquet_value: Scalar, parquet_format: ParquetFormat) -> Any:
         """
         Convert a pyarrow scalar to a value that can be output by the source.
         """
+        if parquet_value.as_py() is None:
+            return None
+
         # Convert date and datetime objects to isoformat strings
         if pa.types.is_time(parquet_value.type) or pa.types.is_timestamp(parquet_value.type) or pa.types.is_date(parquet_value.type):
             return parquet_value.as_py().isoformat()
@@ -109,23 +122,14 @@ class ParquetParser(FileTypeParser):
 
         # Decode binary strings to utf-8
         if ParquetParser._is_binary(parquet_value.type):
-            py_value = parquet_value.as_py()
-            if py_value is None:
-                return py_value
-            return py_value.decode("utf-8")
+            return parquet_value.as_py().decode("utf-8")
+
         if pa.types.is_decimal(parquet_value.type):
             if parquet_format.decimal_as_float:
                 return parquet_value.as_py()
             else:
                 return str(parquet_value.as_py())
 
-        # Dictionaries are stored as two columns: indices and values
-        # The indices column is an array of integers that maps to the values column
-        if pa.types.is_dictionary(parquet_value.type):
-            return {
-                "indices": parquet_value.indices.tolist(),
-                "values": parquet_value.dictionary.tolist(),
-            }
         if pa.types.is_map(parquet_value.type):
             return {k: v for k, v in parquet_value.as_py()}
 
@@ -149,6 +153,20 @@ class ParquetParser(FileTypeParser):
         else:
             return parquet_value.as_py()
 
+    @staticmethod
+    def _dictionary_array_to_python_value(parquet_value: DictionaryArray) -> Dict[str, Any]:
+        """
+        Convert a pyarrow dictionary array to a value that can be output by the source.
+
+        Dictionaries are stored as two columns: indices and values
+        The indices column is an array of integers that maps to the values column
+        """
+
+        return {
+            "indices": parquet_value.indices.tolist(),
+            "values": parquet_value.dictionary.tolist(),
+        }
+
     @staticmethod
     def parquet_type_to_schema_type(parquet_type: pa.DataType, parquet_format: ParquetFormat) -> Mapping[str, str]:
         """

{airbyte-cdk-0.67.1 → airbyte-cdk-0.67.3}/airbyte_cdk/sources/file_based/stream/concurrent/adapters.py

@@ -7,7 +7,7 @@ import logging
 from functools import lru_cache
 from typing import TYPE_CHECKING, Any, Iterable, List, Mapping, MutableMapping, Optional, Union
 
-from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, Level, SyncMode, Type
+from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, ConfiguredAirbyteStream, Level, SyncMode, Type
 from airbyte_cdk.sources import AbstractSource
 from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager
 from airbyte_cdk.sources.file_based.availability_strategy import (
@@ -156,29 +156,13 @@ class FileBasedStreamFacade(AbstractStreamFacade[DefaultStream], AbstractFileBas
     def get_underlying_stream(self) -> DefaultStream:
         return self._abstract_stream
 
-    def read_full_refresh(
+    def read(
         self,
-        cursor_field: Optional[List[str]],
-        logger: logging.Logger,
-        slice_logger: SliceLogger,
-    ) -> Iterable[StreamData]:
-        """
-        Read full refresh. Delegate to the underlying AbstractStream, ignoring all the parameters
-        :param cursor_field: (ignored)
-        :param logger: (ignored)
-        :param slice_logger: (ignored)
-        :return: Iterable of StreamData
-        """
-        yield from self._read_records()
-
-    def read_incremental(
-        self,
-        cursor_field: Optional[List[str]],
+        configured_stream: ConfiguredAirbyteStream,
         logger: logging.Logger,
         slice_logger: SliceLogger,
         stream_state: MutableMapping[str, Any],
         state_manager: ConnectorStateManager,
-        per_stream_state_enabled: bool,
         internal_config: InternalConfig,
     ) -> Iterable[StreamData]:
         yield from self._read_records()

{airbyte-cdk-0.67.1 → airbyte-cdk-0.67.3}/airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_concurrent_cursor.py

@@ -155,9 +155,7 @@ class FileBasedConcurrentCursor(AbstractConcurrentFileBasedCursor):
                 self._stream_namespace,
                 new_state,
             )
-            state_message = self._connector_state_manager.create_state_message(
-                self._stream_name, self._stream_namespace, send_per_stream_state=True
-            )
+            state_message = self._connector_state_manager.create_state_message(self._stream_name, self._stream_namespace)
             self._message_repository.emit_message(state_message)
 
     def _get_new_cursor_value(self) -> str:

airbyte-cdk-0.67.3/airbyte_cdk/sources/streams/__init__.py

@@ -0,0 +1,8 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+# Initialize Streams Package
+from .core import FULL_REFRESH_SENTINEL_STATE_KEY, IncrementalMixin, Stream
+
+__all__ = ["FULL_REFRESH_SENTINEL_STATE_KEY", "IncrementalMixin", "Stream"]

{airbyte-cdk-0.67.1 → airbyte-cdk-0.67.3}/airbyte_cdk/sources/streams/concurrent/adapters.py

@@ -8,7 +8,7 @@ import logging
 from functools import lru_cache
 from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple, Union
 
-from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, AirbyteStream, Level, SyncMode, Type
+from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, AirbyteStream, ConfiguredAirbyteStream, Level, SyncMode, Type
 from airbyte_cdk.sources import AbstractSource, Source
 from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager
 from airbyte_cdk.sources.message import MessageRepository
@@ -116,29 +116,13 @@ class StreamFacade(AbstractStreamFacade[DefaultStream], Stream):
         self._slice_logger = slice_logger
         self._logger = logger
 
-    def read_full_refresh(
+    def read(
         self,
-        cursor_field: Optional[List[str]],
-        logger: logging.Logger,
-        slice_logger: SliceLogger,
-    ) -> Iterable[StreamData]:
-        """
-        Read full refresh. Delegate to the underlying AbstractStream, ignoring all the parameters
-        :param cursor_field: (ignored)
-        :param logger: (ignored)
-        :param slice_logger: (ignored)
-        :return: Iterable of StreamData
-        """
-        yield from self._read_records()
-
-    def read_incremental(
-        self,
-        cursor_field: Optional[List[str]],
+        configured_stream: ConfiguredAirbyteStream,
         logger: logging.Logger,
         slice_logger: SliceLogger,
         stream_state: MutableMapping[str, Any],
         state_manager: ConnectorStateManager,
-        per_stream_state_enabled: bool,
         internal_config: InternalConfig,
     ) -> Iterable[StreamData]:
         yield from self._read_records()

{airbyte-cdk-0.67.1 → airbyte-cdk-0.67.3}/airbyte_cdk/sources/streams/concurrent/cursor.py

@@ -184,9 +184,7 @@ class ConcurrentCursor(Cursor):
         # TODO: if we migrate stored state to the concurrent state format
         #  (aka stop calling self._connector_state_converter.convert_to_sequential_state`), we'll need to cast datetimes to string or
         #  int before emitting state
-        state_message = self._connector_state_manager.create_state_message(
-            self._stream_name, self._stream_namespace, send_per_stream_state=True
-        )
+        state_message = self._connector_state_manager.create_state_message(self._stream_name, self._stream_namespace)
         self._message_repository.emit_message(state_message)
 
     def _merge_partitions(self) -> None:

{airbyte-cdk-0.67.1 → airbyte-cdk-0.67.3}/airbyte_cdk/sources/streams/core.py

@@ -11,7 +11,7 @@ from functools import lru_cache
 from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple, Union
 
 import airbyte_cdk.sources.utils.casing as casing
-from airbyte_cdk.models import AirbyteMessage, AirbyteStream, SyncMode
+from airbyte_cdk.models import AirbyteMessage, AirbyteStream, ConfiguredAirbyteStream, SyncMode
 from airbyte_cdk.models import Type as MessageType
 
 # list of all possible HTTP methods which can be used for sending of request bodies
@@ -31,6 +31,10 @@ StreamData = Union[Mapping[str, Any], AirbyteMessage]
 
 JsonSchema = Mapping[str, Any]
 
+# Streams that only support full refresh don't have a suitable cursor so this sentinel
+# value is used to indicate that stream should not load the incoming state value
+FULL_REFRESH_SENTINEL_STATE_KEY = "__ab_full_refresh_state_message"
+
 
 def package_name_from_class(cls: object) -> str:
     """Find the package name given a class name"""
@@ -107,39 +111,24 @@ class Stream(ABC):
         """
         return None
 
-    def read_full_refresh(
-        self,
-        cursor_field: Optional[List[str]],
-        logger: logging.Logger,
-        slice_logger: SliceLogger,
-    ) -> Iterable[StreamData]:
-        slices = self.stream_slices(sync_mode=SyncMode.full_refresh, cursor_field=cursor_field)
-        logger.debug(f"Processing stream slices for {self.name} (sync_mode: full_refresh)", extra={"stream_slices": slices})
-        for _slice in slices:
-            if slice_logger.should_log_slice_message(logger):
-                yield slice_logger.create_slice_log_message(_slice)
-            yield from self.read_records(
-                stream_slice=_slice,
-                sync_mode=SyncMode.full_refresh,
-                cursor_field=cursor_field,
-            )
-
-    def read_incremental(  # type: ignore  # ignoring typing for ConnectorStateManager because of circular dependencies
+    def read(  # type: ignore  # ignoring typing for ConnectorStateManager because of circular dependencies
         self,
-        cursor_field: Optional[List[str]],
+        configured_stream: ConfiguredAirbyteStream,
         logger: logging.Logger,
         slice_logger: SliceLogger,
         stream_state: MutableMapping[str, Any],
         state_manager,
-        per_stream_state_enabled: bool,
         internal_config: InternalConfig,
     ) -> Iterable[StreamData]:
+        sync_mode = configured_stream.sync_mode
+        cursor_field = configured_stream.cursor_field
+
         slices = self.stream_slices(
             cursor_field=cursor_field,
-            sync_mode=SyncMode.incremental,
+            sync_mode=sync_mode,  # todo: change this interface to no longer rely on sync_mode for behavior
             stream_state=stream_state,
         )
-        logger.debug(f"Processing stream slices for {self.name} (sync_mode: incremental)", extra={"stream_slices": slices})
+        logger.debug(f"Processing stream slices for {self.name} (sync_mode: {sync_mode.name})", extra={"stream_slices": slices})
 
         has_slices = False
         record_counter = 0
@@ -148,7 +137,7 @@ class Stream(ABC):
             if slice_logger.should_log_slice_message(logger):
                 yield slice_logger.create_slice_log_message(_slice)
             records = self.read_records(
-                sync_mode=SyncMode.incremental,
+                sync_mode=sync_mode,  # todo: change this interface to no longer rely on sync_mode for behavior
                 stream_slice=_slice,
                 stream_state=stream_state,
                 cursor_field=cursor_field or None,
@@ -160,20 +149,34 @@ class Stream(ABC):
                 ):
                     record_data = record_data_or_message if isinstance(record_data_or_message, Mapping) else record_data_or_message.record
                     stream_state = self.get_updated_state(stream_state, record_data)
-                    checkpoint_interval = self.state_checkpoint_interval
                     record_counter += 1
-                    if checkpoint_interval and record_counter % checkpoint_interval == 0:
-                        yield self._checkpoint_state(stream_state, state_manager, per_stream_state_enabled)
+
+                    if sync_mode == SyncMode.incremental:
+                        # Checkpoint intervals are a bit controversial, but see below comment about why we're gating it right now
+                        checkpoint_interval = self.state_checkpoint_interval
+                        if checkpoint_interval and record_counter % checkpoint_interval == 0:
+                            airbyte_state_message = self._checkpoint_state(stream_state, state_manager)
+                            yield airbyte_state_message
 
                     if internal_config.is_limit_reached(record_counter):
                         break
 
-            yield self._checkpoint_state(stream_state, state_manager, per_stream_state_enabled)
+            if sync_mode == SyncMode.incremental:
+                # Even though right now, only incremental streams running as incremental mode will emit periodic checkpoints. Rather than
+                # overhaul how refresh interacts with the platform, this positions the code so that once we want to start emitting
+                # periodic checkpoints in full refresh mode it can be done here
+                airbyte_state_message = self._checkpoint_state(stream_state, state_manager)
+                yield airbyte_state_message
+
+        if not has_slices or sync_mode == SyncMode.full_refresh:
+            if sync_mode == SyncMode.full_refresh:
+                # We use a dummy state if there is no suitable value provided by full_refresh streams that do not have a valid cursor.
+                # Incremental streams running full_refresh mode emit a meaningful state
+                stream_state = stream_state or {FULL_REFRESH_SENTINEL_STATE_KEY: True}
 
-        if not has_slices:
-            # Safety net to ensure we always emit at least one state message even if there are no slices
-            checkpoint = self._checkpoint_state(stream_state, state_manager, per_stream_state_enabled)
-            yield checkpoint
+            # We should always emit a final state message for full refresh sync or streams that do not have any slices
+            airbyte_state_message = self._checkpoint_state(stream_state, state_manager)
+            yield airbyte_state_message
 
     @abstractmethod
     def read_records(
@@ -361,7 +364,6 @@ class Stream(ABC):
         self,
         stream_state: Mapping[str, Any],
         state_manager,
-        per_stream_state_enabled: bool,
     ) -> AirbyteMessage:
         # First attempt to retrieve the current state using the stream's state property. We receive an AttributeError if the state
         # property is not implemented by the stream instance and as a fallback, use the stream_state retrieved from the stream
@@ -373,4 +375,4 @@ class Stream(ABC):
 
         except AttributeError:
             state_manager.update_state_for_stream(self.name, self.namespace, stream_state)
-        return state_manager.create_state_message(self.name, self.namespace, send_per_stream_state=per_stream_state_enabled)
+        return state_manager.create_state_message(self.name, self.namespace)

{airbyte-cdk-0.67.1 → airbyte-cdk-0.67.3}/airbyte_cdk.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 0.67.1
+Version: 0.67.3
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://github.com/airbytehq/airbyte
 Author: Airbyte
@@ -48,7 +48,7 @@ Requires-Dist: pytest-mock; extra == "dev"
 Requires-Dist: requests-mock; extra == "dev"
 Requires-Dist: pytest-httpserver; extra == "dev"
 Requires-Dist: pandas==2.0.3; extra == "dev"
-Requires-Dist: pyarrow==12.0.1; extra == "dev"
+Requires-Dist: pyarrow~=15.0.0; extra == "dev"
 Requires-Dist: langchain==0.0.271; extra == "dev"
 Requires-Dist: openai[embeddings]==0.27.9; extra == "dev"
 Requires-Dist: cohere==4.21; extra == "dev"
@@ -66,7 +66,7 @@ Requires-Dist: sphinx-rtd-theme~=1.0; extra == "sphinx-docs"
 Provides-Extra: file-based
 Requires-Dist: avro~=1.11.2; extra == "file-based"
 Requires-Dist: fastavro~=1.8.0; extra == "file-based"
-Requires-Dist: pyarrow==12.0.1; extra == "file-based"
+Requires-Dist: pyarrow~=15.0.0; extra == "file-based"
 Requires-Dist: unstructured==0.10.27; extra == "file-based"
 Requires-Dist: unstructured[docx,pptx]==0.10.27; extra == "file-based"
 Requires-Dist: pdf2image==1.16.3; extra == "file-based"

{airbyte-cdk-0.67.1 → airbyte-cdk-0.67.3}/airbyte_cdk.egg-info/requires.txt

@@ -28,7 +28,7 @@ pytest-mock
 requests-mock
 pytest-httpserver
 pandas==2.0.3
-pyarrow==12.0.1
+pyarrow~=15.0.0
 langchain==0.0.271
 openai[embeddings]==0.27.9
 cohere==4.21
@@ -44,7 +44,7 @@ markdown
 [file-based]
 avro~=1.11.2
 fastavro~=1.8.0
-pyarrow==12.0.1
+pyarrow~=15.0.0
 unstructured==0.10.27
 unstructured[docx,pptx]==0.10.27
 pdf2image==1.16.3

{airbyte-cdk-0.67.1 → airbyte-cdk-0.67.3}/setup.py

@@ -15,7 +15,7 @@ README = (HERE / "README.md").read_text()
 
 avro_dependency = "avro~=1.11.2"
 fastavro_dependency = "fastavro~=1.8.0"
-pyarrow_dependency = "pyarrow==12.0.1"
+pyarrow_dependency = "pyarrow~=15.0.0"
 
 langchain_dependency = "langchain==0.0.271"
 openai_dependency = "openai[embeddings]==0.27.9"
@@ -36,7 +36,7 @@ setup(
     name="airbyte-cdk",
     # The version of the airbyte-cdk package is used at runtime to validate manifests. That validation must be
     # updated if our semver format changes such as using release candidate versions.
-    version="0.67.1",
+    version="0.67.3",
     description="A framework for writing Airbyte Connectors.",
     long_description=README,
     long_description_content_type="text/markdown",

{airbyte-cdk-0.67.1 → airbyte-cdk-0.67.3}/unit_tests/sources/concurrent_source/test_concurrent_source_adapter.py

@@ -81,13 +81,14 @@ def test_concurrent_source_adapter():
 def _mock_stream(name: str, data=[], available: bool = True):
     s = Mock()
     s.name = name
+    s.namespace = None
     s.as_airbyte_stream.return_value = AirbyteStream(
         name=name,
         json_schema={},
         supported_sync_modes=[SyncMode.full_refresh],
     )
     s.check_availability.return_value = (True, None) if available else (False, "not available")
-    s.read_full_refresh.return_value = iter(data)
+    s.read.return_value = iter(data)
     s.primary_key = None
     return s