airbyte-cdk 6.28.0__py3-none-any.whl → 6.28.0.dev0__py3-none-any.whl

airbyte_cdk/sources/declarative/incremental/concurrent_partition_cursor.py

@@ -22,6 +22,9 @@ from airbyte_cdk.sources.streams.checkpoint.per_partition_key_serializer import
 )
 from airbyte_cdk.sources.streams.concurrent.cursor import ConcurrentCursor, Cursor, CursorField
 from airbyte_cdk.sources.streams.concurrent.partitions.partition import Partition
+from airbyte_cdk.sources.streams.concurrent.state_converters.abstract_stream_state_converter import (
+    AbstractStreamStateConverter,
+)
 from airbyte_cdk.sources.types import Record, StreamSlice, StreamState
 
 logger = logging.getLogger("airbyte")
@@ -72,6 +75,7 @@ class ConcurrentPerPartitionCursor(Cursor):
         stream_state: Any,
         message_repository: MessageRepository,
         connector_state_manager: ConnectorStateManager,
+        connector_state_converter: AbstractStreamStateConverter,
         cursor_field: CursorField,
     ) -> None:
         self._global_cursor: Optional[StreamState] = {}
@@ -79,6 +83,7 @@ class ConcurrentPerPartitionCursor(Cursor):
         self._stream_namespace = stream_namespace
         self._message_repository = message_repository
         self._connector_state_manager = connector_state_manager
+        self._connector_state_converter = connector_state_converter
         self._cursor_field = cursor_field
 
         self._cursor_factory = cursor_factory
@@ -301,8 +306,7 @@ class ConcurrentPerPartitionCursor(Cursor):
         ):
             # We assume that `stream_state` is in a global format that can be applied to all partitions.
             # Example: {"global_state_format_key": "global_state_format_value"}
-            self._global_cursor = deepcopy(stream_state)
-            self._new_global_cursor = deepcopy(stream_state)
+            self._set_global_state(stream_state)
 
         else:
             self._use_global_cursor = stream_state.get("use_global_cursor", False)
@@ -319,8 +323,7 @@ class ConcurrentPerPartitionCursor(Cursor):
 
             # set default state for missing partitions if it is per partition with fallback to global
             if self._GLOBAL_STATE_KEY in stream_state:
-                self._global_cursor = deepcopy(stream_state[self._GLOBAL_STATE_KEY])
-                self._new_global_cursor = deepcopy(stream_state[self._GLOBAL_STATE_KEY])
+                self._set_global_state(stream_state[self._GLOBAL_STATE_KEY])
 
         # Set initial parent state
         if stream_state.get("parent_state"):
@@ -329,6 +332,27 @@ class ConcurrentPerPartitionCursor(Cursor):
         # Set parent state for partition routers based on parent streams
         self._partition_router.set_initial_state(stream_state)
 
+    def _set_global_state(self, stream_state: Mapping[str, Any]) -> None:
+        """
+        Initializes the global cursor state from the provided stream state.
+
+        If the cursor field key is present in the stream state, its value is parsed,
+        formatted, and stored as the global cursor. This ensures consistency in state
+        representation across partitions.
+        """
+        if self.cursor_field.cursor_field_key in stream_state:
+            global_state_value = stream_state[self.cursor_field.cursor_field_key]
+            final_format_global_state_value = self._connector_state_converter.output_format(
+                self._connector_state_converter.parse_value(global_state_value)
+            )
+
+            fixed_global_state = {
+                self.cursor_field.cursor_field_key: final_format_global_state_value
+            }
+
+            self._global_cursor = deepcopy(fixed_global_state)
+            self._new_global_cursor = deepcopy(fixed_global_state)
+
     def observe(self, record: Record) -> None:
         if not self._use_global_cursor and self.limit_reached():
             self._use_global_cursor = True
@@ -372,5 +396,10 @@ class ConcurrentPerPartitionCursor(Cursor):
         cursor = self._cursor_per_partition[partition_key]
         return cursor
 
+    def _extract_cursor_value_from_state(self, state: StreamState) -> Any:
+        return self._connector_state_converter.parse_value(
+            state[self.cursor_field.cursor_field_key]
+        )
+
     def limit_reached(self) -> bool:
         return self._over_limit > self.DEFAULT_MAX_PARTITIONS_NUMBER

airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py

@@ -1202,6 +1202,22 @@ class ModelToComponentFactory:
         )
         cursor_field = CursorField(interpolated_cursor_field.eval(config=config))
 
+        datetime_format = datetime_based_cursor_model.datetime_format
+
+        cursor_granularity = (
+            parse_duration(datetime_based_cursor_model.cursor_granularity)
+            if datetime_based_cursor_model.cursor_granularity
+            else None
+        )
+
+        connector_state_converter: DateTimeStreamStateConverter
+        connector_state_converter = CustomFormatConcurrentStreamStateConverter(
+            datetime_format=datetime_format,
+            input_datetime_formats=datetime_based_cursor_model.cursor_datetime_formats,
+            is_sequential_state=True,  # ConcurrentPerPartitionCursor only works with sequential state
+            cursor_granularity=cursor_granularity,
+        )
+
         # Create the cursor factory
         cursor_factory = ConcurrentCursorFactory(
             partial(
@@ -1225,6 +1241,7 @@ class ModelToComponentFactory:
             stream_state=stream_state,
             message_repository=self._message_repository,  # type: ignore
             connector_state_manager=state_manager,
+            connector_state_converter=connector_state_converter,
             cursor_field=cursor_field,
         )
 

{airbyte_cdk-6.28.0.dist-info → airbyte_cdk-6.28.0.dev0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 6.28.0
+Version: 6.28.0.dev0
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://airbyte.com
 License: MIT

{airbyte_cdk-6.28.0.dist-info → airbyte_cdk-6.28.0.dev0.dist-info}/RECORD

@@ -88,7 +88,7 @@ airbyte_cdk/sources/declarative/extractors/record_selector.py,sha256=tjNwcURmlyD
 airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py,sha256=LhqGDfX06_dDYLKsIVnwQ_nAWCln-v8PV7Wgt_QVeTI,6533
 airbyte_cdk/sources/declarative/extractors/type_transformer.py,sha256=d6Y2Rfg8pMVEEnHllfVksWZdNVOU55yk34O03dP9muY,1626
 airbyte_cdk/sources/declarative/incremental/__init__.py,sha256=U1oZKtBaEC6IACmvziY9Wzg7Z8EgF4ZuR7NwvjlB_Sk,1255
-airbyte_cdk/sources/declarative/incremental/concurrent_partition_cursor.py,sha256=ndUZaPW0MsN4_3cGpB-0Uu0otSPnmPmHgbixoJeDmjA,16660
+airbyte_cdk/sources/declarative/incremental/concurrent_partition_cursor.py,sha256=maTJX48fXXbm3qZoX6DzRVujUTeF48eeyC1ii96BXj0,17951
 airbyte_cdk/sources/declarative/incremental/datetime_based_cursor.py,sha256=_UzUnSIUsDbRgbFTXgSyZEFb4ws-KdhdQPWO8mFbV7U,22028
 airbyte_cdk/sources/declarative/incremental/declarative_cursor.py,sha256=5Bhw9VRPyIuCaD0wmmq_L3DZsa-rJgtKSEUzSd8YYD0,536
 airbyte_cdk/sources/declarative/incremental/global_substream_cursor.py,sha256=9HO-QbL9akvjq2NP7l498RwLA4iQZlBMQW1tZbt34I8,15943
@@ -115,7 +115,7 @@ airbyte_cdk/sources/declarative/parsers/custom_code_compiler.py,sha256=958MMX6_Z
 airbyte_cdk/sources/declarative/parsers/custom_exceptions.py,sha256=Rir9_z3Kcd5Es0-LChrzk-0qubAsiK_RSEnLmK2OXm8,553
 airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py,sha256=CXwTfD3wSQq3okcqwigpprbHhSURUokh4GK2OmOyKC8,9132
 airbyte_cdk/sources/declarative/parsers/manifest_reference_resolver.py,sha256=IWUOdF03o-aQn0Occo1BJCxU0Pz-QILk5L67nzw2thw,6803
-airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py,sha256=zo1FbRgLL-wA7O0rpyJ7IaRkQ6OhyP6Lq4KZGqU4ijw,125265
+airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py,sha256=x3QC5ECRX3gn9DeWwKR8ropQXPGbDN6inQueqpDkK-k,126044
 airbyte_cdk/sources/declarative/partition_routers/__init__.py,sha256=HJ-Syp3p7RpyR_OK0X_a2kSyISfu3W-PKrRI16iY0a8,957
 airbyte_cdk/sources/declarative/partition_routers/async_job_partition_router.py,sha256=n82J15S8bjeMZ5uROu--P3hnbQoxkY5v7RPHYx7g7ro,2929
 airbyte_cdk/sources/declarative/partition_routers/cartesian_product_stream_slicer.py,sha256=c5cuVFM6NFkuQqG8Z5IwkBuwDrvXZN1CunUOM_L0ezg,6892
@@ -351,9 +351,9 @@ airbyte_cdk/utils/slice_hasher.py,sha256=EDxgROHDbfG-QKQb59m7h_7crN1tRiawdf5uU7G
 airbyte_cdk/utils/spec_schema_transformations.py,sha256=-5HTuNsnDBAhj-oLeQXwpTGA0HdcjFOf2zTEMUTTg_Y,816
 airbyte_cdk/utils/stream_status_utils.py,sha256=ZmBoiy5HVbUEHAMrUONxZvxnvfV9CesmQJLDTAIWnWw,1171
 airbyte_cdk/utils/traced_exception.py,sha256=C8uIBuCL_E4WnBAOPSxBicD06JAldoN9fGsQDp463OY,6292
-airbyte_cdk-6.28.0.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
-airbyte_cdk-6.28.0.dist-info/LICENSE_SHORT,sha256=aqF6D1NcESmpn-cqsxBtszTEnHKnlsp8L4x9wAh3Nxg,55
-airbyte_cdk-6.28.0.dist-info/METADATA,sha256=EYHArwfUWLBPhnr-8IYPz0Vhh8cA3Nx5BKznR26QB2g,6010
-airbyte_cdk-6.28.0.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-airbyte_cdk-6.28.0.dist-info/entry_points.txt,sha256=fj-e3PAQvsxsQzyyq8UkG1k8spunWnD4BAH2AwlR6NM,95
-airbyte_cdk-6.28.0.dist-info/RECORD,,
+airbyte_cdk-6.28.0.dev0.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
+airbyte_cdk-6.28.0.dev0.dist-info/LICENSE_SHORT,sha256=aqF6D1NcESmpn-cqsxBtszTEnHKnlsp8L4x9wAh3Nxg,55
+airbyte_cdk-6.28.0.dev0.dist-info/METADATA,sha256=lOUTgiqxemt8MFagRsnNAwvwURmnoOV8WbRqSB97Kb8,6015
+airbyte_cdk-6.28.0.dev0.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+airbyte_cdk-6.28.0.dev0.dist-info/entry_points.txt,sha256=fj-e3PAQvsxsQzyyq8UkG1k8spunWnD4BAH2AwlR6NM,95
+airbyte_cdk-6.28.0.dev0.dist-info/RECORD,,