airbyte-cdk 6.26.0__py3-none-any.whl → 6.26.0.dev0__py3-none-any.whl

airbyte_cdk/sources/declarative/incremental/per_partition_cursor.py

@@ -222,8 +222,6 @@ class PerPartitionCursor(DeclarativeCursor):
         next_page_token: Optional[Mapping[str, Any]] = None,
     ) -> Mapping[str, Any]:
         if stream_slice:
-            if self._to_partition_key(stream_slice.partition) not in self._cursor_per_partition:
-                self._create_cursor_for_partition(self._to_partition_key(stream_slice.partition))
             return self._partition_router.get_request_params(  # type: ignore # this always returns a mapping
                 stream_state=stream_state,
                 stream_slice=StreamSlice(partition=stream_slice.partition, cursor_slice={}),
@@ -246,8 +244,6 @@ class PerPartitionCursor(DeclarativeCursor):
         next_page_token: Optional[Mapping[str, Any]] = None,
     ) -> Mapping[str, Any]:
         if stream_slice:
-            if self._to_partition_key(stream_slice.partition) not in self._cursor_per_partition:
-                self._create_cursor_for_partition(self._to_partition_key(stream_slice.partition))
             return self._partition_router.get_request_headers(  # type: ignore # this always returns a mapping
                 stream_state=stream_state,
                 stream_slice=StreamSlice(partition=stream_slice.partition, cursor_slice={}),
@@ -270,8 +266,6 @@ class PerPartitionCursor(DeclarativeCursor):
         next_page_token: Optional[Mapping[str, Any]] = None,
     ) -> Union[Mapping[str, Any], str]:
         if stream_slice:
-            if self._to_partition_key(stream_slice.partition) not in self._cursor_per_partition:
-                self._create_cursor_for_partition(self._to_partition_key(stream_slice.partition))
             return self._partition_router.get_request_body_data(  # type: ignore # this always returns a mapping
                 stream_state=stream_state,
                 stream_slice=StreamSlice(partition=stream_slice.partition, cursor_slice={}),
@@ -294,8 +288,6 @@ class PerPartitionCursor(DeclarativeCursor):
         next_page_token: Optional[Mapping[str, Any]] = None,
     ) -> Mapping[str, Any]:
         if stream_slice:
-            if self._to_partition_key(stream_slice.partition) not in self._cursor_per_partition:
-                self._create_cursor_for_partition(self._to_partition_key(stream_slice.partition))
             return self._partition_router.get_request_body_json(  # type: ignore # this always returns a mapping
                 stream_state=stream_state,
                 stream_slice=StreamSlice(partition=stream_slice.partition, cursor_slice={}),
@@ -311,6 +303,21 @@ class PerPartitionCursor(DeclarativeCursor):
             raise ValueError("A partition needs to be provided in order to get request body json")
 
     def should_be_synced(self, record: Record) -> bool:
+        if (
+            record.associated_slice
+            and self._to_partition_key(record.associated_slice.partition)
+            not in self._cursor_per_partition
+        ):
+            partition_state = (
+                self._state_to_migrate_from
+                if self._state_to_migrate_from
+                else self._NO_CURSOR_STATE
+            )
+            cursor = self._create_cursor(partition_state)
+
+            self._cursor_per_partition[
+                self._to_partition_key(record.associated_slice.partition)
+            ] = cursor
         return self._get_cursor(record).should_be_synced(
             self._convert_record_to_cursor_record(record)
         )
@@ -349,32 +356,8 @@ class PerPartitionCursor(DeclarativeCursor):
             )
         partition_key = self._to_partition_key(record.associated_slice.partition)
         if partition_key not in self._cursor_per_partition:
-            self._create_cursor_for_partition(partition_key)
+            raise ValueError(
+                "Invalid state as stream slices that are emitted should refer to an existing cursor"
+            )
         cursor = self._cursor_per_partition[partition_key]
         return cursor
-
-    def _create_cursor_for_partition(self, partition_key: str) -> None:
-        """
-        Dynamically creates and initializes a cursor for the specified partition.
-
-        This method is required for `ConcurrentPerPartitionCursor`. For concurrent cursors,
-        stream_slices is executed only for the concurrent cursor, so cursors per partition
-        are not created for the declarative cursor. This method ensures that a cursor is available
-        to create requests for the specified partition. The cursor is initialized
-        with the per-partition state if present in the initial state, or with the global state
-        adjusted by the lookback window, or with the state to migrate from.
-
-        Note:
-            This is a temporary workaround and should be removed once the declarative cursor
-            is decoupled from the concurrent cursor implementation.
-
-        Args:
-            partition_key (str): The unique identifier for the partition for which the cursor
-            needs to be created.
-        """
-        partition_state = (
-            self._state_to_migrate_from if self._state_to_migrate_from else self._NO_CURSOR_STATE
-        )
-        cursor = self._create_cursor(partition_state)
-
-        self._cursor_per_partition[partition_key] = cursor

airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py

@@ -2407,7 +2407,7 @@ class ModelToComponentFactory:
         if (
             not isinstance(stream_slicer, DatetimeBasedCursor)
             or type(stream_slicer) is not DatetimeBasedCursor
-        ):
+        ) and not isinstance(stream_slicer, PerPartitionWithGlobalCursor):
             # Many of the custom component implementations of DatetimeBasedCursor override get_request_params() (or other methods).
             # Because we're decoupling RequestOptionsProvider from the Cursor, custom components will eventually need to reimplement
             # their own RequestOptionsProvider. However, right now the existing StreamSlicer/Cursor still can act as the SimpleRetriever's

airbyte_cdk/sources/declarative/partition_routers/substream_partition_router.py

@@ -296,12 +296,8 @@ class SubstreamPartitionRouter(PartitionRouter):
 
         if not parent_state and incremental_dependency:
             # Attempt to retrieve child state
-            substream_state_values = list(stream_state.values())
-            substream_state = substream_state_values[0] if substream_state_values else {}
-            # Filter out per partition state. Because we pass the state to the parent stream in the format {cursor_field: substream_state}
-            if isinstance(substream_state, (list, dict)):
-                substream_state = {}
-
+            substream_state = list(stream_state.values())
+            substream_state = substream_state[0] if substream_state else {}  # type: ignore [assignment]  # Incorrect type for assignment
             parent_state = {}
 
             # Copy child state to parent streams with incremental dependencies

airbyte_cdk/sources/declarative/schema/dynamic_schema_loader.py

@@ -154,9 +154,8 @@ class DynamicSchemaLoader(SchemaLoader):
         transformed_properties = self._transform(properties, {})
 
         return {
-            "$schema": "https://json-schema.org/draft-07/schema#",
+            "$schema": "http://json-schema.org/draft-07/schema#",
             "type": "object",
-            "additionalProperties": True,
             "properties": transformed_properties,
         }
 
@@ -221,17 +220,25 @@ class DynamicSchemaLoader(SchemaLoader):
             )
 
     def _resolve_complex_type(self, complex_type: ComplexFieldType) -> Mapping[str, Any]:
-        if not complex_type.items:
-            return self._get_airbyte_type(complex_type.field_type)
-
-        field_type = self._get_airbyte_type(complex_type.field_type)
-        field_type["items"] = (
-            self._get_airbyte_type(complex_type.items)
-            if isinstance(complex_type.items, str)
-            else self._resolve_complex_type(complex_type.items)
-        )
-
-        return field_type
+        types = [complex_type]
+        resolved_type: MutableMapping[str, Any] = {}
+
+        while types:
+            current_type = types.pop()
+            if not current_type.items:
+                resolved_type = self._get_airbyte_type(current_type.field_type)
+            else:
+                field_type = self._get_airbyte_type(current_type.field_type)
+
+                if isinstance(current_type.items, str):
+                    items_type = current_type.items
+                else:
+                    types.append(current_type.items)
+                    continue  # Skip the following lines until the stack is resolved
+                field_type["items"] = self._get_airbyte_type(items_type)
+                resolved_type = field_type
+
+        return resolved_type
 
     def _replace_type_if_not_valid(
         self,

{airbyte_cdk-6.26.0.dist-info → airbyte_cdk-6.26.0.dev0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: airbyte-cdk
-Version: 6.26.0
+Version: 6.26.0.dev0
 Summary: A framework for writing Airbyte Connectors.
 License: MIT
 Keywords: airbyte,connector-development-kit,cdk

{airbyte_cdk-6.26.0.dist-info → airbyte_cdk-6.26.0.dev0.dist-info}/RECORD

@@ -92,7 +92,7 @@ airbyte_cdk/sources/declarative/incremental/concurrent_partition_cursor.py,sha25
 airbyte_cdk/sources/declarative/incremental/datetime_based_cursor.py,sha256=_UzUnSIUsDbRgbFTXgSyZEFb4ws-KdhdQPWO8mFbV7U,22028
 airbyte_cdk/sources/declarative/incremental/declarative_cursor.py,sha256=5Bhw9VRPyIuCaD0wmmq_L3DZsa-rJgtKSEUzSd8YYD0,536
 airbyte_cdk/sources/declarative/incremental/global_substream_cursor.py,sha256=9HO-QbL9akvjq2NP7l498RwLA4iQZlBMQW1tZbt34I8,15943
-airbyte_cdk/sources/declarative/incremental/per_partition_cursor.py,sha256=9IAJTCiRUXvhFFz-IhZtYh_KfAjLHqthsYf2jErQRls,17728
+airbyte_cdk/sources/declarative/incremental/per_partition_cursor.py,sha256=_FSJjAwL4Zu-i2CngnhTtx8j-NPVSBKj5LwDSPta3Cg,16305
 airbyte_cdk/sources/declarative/incremental/per_partition_with_global.py,sha256=2YBOA2NnwAeIKlIhSwUB_W-FaGnPcmrG_liY7b4mV2Y,8365
 airbyte_cdk/sources/declarative/incremental/resumable_full_refresh_cursor.py,sha256=10LFv1QPM-agVKl6eaANmEBOfd7gZgBrkoTcMggsieQ,4809
 airbyte_cdk/sources/declarative/interpolation/__init__.py,sha256=tjUJkn3B-iZ-p7RP2c3dVZejrGiQeooGmS5ibWTuUL4,437
@@ -115,14 +115,14 @@ airbyte_cdk/sources/declarative/parsers/custom_code_compiler.py,sha256=958MMX6_Z
 airbyte_cdk/sources/declarative/parsers/custom_exceptions.py,sha256=Rir9_z3Kcd5Es0-LChrzk-0qubAsiK_RSEnLmK2OXm8,553
 airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py,sha256=CXwTfD3wSQq3okcqwigpprbHhSURUokh4GK2OmOyKC8,9132
 airbyte_cdk/sources/declarative/parsers/manifest_reference_resolver.py,sha256=IWUOdF03o-aQn0Occo1BJCxU0Pz-QILk5L67nzw2thw,6803
-airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py,sha256=zfWJLlopJklDK1xvoUy2qMFcnSklmQ7wwEbdWVxYlw0,122917
+airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py,sha256=mf0Iwq0VtetpC2HYq93_hZUOuFhWUB7aPjacC6RarYk,122981
 airbyte_cdk/sources/declarative/partition_routers/__init__.py,sha256=HJ-Syp3p7RpyR_OK0X_a2kSyISfu3W-PKrRI16iY0a8,957
 airbyte_cdk/sources/declarative/partition_routers/async_job_partition_router.py,sha256=n82J15S8bjeMZ5uROu--P3hnbQoxkY5v7RPHYx7g7ro,2929
 airbyte_cdk/sources/declarative/partition_routers/cartesian_product_stream_slicer.py,sha256=c5cuVFM6NFkuQqG8Z5IwkBuwDrvXZN1CunUOM_L0ezg,6892
 airbyte_cdk/sources/declarative/partition_routers/list_partition_router.py,sha256=t7pRdFWfFWJtQQG19c9PVeMODyO2BknRTakpM5U9N-8,4844
 airbyte_cdk/sources/declarative/partition_routers/partition_router.py,sha256=YyEIzdmLd1FjbVP3QbQ2VFCLW_P-OGbVh6VpZShp54k,2218
 airbyte_cdk/sources/declarative/partition_routers/single_partition_router.py,sha256=SKzKjSyfccq4dxGIh-J6ejrgkCHzaiTIazmbmeQiRD4,1942
-airbyte_cdk/sources/declarative/partition_routers/substream_partition_router.py,sha256=pPb0blnNx598bk47Khgs0cvwhN02SWSmg7lnJKb9K6Q,15577
+airbyte_cdk/sources/declarative/partition_routers/substream_partition_router.py,sha256=5bgXoJfBg_6i53krQMptAGb50XB5XoVfqQxKQhlLtBA,15383
 airbyte_cdk/sources/declarative/requesters/README.md,sha256=eL1I4iLkxaw7hJi9S9d18_XcRl-R8lUSjqBVJJzvXmg,2656
 airbyte_cdk/sources/declarative/requesters/__init__.py,sha256=d7a3OoHbqaJDyyPli3nqqJ2yAW_SLX6XDaBAKOwvpxw,364
 airbyte_cdk/sources/declarative/requesters/error_handlers/__init__.py,sha256=SkEDcJxlT1683rNx93K9whoS0OyUukkuOfToGtgpF58,776
@@ -170,7 +170,7 @@ airbyte_cdk/sources/declarative/retrievers/retriever.py,sha256=XPLs593Xv8c5cKMc3
 airbyte_cdk/sources/declarative/retrievers/simple_retriever.py,sha256=kgnhVQxRlFqJs2-rDu2-QH-p-GzQU3nKmSp6_aq8u0s,24550
 airbyte_cdk/sources/declarative/schema/__init__.py,sha256=xU45UvM5O4c1PSM13UHpCdh5hpW3HXy9vRRGEiAC1rg,795
 airbyte_cdk/sources/declarative/schema/default_schema_loader.py,sha256=KTACrIE23a83wsm3Rd9Eb4K6-20lrGqYxTHNp9yxsso,1820
-airbyte_cdk/sources/declarative/schema/dynamic_schema_loader.py,sha256=d8tfDiDcJiunvN_Yalyfx5ISY5A-iIW3HbPwX2Hagh4,10702
+airbyte_cdk/sources/declarative/schema/dynamic_schema_loader.py,sha256=QG0ApZFFPVZyG8ZsdVJ5vx1vrPb7QTUdBJY8VJIaPlQ,11022
 airbyte_cdk/sources/declarative/schema/inline_schema_loader.py,sha256=bVETE10hRsatRJq3R3BeyRR0wIoK3gcP1gcpVRQ_P5U,464
 airbyte_cdk/sources/declarative/schema/json_file_schema_loader.py,sha256=5Wl-fqW-pVf_dxJ4yGHMAFfC4JjKHYJhqFJT1xA57F4,4177
 airbyte_cdk/sources/declarative/schema/schema_loader.py,sha256=kjt8v0N5wWKA5zyLnrDLxf1PJKdUqvQq2RVnAOAzNSY,379
@@ -350,8 +350,8 @@ airbyte_cdk/utils/slice_hasher.py,sha256=-pHexlNYoWYPnXNH-M7HEbjmeJe9Zk7SJijdQ7d
 airbyte_cdk/utils/spec_schema_transformations.py,sha256=-5HTuNsnDBAhj-oLeQXwpTGA0HdcjFOf2zTEMUTTg_Y,816
 airbyte_cdk/utils/stream_status_utils.py,sha256=ZmBoiy5HVbUEHAMrUONxZvxnvfV9CesmQJLDTAIWnWw,1171
 airbyte_cdk/utils/traced_exception.py,sha256=C8uIBuCL_E4WnBAOPSxBicD06JAldoN9fGsQDp463OY,6292
-airbyte_cdk-6.26.0.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
-airbyte_cdk-6.26.0.dist-info/METADATA,sha256=tzGmGFZqFNZ_5jEwuijPrrzH5z1d5ITGm2VdZV9TzTk,5996
-airbyte_cdk-6.26.0.dist-info/WHEEL,sha256=IYZQI976HJqqOpQU6PHkJ8fb3tMNBFjg-Cn-pwAbaFM,88
-airbyte_cdk-6.26.0.dist-info/entry_points.txt,sha256=fj-e3PAQvsxsQzyyq8UkG1k8spunWnD4BAH2AwlR6NM,95
-airbyte_cdk-6.26.0.dist-info/RECORD,,
+airbyte_cdk-6.26.0.dev0.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
+airbyte_cdk-6.26.0.dev0.dist-info/METADATA,sha256=g4Da91E4C4Ye-yiU-Rmg7E52t4O0pHx_w4KVJ2meuBc,6001
+airbyte_cdk-6.26.0.dev0.dist-info/WHEEL,sha256=IYZQI976HJqqOpQU6PHkJ8fb3tMNBFjg-Cn-pwAbaFM,88
+airbyte_cdk-6.26.0.dev0.dist-info/entry_points.txt,sha256=fj-e3PAQvsxsQzyyq8UkG1k8spunWnD4BAH2AwlR6NM,95
+airbyte_cdk-6.26.0.dev0.dist-info/RECORD,,