airbyte-cdk 6.26.0__tar.gz → 6.26.0.dev0__tar.gz

{airbyte_cdk-6.26.0 → airbyte_cdk-6.26.0.dev0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: airbyte-cdk
-Version: 6.26.0
+Version: 6.26.0.dev0
 Summary: A framework for writing Airbyte Connectors.
 License: MIT
 Keywords: airbyte,connector-development-kit,cdk

{airbyte_cdk-6.26.0 → airbyte_cdk-6.26.0.dev0}/airbyte_cdk/sources/declarative/incremental/per_partition_cursor.py

@@ -222,8 +222,6 @@ class PerPartitionCursor(DeclarativeCursor):
         next_page_token: Optional[Mapping[str, Any]] = None,
     ) -> Mapping[str, Any]:
         if stream_slice:
-            if self._to_partition_key(stream_slice.partition) not in self._cursor_per_partition:
-                self._create_cursor_for_partition(self._to_partition_key(stream_slice.partition))
             return self._partition_router.get_request_params(  # type: ignore # this always returns a mapping
                 stream_state=stream_state,
                 stream_slice=StreamSlice(partition=stream_slice.partition, cursor_slice={}),
@@ -246,8 +244,6 @@ class PerPartitionCursor(DeclarativeCursor):
         next_page_token: Optional[Mapping[str, Any]] = None,
     ) -> Mapping[str, Any]:
         if stream_slice:
-            if self._to_partition_key(stream_slice.partition) not in self._cursor_per_partition:
-                self._create_cursor_for_partition(self._to_partition_key(stream_slice.partition))
             return self._partition_router.get_request_headers(  # type: ignore # this always returns a mapping
                 stream_state=stream_state,
                 stream_slice=StreamSlice(partition=stream_slice.partition, cursor_slice={}),
@@ -270,8 +266,6 @@ class PerPartitionCursor(DeclarativeCursor):
         next_page_token: Optional[Mapping[str, Any]] = None,
     ) -> Union[Mapping[str, Any], str]:
         if stream_slice:
-            if self._to_partition_key(stream_slice.partition) not in self._cursor_per_partition:
-                self._create_cursor_for_partition(self._to_partition_key(stream_slice.partition))
             return self._partition_router.get_request_body_data(  # type: ignore # this always returns a mapping
                 stream_state=stream_state,
                 stream_slice=StreamSlice(partition=stream_slice.partition, cursor_slice={}),
@@ -294,8 +288,6 @@ class PerPartitionCursor(DeclarativeCursor):
         next_page_token: Optional[Mapping[str, Any]] = None,
     ) -> Mapping[str, Any]:
         if stream_slice:
-            if self._to_partition_key(stream_slice.partition) not in self._cursor_per_partition:
-                self._create_cursor_for_partition(self._to_partition_key(stream_slice.partition))
             return self._partition_router.get_request_body_json(  # type: ignore # this always returns a mapping
                 stream_state=stream_state,
                 stream_slice=StreamSlice(partition=stream_slice.partition, cursor_slice={}),
@@ -311,6 +303,21 @@ class PerPartitionCursor(DeclarativeCursor):
             raise ValueError("A partition needs to be provided in order to get request body json")
 
     def should_be_synced(self, record: Record) -> bool:
+        if (
+            record.associated_slice
+            and self._to_partition_key(record.associated_slice.partition)
+            not in self._cursor_per_partition
+        ):
+            partition_state = (
+                self._state_to_migrate_from
+                if self._state_to_migrate_from
+                else self._NO_CURSOR_STATE
+            )
+            cursor = self._create_cursor(partition_state)
+
+            self._cursor_per_partition[
+                self._to_partition_key(record.associated_slice.partition)
+            ] = cursor
         return self._get_cursor(record).should_be_synced(
             self._convert_record_to_cursor_record(record)
         )
@@ -349,32 +356,8 @@ class PerPartitionCursor(DeclarativeCursor):
             )
         partition_key = self._to_partition_key(record.associated_slice.partition)
         if partition_key not in self._cursor_per_partition:
-            self._create_cursor_for_partition(partition_key)
+            raise ValueError(
+                "Invalid state as stream slices that are emitted should refer to an existing cursor"
+            )
         cursor = self._cursor_per_partition[partition_key]
         return cursor
-
-    def _create_cursor_for_partition(self, partition_key: str) -> None:
-        """
-        Dynamically creates and initializes a cursor for the specified partition.
-
-        This method is required for `ConcurrentPerPartitionCursor`. For concurrent cursors,
-        stream_slices is executed only for the concurrent cursor, so cursors per partition
-        are not created for the declarative cursor. This method ensures that a cursor is available
-        to create requests for the specified partition. The cursor is initialized
-        with the per-partition state if present in the initial state, or with the global state
-        adjusted by the lookback window, or with the state to migrate from.
-
-        Note:
-            This is a temporary workaround and should be removed once the declarative cursor
-            is decoupled from the concurrent cursor implementation.
-
-        Args:
-            partition_key (str): The unique identifier for the partition for which the cursor
-            needs to be created.
-        """
-        partition_state = (
-            self._state_to_migrate_from if self._state_to_migrate_from else self._NO_CURSOR_STATE
-        )
-        cursor = self._create_cursor(partition_state)
-
-        self._cursor_per_partition[partition_key] = cursor

{airbyte_cdk-6.26.0 → airbyte_cdk-6.26.0.dev0}/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py

@@ -2407,7 +2407,7 @@ class ModelToComponentFactory:
         if (
             not isinstance(stream_slicer, DatetimeBasedCursor)
             or type(stream_slicer) is not DatetimeBasedCursor
-        ):
+        ) and not isinstance(stream_slicer, PerPartitionWithGlobalCursor):
             # Many of the custom component implementations of DatetimeBasedCursor override get_request_params() (or other methods).
             # Because we're decoupling RequestOptionsProvider from the Cursor, custom components will eventually need to reimplement
             # their own RequestOptionsProvider. However, right now the existing StreamSlicer/Cursor still can act as the SimpleRetriever's

{airbyte_cdk-6.26.0 → airbyte_cdk-6.26.0.dev0}/airbyte_cdk/sources/declarative/partition_routers/substream_partition_router.py

@@ -296,12 +296,8 @@ class SubstreamPartitionRouter(PartitionRouter):
 
         if not parent_state and incremental_dependency:
             # Attempt to retrieve child state
-            substream_state_values = list(stream_state.values())
-            substream_state = substream_state_values[0] if substream_state_values else {}
-            # Filter out per partition state. Because we pass the state to the parent stream in the format {cursor_field: substream_state}
-            if isinstance(substream_state, (list, dict)):
-                substream_state = {}
-
+            substream_state = list(stream_state.values())
+            substream_state = substream_state[0] if substream_state else {}  # type: ignore [assignment]  # Incorrect type for assignment
             parent_state = {}
 
             # Copy child state to parent streams with incremental dependencies

{airbyte_cdk-6.26.0 → airbyte_cdk-6.26.0.dev0}/airbyte_cdk/sources/declarative/schema/dynamic_schema_loader.py

@@ -154,9 +154,8 @@ class DynamicSchemaLoader(SchemaLoader):
         transformed_properties = self._transform(properties, {})
 
         return {
-            "$schema": "https://json-schema.org/draft-07/schema#",
+            "$schema": "http://json-schema.org/draft-07/schema#",
             "type": "object",
-            "additionalProperties": True,
             "properties": transformed_properties,
         }
 
@@ -221,17 +220,25 @@ class DynamicSchemaLoader(SchemaLoader):
             )
 
     def _resolve_complex_type(self, complex_type: ComplexFieldType) -> Mapping[str, Any]:
-        if not complex_type.items:
-            return self._get_airbyte_type(complex_type.field_type)
-
-        field_type = self._get_airbyte_type(complex_type.field_type)
-        field_type["items"] = (
-            self._get_airbyte_type(complex_type.items)
-            if isinstance(complex_type.items, str)
-            else self._resolve_complex_type(complex_type.items)
-        )
-
-        return field_type
+        types = [complex_type]
+        resolved_type: MutableMapping[str, Any] = {}
+
+        while types:
+            current_type = types.pop()
+            if not current_type.items:
+                resolved_type = self._get_airbyte_type(current_type.field_type)
+            else:
+                field_type = self._get_airbyte_type(current_type.field_type)
+
+                if isinstance(current_type.items, str):
+                    items_type = current_type.items
+                else:
+                    types.append(current_type.items)
+                    continue  # Skip the following lines until the stack is resolved
+                field_type["items"] = self._get_airbyte_type(items_type)
+                resolved_type = field_type
+
+        return resolved_type
 
     def _replace_type_if_not_valid(
         self,

{airbyte_cdk-6.26.0 → airbyte_cdk-6.26.0.dev0}/pyproject.toml

@@ -23,7 +23,7 @@ classifiers = [
 keywords = ["airbyte", "connector-development-kit", "cdk"]
 
 # Python CDK uses dynamic versioning: https://github.com/mtkennerly/poetry-dynamic-versioning
-version = "6.26.0"  # Version will be calculated dynamically.
+version = "6.26.0dev0"  # Version will be calculated dynamically.
 
 [tool.poetry-dynamic-versioning]
 enable = false