airbyte-cdk 6.8.1rc10__py3-none-any.whl → 6.8.2.dev1__py3-none-any.whl

airbyte_cdk/sources/declarative/manifest_declarative_source.py

@@ -39,7 +39,6 @@ from airbyte_cdk.sources.declarative.parsers.manifest_reference_resolver import
 from airbyte_cdk.sources.declarative.parsers.model_to_component_factory import (
     ModelToComponentFactory,
 )
-from airbyte_cdk.sources.declarative.resolvers import COMPONENTS_RESOLVER_TYPE_MAPPING
 from airbyte_cdk.sources.message import MessageRepository
 from airbyte_cdk.sources.streams.core import Stream
 from airbyte_cdk.sources.types import ConnectionDefinition
@@ -121,10 +120,7 @@ class ManifestDeclarativeSource(DeclarativeSource):
         self._emit_manifest_debug_message(
             extra_args={"source_name": self.name, "parsed_config": json.dumps(self._source_config)}
         )
-
-        stream_configs = self._stream_configs(self._source_config) + self._dynamic_stream_configs(
-            self._source_config, config
-        )
+        stream_configs = self._stream_configs(self._source_config)
 
         source_streams = [
             self._constructor.create_component(
@@ -238,8 +234,7 @@ class ManifestDeclarativeSource(DeclarativeSource):
             )
 
         streams = self._source_config.get("streams")
-        dynamic_streams = self._source_config.get("dynamic_streams")
-        if not (streams or dynamic_streams):
+        if not streams:
             raise ValidationError(
                 f"A valid manifest should have at least one stream defined. Got {streams}"
             )
@@ -308,51 +303,5 @@ class ManifestDeclarativeSource(DeclarativeSource):
                 s["type"] = "DeclarativeStream"
         return stream_configs
 
-    def _dynamic_stream_configs(
-        self, manifest: Mapping[str, Any], config: Mapping[str, Any]
-    ) -> List[Dict[str, Any]]:
-        dynamic_stream_definitions: List[Dict[str, Any]] = manifest.get("dynamic_streams", [])
-        dynamic_stream_configs: List[Dict[str, Any]] = []
-
-        for dynamic_definition in dynamic_stream_definitions:
-            components_resolver_config = dynamic_definition["components_resolver"]
-
-            if not components_resolver_config:
-                raise ValueError(
-                    f"Missing 'components_resolver' in dynamic definition: {dynamic_definition}"
-                )
-
-            resolver_type = components_resolver_config.get("type")
-            if not resolver_type:
-                raise ValueError(
-                    f"Missing 'type' in components resolver configuration: {components_resolver_config}"
-                )
-
-            if resolver_type not in COMPONENTS_RESOLVER_TYPE_MAPPING:
-                raise ValueError(
-                    f"Invalid components resolver type '{resolver_type}'. "
-                    f"Expected one of {list(COMPONENTS_RESOLVER_TYPE_MAPPING.keys())}."
-                )
-
-            if "retriever" in components_resolver_config:
-                components_resolver_config["retriever"]["requester"]["use_cache"] = True
-
-            # Create a resolver for dynamic components based on type
-            components_resolver = self._constructor.create_component(
-                COMPONENTS_RESOLVER_TYPE_MAPPING[resolver_type], components_resolver_config, config
-            )
-
-            stream_template_config = dynamic_definition["stream_template"]
-
-            for dynamic_stream in components_resolver.resolve_components(
-                stream_template_config=stream_template_config
-            ):
-                if "type" not in dynamic_stream:
-                    dynamic_stream["type"] = "DeclarativeStream"
-
-                dynamic_stream_configs.append(dynamic_stream)
-
-        return dynamic_stream_configs
-
     def _emit_manifest_debug_message(self, extra_args: dict[str, Any]) -> None:
         self.logger.debug("declarative source created from manifest", extra=extra_args)

airbyte_cdk/sources/declarative/models/declarative_component_schema.py

@@ -1158,37 +1158,6 @@ class WaitUntilTimeFromHeader(BaseModel):
     parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
 
 
-class ComponentMappingDefinition(BaseModel):
-    type: Literal["ComponentMappingDefinition"]
-    field_path: List[str] = Field(
-        ...,
-        description="A list of potentially nested fields indicating the full path where value will be added or updated.",
-        examples=[
-            ["data"],
-            ["data", "records"],
-            ["data", "{{ parameters.name }}"],
-            ["data", "*", "record"],
-        ],
-        title="Field Path",
-    )
-    value: str = Field(
-        ...,
-        description="The dynamic or static value to assign to the key. Interpolated values can be used to dynamically determine the value during runtime.",
-        examples=[
-            "{{ components_values['updates'] }}",
-            "{{ components_values['MetaData']['LastUpdatedTime'] }}",
-            "{{ config['segment_id'] }}",
-        ],
-        title="Value",
-    )
-    value_type: Optional[ValueType] = Field(
-        None,
-        description="The expected data type of the value. If omitted, the type will be inferred from the value provided.",
-        title="Value Type",
-    )
-    parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
-
-
 class AddedFieldDefinition(BaseModel):
     type: Literal["AddedFieldDefinition"]
     path: List[str] = Field(
@@ -1486,40 +1455,13 @@ class CompositeErrorHandler(BaseModel):
     parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
 
 
-class DeclarativeSource1(BaseModel):
+class DeclarativeSource(BaseModel):
     class Config:
         extra = Extra.forbid
 
     type: Literal["DeclarativeSource"]
     check: CheckStream
     streams: List[DeclarativeStream]
-    dynamic_streams: Optional[List[DynamicDeclarativeStream]] = None
-    version: str = Field(
-        ...,
-        description="The version of the Airbyte CDK used to build and test the source.",
-    )
-    schemas: Optional[Schemas] = None
-    definitions: Optional[Dict[str, Any]] = None
-    spec: Optional[Spec] = None
-    concurrency_level: Optional[ConcurrencyLevel] = None
-    metadata: Optional[Dict[str, Any]] = Field(
-        None,
-        description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.",
-    )
-    description: Optional[str] = Field(
-        None,
-        description="A description of the connector. It will be presented on the Source documentation page.",
-    )
-
-
-class DeclarativeSource2(BaseModel):
-    class Config:
-        extra = Extra.forbid
-
-    type: Literal["DeclarativeSource"]
-    check: CheckStream
-    streams: Optional[List[DeclarativeStream]] = None
-    dynamic_streams: List[DynamicDeclarativeStream]
     version: str = Field(
         ...,
         description="The version of the Airbyte CDK used to build and test the source.",
@@ -1538,17 +1480,6 @@ class DeclarativeSource2(BaseModel):
     )
 
 
-class DeclarativeSource(BaseModel):
-    class Config:
-        extra = Extra.forbid
-
-    __root__: Union[DeclarativeSource1, DeclarativeSource2] = Field(
-        ...,
-        description="An API source that extracts data according to its declarative components.",
-        title="DeclarativeSource",
-    )
-
-
 class SelectiveAuthenticator(BaseModel):
     class Config:
         extra = Extra.allow
@@ -1952,32 +1883,8 @@ class SubstreamPartitionRouter(BaseModel):
     parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
 
 
-class HttpComponentsResolver(BaseModel):
-    type: Literal["HttpComponentsResolver"]
-    retriever: Union[AsyncRetriever, CustomRetriever, SimpleRetriever] = Field(
-        ...,
-        description="Component used to coordinate how records are extracted across stream slices and request pages.",
-        title="Retriever",
-    )
-    components_mapping: List[ComponentMappingDefinition]
-    parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
-
-
-class DynamicDeclarativeStream(BaseModel):
-    type: Literal["DynamicDeclarativeStream"]
-    stream_template: DeclarativeStream = Field(
-        ..., description="Reference to the stream template.", title="Stream Template"
-    )
-    components_resolver: HttpComponentsResolver = Field(
-        ...,
-        description="Component resolve and populates stream templates with components values.",
-        title="Components Resolver",
-    )
-
-
 CompositeErrorHandler.update_forward_refs()
-DeclarativeSource1.update_forward_refs()
-DeclarativeSource2.update_forward_refs()
+DeclarativeSource.update_forward_refs()
 SelectiveAuthenticator.update_forward_refs()
 DeclarativeStream.update_forward_refs()
 SessionTokenAuthenticator.update_forward_refs()

airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py

@@ -31,12 +31,6 @@ DEFAULT_MODEL_TYPES: Mapping[str, str] = {
     # DeclarativeStream
     "DeclarativeStream.retriever": "SimpleRetriever",
     "DeclarativeStream.schema_loader": "JsonFileSchemaLoader",
-    # DynamicDeclarativeStream
-    "DynamicDeclarativeStream.stream_template": "DeclarativeStream",
-    "DynamicDeclarativeStream.components_resolver": "HttpComponentsResolver",
-    # HttpComponentsResolver
-    "HttpComponentsResolver.retriever": "SimpleRetriever",
-    "HttpComponentsResolver.components_mapping": "ComponentMappingDefinition",
     # DefaultErrorHandler
     "DefaultErrorHandler.response_filters": "HttpResponseFilter",
     # DefaultPaginator

airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py

@@ -81,6 +81,8 @@ from airbyte_cdk.sources.declarative.extractors.record_selector import (
 )
 from airbyte_cdk.sources.declarative.incremental import (
     ChildPartitionResumableFullRefreshCursor,
+    ConcurrentCursorFactory,
+    ConcurrentPerPartitionCursor,
     CursorFactory,
     DatetimeBasedCursor,
     DeclarativeCursor,
@@ -119,9 +121,6 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     CheckStream as CheckStreamModel,
 )
-from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
-    ComponentMappingDefinition as ComponentMappingDefinitionModel,
-)
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     CompositeErrorHandler as CompositeErrorHandlerModel,
 )
@@ -194,9 +193,6 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     GzipJsonDecoder as GzipJsonDecoderModel,
 )
-from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
-    HttpComponentsResolver as HttpComponentsResolverModel,
-)
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     HttpRequester as HttpRequesterModel,
 )
@@ -304,7 +300,6 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import
 from airbyte_cdk.sources.declarative.partition_routers import (
     CartesianProductStreamSlicer,
     ListPartitionRouter,
-    PartitionRouter,
     SinglePartitionRouter,
     SubstreamPartitionRouter,
 )
@@ -345,10 +340,6 @@ from airbyte_cdk.sources.declarative.requesters.request_options import (
 )
 from airbyte_cdk.sources.declarative.requesters.request_path import RequestPath
 from airbyte_cdk.sources.declarative.requesters.requester import HttpMethod
-from airbyte_cdk.sources.declarative.resolvers import (
-    ComponentMappingDefinition,
-    HttpComponentsResolver,
-)
 from airbyte_cdk.sources.declarative.retrievers import (
     AsyncRetriever,
     SimpleRetriever,
@@ -478,8 +469,6 @@ class ModelToComponentFactory:
             WaitTimeFromHeaderModel: self.create_wait_time_from_header,
             WaitUntilTimeFromHeaderModel: self.create_wait_until_time_from_header,
             AsyncRetrieverModel: self.create_async_retriever,
-            HttpComponentsResolverModel: self.create_http_components_resolver,
-            ComponentMappingDefinitionModel: self.create_components_mapping_definition,
         }
 
         # Needed for the case where we need to perform a second parse on the fields of a custom component
@@ -918,6 +907,62 @@ class ModelToComponentFactory:
             cursor_granularity=cursor_granularity,
         )
 
+    def create_concurrent_cursor_from_perpartition_cursor(
+        self,
+        state_manager: ConnectorStateManager,
+        model_type: Type[BaseModel],
+        component_definition: ComponentDefinition,
+        stream_name: str,
+        stream_namespace: Optional[str],
+        config: Config,
+        stream_state: MutableMapping[str, Any],
+        partition_router,
+            **kwargs: Any,
+    ) -> ConcurrentPerPartitionCursor:
+        component_type = component_definition.get("type")
+        if component_definition.get("type") != model_type.__name__:
+            raise ValueError(
+                f"Expected manifest component of type {model_type.__name__}, but received {component_type} instead"
+            )
+
+        datetime_based_cursor_model = model_type.parse_obj(component_definition)
+
+        if not isinstance(datetime_based_cursor_model, DatetimeBasedCursorModel):
+            raise ValueError(
+                f"Expected {model_type.__name__} component, but received {datetime_based_cursor_model.__class__.__name__}"
+            )
+
+        interpolated_cursor_field = InterpolatedString.create(
+            datetime_based_cursor_model.cursor_field,
+            parameters=datetime_based_cursor_model.parameters or {},
+        )
+        cursor_field = CursorField(interpolated_cursor_field.eval(config=config))
+
+        # Create the cursor factory
+        cursor_factory = ConcurrentCursorFactory(
+            partial(
+                self.create_concurrent_cursor_from_datetime_based_cursor,
+                state_manager=state_manager,
+                model_type=model_type,
+                component_definition=component_definition,
+                stream_name=stream_name,
+                stream_namespace=stream_namespace,
+                config=config,
+            )
+        )
+
+        # Return the concurrent cursor and state converter
+        return ConcurrentPerPartitionCursor(
+                cursor_factory=cursor_factory,
+                partition_router=partition_router,
+                stream_name=stream_name,
+                stream_namespace=stream_namespace,
+                stream_state=stream_state,
+                message_repository=self._message_repository,  # type: ignore
+                connector_state_manager=state_manager,
+                cursor_field=cursor_field,
+            )
+
     @staticmethod
     def create_constant_backoff_strategy(
         model: ConstantBackoffStrategyModel, config: Config, **kwargs: Any
@@ -1200,17 +1245,14 @@ class ModelToComponentFactory:
                 raise ValueError(
                     "Unsupported Slicer is used. PerPartitionWithGlobalCursor should be used here instead"
                 )
+            cursor = combined_slicers if isinstance(
+                combined_slicers, (PerPartitionWithGlobalCursor, GlobalSubstreamCursor)
+            ) else self._create_component_from_model(
+                model=model.incremental_sync, config=config
+            )
+
             client_side_incremental_sync = {
-                "date_time_based_cursor": self._create_component_from_model(
-                    model=model.incremental_sync, config=config
-                ),
-                "substream_cursor": (
-                    combined_slicers
-                    if isinstance(
-                        combined_slicers, (PerPartitionWithGlobalCursor, GlobalSubstreamCursor)
-                    )
-                    else None
-                ),
+                "cursor": cursor
             }
 
         if model.incremental_sync and isinstance(model.incremental_sync, DatetimeBasedCursorModel):
@@ -1294,20 +1336,19 @@ class ModelToComponentFactory:
             parameters=model.parameters or {},
         )
 
-    def _build_stream_slicer_from_partition_router(
-        self,
-        model: Union[AsyncRetrieverModel, CustomRetrieverModel, SimpleRetrieverModel],
-        config: Config,
-    ) -> Optional[PartitionRouter]:
+    def _merge_stream_slicers(
+        self, model: DeclarativeStreamModel, config: Config
+    ) -> Optional[StreamSlicer]:
+        stream_slicer = None
         if (
-            hasattr(model, "partition_router")
-            and isinstance(model, SimpleRetrieverModel)
-            and model.partition_router
+            hasattr(model.retriever, "partition_router")
+            and isinstance(model.retriever, SimpleRetrieverModel)
+            and model.retriever.partition_router
         ):
-            stream_slicer_model = model.partition_router
+            stream_slicer_model = model.retriever.partition_router
 
             if isinstance(stream_slicer_model, list):
-                return CartesianProductStreamSlicer(
+                stream_slicer = CartesianProductStreamSlicer(
                     [
                         self._create_component_from_model(model=slicer, config=config)
                         for slicer in stream_slicer_model
@@ -1315,24 +1356,9 @@ class ModelToComponentFactory:
                     parameters={},
                 )
             else:
-                return self._create_component_from_model(model=stream_slicer_model, config=config)  # type: ignore[no-any-return]
-                # Will be created PartitionRouter as stream_slicer_model is model.partition_router
-        return None
-
-    def _build_resumable_cursor_from_paginator(
-        self,
-        model: Union[AsyncRetrieverModel, CustomRetrieverModel, SimpleRetrieverModel],
-        stream_slicer: Optional[StreamSlicer],
-    ) -> Optional[StreamSlicer]:
-        if hasattr(model, "paginator") and model.paginator and not stream_slicer:
-            # For the regular Full-Refresh streams, we use the high level `ResumableFullRefreshCursor`
-            return ResumableFullRefreshCursor(parameters={})
-        return None
-
-    def _merge_stream_slicers(
-        self, model: DeclarativeStreamModel, config: Config
-    ) -> Optional[StreamSlicer]:
-        stream_slicer = self._build_stream_slicer_from_partition_router(model.retriever, config)
+                stream_slicer = self._create_component_from_model(
+                    model=stream_slicer_model, config=config
+                )
 
         if model.incremental_sync and stream_slicer:
             incremental_sync_model = model.incremental_sync
@@ -1375,7 +1401,15 @@ class ModelToComponentFactory:
                 ),
                 partition_router=stream_slicer,
             )
-        return self._build_resumable_cursor_from_paginator(model.retriever, stream_slicer)
+        elif (
+            hasattr(model.retriever, "paginator")
+            and model.retriever.paginator
+            and not stream_slicer
+        ):
+            # For the regular Full-Refresh streams, we use the high level `ResumableFullRefreshCursor`
+            return ResumableFullRefreshCursor(parameters={})
+        else:
+            return None
 
     def create_default_error_handler(
         self, model: DefaultErrorHandlerModel, config: Config, **kwargs: Any
@@ -1932,7 +1966,7 @@ class ModelToComponentFactory:
         if (
             not isinstance(stream_slicer, DatetimeBasedCursor)
             or type(stream_slicer) is not DatetimeBasedCursor
-        ):
+        ) and not isinstance(stream_slicer, PerPartitionWithGlobalCursor):
             # Many of the custom component implementations of DatetimeBasedCursor override get_request_params() (or other methods).
             # Because we're decoupling RequestOptionsProvider from the Cursor, custom components will eventually need to reimplement
             # their own RequestOptionsProvider. However, right now the existing StreamSlicer/Cursor still can act as the SimpleRetriever's
@@ -2239,56 +2273,3 @@ class ModelToComponentFactory:
 
     def _evaluate_log_level(self, emit_connector_builder_messages: bool) -> Level:
         return Level.DEBUG if emit_connector_builder_messages else Level.INFO
-
-    @staticmethod
-    def create_components_mapping_definition(
-        model: ComponentMappingDefinitionModel, config: Config, **kwargs: Any
-    ) -> ComponentMappingDefinition:
-        interpolated_value = InterpolatedString.create(
-            model.value, parameters=model.parameters or {}
-        )
-        field_path = [
-            InterpolatedString.create(path, parameters=model.parameters or {})
-            for path in model.field_path
-        ]
-        return ComponentMappingDefinition(
-            field_path=field_path,  # type: ignore[arg-type] # field_path can be str and InterpolatedString
-            value=interpolated_value,
-            value_type=ModelToComponentFactory._json_schema_type_name_to_type(model.value_type),
-            parameters=model.parameters or {},
-        )
-
-    def create_http_components_resolver(
-        self, model: HttpComponentsResolverModel, config: Config
-    ) -> Any:
-        stream_slicer = self._build_stream_slicer_from_partition_router(model.retriever, config)
-        combined_slicers = self._build_resumable_cursor_from_paginator(
-            model.retriever, stream_slicer
-        )
-
-        retriever = self._create_component_from_model(
-            model=model.retriever,
-            config=config,
-            name="",
-            primary_key=None,
-            stream_slicer=combined_slicers,
-            transformations=[],
-        )
-
-        components_mapping = [
-            self._create_component_from_model(
-                model=components_mapping_definition_model,
-                value_type=ModelToComponentFactory._json_schema_type_name_to_type(
-                    components_mapping_definition_model.value_type
-                ),
-                config=config,
-            )
-            for components_mapping_definition_model in model.components_mapping
-        ]
-
-        return HttpComponentsResolver(
-            retriever=retriever,
-            config=config,
-            components_mapping=components_mapping,
-            parameters=model.parameters or {},
-        )

airbyte_cdk/sources/declarative/partition_routers/__init__.py

@@ -6,6 +6,5 @@ from airbyte_cdk.sources.declarative.partition_routers.cartesian_product_stream_
 from airbyte_cdk.sources.declarative.partition_routers.list_partition_router import ListPartitionRouter
 from airbyte_cdk.sources.declarative.partition_routers.single_partition_router import SinglePartitionRouter
 from airbyte_cdk.sources.declarative.partition_routers.substream_partition_router import SubstreamPartitionRouter
-from airbyte_cdk.sources.declarative.partition_routers.partition_router import PartitionRouter
 
-__all__ = ["CartesianProductStreamSlicer", "ListPartitionRouter", "SinglePartitionRouter", "SubstreamPartitionRouter", "PartitionRouter"]
+__all__ = ["CartesianProductStreamSlicer", "ListPartitionRouter", "SinglePartitionRouter", "SubstreamPartitionRouter"]

airbyte_cdk/sources/declarative/retrievers/simple_retriever.py

@@ -178,7 +178,7 @@ class SimpleRetriever(Retriever):
             stream_slice,
             next_page_token,
             self._paginator.get_request_headers,
-            self.stream_slicer.get_request_headers,
+            self.request_option_provider.get_request_headers,
         )
         if isinstance(headers, str):
             raise ValueError("Request headers cannot be a string")

airbyte_cdk/sources/declarative/stream_slicers/declarative_partition_generator.py

@@ -38,7 +38,6 @@ class DeclarativePartitionFactory:
             stream_slice,
         )
 
-
 class DeclarativePartition(Partition):
     def __init__(
         self,

airbyte_cdk/sources/streams/concurrent/cursor.py

@@ -240,6 +240,15 @@ class ConcurrentCursor(Cursor):
     def _extract_cursor_value(self, record: Record) -> Any:
         return self._connector_state_converter.parse_value(self._cursor_field.extract_value(record))
 
+    def close_partition_without_emit(self, partition: Partition) -> None:
+        slice_count_before = len(self.state.get("slices", []))
+        self._add_slice_to_state(partition)
+        if slice_count_before < len(
+            self.state["slices"]
+        ):  # only emit if at least one slice has been processed
+            self._merge_partitions()
+        self._has_closed_at_least_one_slice = True
+
     def close_partition(self, partition: Partition) -> None:
         slice_count_before = len(self.state.get("slices", []))
         self._add_slice_to_state(partition)

{airbyte_cdk-6.8.1rc10.dist-info → airbyte_cdk-6.8.2.dev1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 6.8.1rc10
+Version: 6.8.2.dev1
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://airbyte.com
 License: MIT