airbyte-cdk 6.33.0.dev0__tar.gz → 6.33.1.dev0__tar.gz

{airbyte_cdk-6.33.0.dev0 → airbyte_cdk-6.33.1.dev0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 6.33.0.dev0
+Version: 6.33.1.dev0
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://airbyte.com
 License: MIT

{airbyte_cdk-6.33.0.dev0 → airbyte_cdk-6.33.1.dev0}/airbyte_cdk/sources/declarative/auth/token.py

@@ -5,7 +5,7 @@
 import base64
 import logging
 from dataclasses import InitVar, dataclass
-from typing import Any, Mapping, Union
+from typing import Any, Mapping, MutableMapping, Union
 
 import requests
 from cachetools import TTLCache, cached
@@ -45,11 +45,6 @@ class ApiKeyAuthenticator(DeclarativeAuthenticator):
     config: Config
     parameters: InitVar[Mapping[str, Any]]
 
-    def __post_init__(self, parameters: Mapping[str, Any]) -> None:
-        self._field_name = InterpolatedString.create(
-            self.request_option.field_name, parameters=parameters
-        )
-
     @property
     def auth_header(self) -> str:
         options = self._get_request_options(RequestOptionType.header)
@@ -60,9 +55,9 @@ class ApiKeyAuthenticator(DeclarativeAuthenticator):
         return self.token_provider.get_token()
 
     def _get_request_options(self, option_type: RequestOptionType) -> Mapping[str, Any]:
-        options = {}
+        options: MutableMapping[str, Any] = {}
         if self.request_option.inject_into == option_type:
-            options[self._field_name.eval(self.config)] = self.token
+            self.request_option.inject_into_request(options, self.token, self.config)
         return options
 
     def get_request_params(self) -> Mapping[str, Any]:

{airbyte_cdk-6.33.0.dev0 → airbyte_cdk-6.33.1.dev0}/airbyte_cdk/sources/declarative/concurrent_declarative_source.py

@@ -475,10 +475,21 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
             # Also a temporary hack. In the legacy Stream implementation, as part of the read,
             # set_initial_state() is called to instantiate incoming state on the cursor. Although we no
             # longer rely on the legacy low-code cursor for concurrent checkpointing, low-code components
-            # like StopConditionPaginationStrategyDecorator and ClientSideIncrementalRecordFilterDecorator
-            # still rely on a DatetimeBasedCursor that is properly initialized with state.
+            # like StopConditionPaginationStrategyDecorator still rely on a DatetimeBasedCursor that is
+            # properly initialized with state.
             if retriever.cursor:
                 retriever.cursor.set_initial_state(stream_state=stream_state)
+
+            # Similar to above, the ClientSideIncrementalRecordFilterDecorator cursor is a separate instance
+            # from the one initialized on the SimpleRetriever, so it also must also have state initialized
+            # for semi-incremental streams using is_client_side_incremental to filter properly
+            if isinstance(retriever.record_selector, RecordSelector) and isinstance(
+                retriever.record_selector.record_filter, ClientSideIncrementalRecordFilterDecorator
+            ):
+                retriever.record_selector.record_filter._cursor.set_initial_state(
+                    stream_state=stream_state
+                )  # type: ignore  # After non-concurrent cursors are deprecated we can remove these cursor workarounds
+
             # We zero it out here, but since this is a cursor reference, the state is still properly
             # instantiated for the other components that reference it
             retriever.cursor = None

{airbyte_cdk-6.33.0.dev0 → airbyte_cdk-6.33.1.dev0}/airbyte_cdk/sources/declarative/declarative_component_schema.yaml

@@ -1478,7 +1478,7 @@ definitions:
         description: List of matchers that define which requests this policy applies to.
         type: array
         items:
-          "$ref": "#/definitions/HttpRequestMatcher"
+          "$ref": "#/definitions/HttpRequestRegexMatcher"
     additionalProperties: true
   MovingWindowCallRatePolicy:
     title: Moving Window Call Rate Policy
@@ -1503,7 +1503,7 @@ definitions:
         description: List of matchers that define which requests this policy applies to.
         type: array
         items:
-          "$ref": "#/definitions/HttpRequestMatcher"
+          "$ref": "#/definitions/HttpRequestRegexMatcher"
     additionalProperties: true
   UnlimitedCallRatePolicy:
     title: Unlimited Call Rate Policy
@@ -1521,7 +1521,7 @@ definitions:
         description: List of matchers that define which requests this policy applies to.
         type: array
         items:
-          "$ref": "#/definitions/HttpRequestMatcher"
+          "$ref": "#/definitions/HttpRequestRegexMatcher"
     additionalProperties: true
   Rate:
     title: Rate
@@ -1541,7 +1541,7 @@ definitions:
         type: string
         format: duration
     additionalProperties: true
-  HttpRequestMatcher:
+  HttpRequestRegexMatcher:
     title: HTTP Request Matcher
     description: >
       Matches HTTP requests based on method, base URL, URL path pattern, query parameters, and headers.
@@ -3054,25 +3054,35 @@ definitions:
         enum: [RequestPath]
   RequestOption:
     title: Request Option
-    description: Specifies the key field and where in the request a component's value should be injected.
+    description: Specifies the key field or path and where in the request a component's value should be injected.
     type: object
     required:
       - type
-      - field_name
       - inject_into
     properties:
       type:
         type: string
         enum: [RequestOption]
       field_name:
-        title: Request Option
-        description: Configures which key should be used in the location that the descriptor is being injected into
+        title: Field Name
+        description: Configures which key should be used in the location that the descriptor is being injected into. We hope to eventually deprecate this field in favor of `field_path` for all request_options, but must currently maintain it for backwards compatibility in the Builder.
         type: string
         examples:
           - segment_id
         interpolation_context:
           - config
           - parameters
+      field_path:
+        title: Field Path
+        description: Configures a path to be used for nested structures in JSON body requests (e.g. GraphQL queries)
+        type: array
+        items:
+          type: string
+        examples:
+          - ["data", "viewer", "id"]
+        interpolation_context:
+          - config
+          - parameters
       inject_into:
         title: Inject Into
         description: Configures where the descriptor should be set on the HTTP requests. Note that request parameters that are already encoded in the URL path will not be duplicated.

{airbyte_cdk-6.33.0.dev0 → airbyte_cdk-6.33.1.dev0}/airbyte_cdk/sources/declarative/incremental/datetime_based_cursor.py

@@ -365,14 +365,15 @@ class DatetimeBasedCursor(DeclarativeCursor):
         options: MutableMapping[str, Any] = {}
         if not stream_slice:
             return options
+
         if self.start_time_option and self.start_time_option.inject_into == option_type:
-            options[self.start_time_option.field_name.eval(config=self.config)] = stream_slice.get(  # type: ignore # field_name is always casted to an interpolated string
-                self._partition_field_start.eval(self.config)
-            )
+            start_time_value = stream_slice.get(self._partition_field_start.eval(self.config))
+            self.start_time_option.inject_into_request(options, start_time_value, self.config)
+
         if self.end_time_option and self.end_time_option.inject_into == option_type:
-            options[self.end_time_option.field_name.eval(config=self.config)] = stream_slice.get(  # type: ignore [union-attr]
-                self._partition_field_end.eval(self.config)
-            )
+            end_time_value = stream_slice.get(self._partition_field_end.eval(self.config))
+            self.end_time_option.inject_into_request(options, end_time_value, self.config)
+
         return options
 
     def should_be_synced(self, record: Record) -> bool:

{airbyte_cdk-6.33.0.dev0 → airbyte_cdk-6.33.1.dev0}/airbyte_cdk/sources/declarative/models/declarative_component_schema.py

@@ -657,7 +657,7 @@ class Rate(BaseModel):
     )
 
 
-class HttpRequestMatcher(BaseModel):
+class HttpRequestRegexMatcher(BaseModel):
     class Config:
         extra = Extra.allow
 
@@ -1240,11 +1240,17 @@ class InjectInto(Enum):
 
 class RequestOption(BaseModel):
     type: Literal["RequestOption"]
-    field_name: str = Field(
-        ...,
-        description="Configures which key should be used in the location that the descriptor is being injected into",
+    field_name: Optional[str] = Field(
+        None,
+        description="Configures which key should be used in the location that the descriptor is being injected into. We hope to eventually deprecate this field in favor of `field_path` for all request_options, but must currently maintain it for backwards compatibility in the Builder.",
         examples=["segment_id"],
-        title="Request Option",
+        title="Field Name",
+    )
+    field_path: Optional[List[str]] = Field(
+        None,
+        description="Configures a path to be used for nested structures in JSON body requests (e.g. GraphQL queries)",
+        examples=[["data", "viewer", "id"]],
+        title="Field Path",
     )
     inject_into: InjectInto = Field(
         ...,
@@ -1636,7 +1642,7 @@ class FixedWindowCallRatePolicy(BaseModel):
         description="The maximum number of calls allowed within the period.",
         title="Call Limit",
     )
-    matchers: List[HttpRequestMatcher] = Field(
+    matchers: List[HttpRequestRegexMatcher] = Field(
         ...,
         description="List of matchers that define which requests this policy applies to.",
         title="Matchers",
@@ -1653,7 +1659,7 @@ class MovingWindowCallRatePolicy(BaseModel):
         description="List of rates that define the call limits for different time intervals.",
         title="Rates",
     )
-    matchers: List[HttpRequestMatcher] = Field(
+    matchers: List[HttpRequestRegexMatcher] = Field(
         ...,
         description="List of matchers that define which requests this policy applies to.",
         title="Matchers",
@@ -1665,7 +1671,7 @@ class UnlimitedCallRatePolicy(BaseModel):
         extra = Extra.allow
 
     type: Literal["UnlimitedCallRatePolicy"]
-    matchers: List[HttpRequestMatcher] = Field(
+    matchers: List[HttpRequestRegexMatcher] = Field(
         ...,
         description="List of matchers that define which requests this policy applies to.",
         title="Matchers",

{airbyte_cdk-6.33.0.dev0 → airbyte_cdk-6.33.1.dev0}/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py

@@ -251,7 +251,7 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import
     HttpRequester as HttpRequesterModel,
 )
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
-    HttpRequestMatcher as HttpRequestMatcherModel,
+    HttpRequestRegexMatcher as HttpRequestRegexMatcherModel,
 )
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     HttpResponseFilter as HttpResponseFilterModel,
@@ -494,7 +494,7 @@ from airbyte_cdk.sources.streams.call_rate import (
     APIBudget,
     FixedWindowCallRatePolicy,
     HttpAPIBudget,
-    HttpRequestMatcher,
+    HttpRequestRegexMatcher,
     MovingWindowCallRatePolicy,
     Rate,
     UnlimitedCallRatePolicy,
@@ -644,7 +644,7 @@ class ModelToComponentFactory:
             MovingWindowCallRatePolicyModel: self.create_moving_window_call_rate_policy,
             UnlimitedCallRatePolicyModel: self.create_unlimited_call_rate_policy,
             RateModel: self.create_rate,
-            HttpRequestMatcherModel: self.create_http_request_matcher,
+            HttpRequestRegexMatcherModel: self.create_http_request_matcher,
         }
 
         # Needed for the case where we need to perform a second parse on the fields of a custom component
@@ -771,8 +771,8 @@ class ModelToComponentFactory:
         }
         return names_to_types[value_type]
 
-    @staticmethod
     def create_api_key_authenticator(
+        self,
         model: ApiKeyAuthenticatorModel,
         config: Config,
         token_provider: Optional[TokenProvider] = None,
@@ -794,10 +794,8 @@ class ModelToComponentFactory:
             )
 
         request_option = (
-            RequestOption(
-                inject_into=RequestOptionType(model.inject_into.inject_into.value),
-                field_name=model.inject_into.field_name,
-                parameters=model.parameters or {},
+            self._create_component_from_model(
+                model.inject_into, config, parameters=model.parameters or {}
             )
             if model.inject_into
             else RequestOption(
@@ -806,6 +804,7 @@ class ModelToComponentFactory:
                 parameters=model.parameters or {},
             )
         )
+
         return ApiKeyAuthenticator(
             token_provider=(
                 token_provider
@@ -887,7 +886,7 @@ class ModelToComponentFactory:
                 token_provider=token_provider,
             )
         else:
-            return ModelToComponentFactory.create_api_key_authenticator(
+            return self.create_api_key_authenticator(
                 ApiKeyAuthenticatorModel(
                     type="ApiKeyAuthenticator",
                     api_token="",
@@ -1527,19 +1526,15 @@ class ModelToComponentFactory:
             )
 
         end_time_option = (
-            RequestOption(
-                inject_into=RequestOptionType(model.end_time_option.inject_into.value),
-                field_name=model.end_time_option.field_name,
-                parameters=model.parameters or {},
+            self._create_component_from_model(
+                model.end_time_option, config, parameters=model.parameters or {}
             )
             if model.end_time_option
             else None
         )
         start_time_option = (
-            RequestOption(
-                inject_into=RequestOptionType(model.start_time_option.inject_into.value),
-                field_name=model.start_time_option.field_name,
-                parameters=model.parameters or {},
+            self._create_component_from_model(
+                model.start_time_option, config, parameters=model.parameters or {}
             )
             if model.start_time_option
             else None
@@ -1610,19 +1605,15 @@ class ModelToComponentFactory:
             cursor_model = model.incremental_sync
 
             end_time_option = (
-                RequestOption(
-                    inject_into=RequestOptionType(cursor_model.end_time_option.inject_into.value),
-                    field_name=cursor_model.end_time_option.field_name,
-                    parameters=cursor_model.parameters or {},
+                self._create_component_from_model(
+                    cursor_model.end_time_option, config, parameters=cursor_model.parameters or {}
                 )
                 if cursor_model.end_time_option
                 else None
             )
             start_time_option = (
-                RequestOption(
-                    inject_into=RequestOptionType(cursor_model.start_time_option.inject_into.value),
-                    field_name=cursor_model.start_time_option.field_name,
-                    parameters=cursor_model.parameters or {},
+                self._create_component_from_model(
+                    cursor_model.start_time_option, config, parameters=cursor_model.parameters or {}
                 )
                 if cursor_model.start_time_option
                 else None
@@ -2191,16 +2182,11 @@ class ModelToComponentFactory:
             additional_jwt_payload=model.additional_jwt_payload,
         )
 
-    @staticmethod
     def create_list_partition_router(
-        model: ListPartitionRouterModel, config: Config, **kwargs: Any
+        self, model: ListPartitionRouterModel, config: Config, **kwargs: Any
     ) -> ListPartitionRouter:
         request_option = (
-            RequestOption(
-                inject_into=RequestOptionType(model.request_option.inject_into.value),
-                field_name=model.request_option.field_name,
-                parameters=model.parameters or {},
-            )
+            self._create_component_from_model(model.request_option, config)
             if model.request_option
             else None
         )
@@ -2396,7 +2382,25 @@ class ModelToComponentFactory:
         model: RequestOptionModel, config: Config, **kwargs: Any
     ) -> RequestOption:
         inject_into = RequestOptionType(model.inject_into.value)
-        return RequestOption(field_name=model.field_name, inject_into=inject_into, parameters={})
+        field_path: Optional[List[Union[InterpolatedString, str]]] = (
+            [
+                InterpolatedString.create(segment, parameters=kwargs.get("parameters", {}))
+                for segment in model.field_path
+            ]
+            if model.field_path
+            else None
+        )
+        field_name = (
+            InterpolatedString.create(model.field_name, parameters=kwargs.get("parameters", {}))
+            if model.field_name
+            else None
+        )
+        return RequestOption(
+            field_name=field_name,
+            field_path=field_path,
+            inject_into=inject_into,
+            parameters=kwargs.get("parameters", {}),
+        )
 
     def create_record_selector(
         self,
@@ -3036,9 +3040,9 @@ class ModelToComponentFactory:
         )
 
     def create_http_request_matcher(
-        self, model: HttpRequestMatcherModel, config: Config, **kwargs: Any
-    ) -> HttpRequestMatcher:
-        return HttpRequestMatcher(
+        self, model: HttpRequestRegexMatcherModel, config: Config, **kwargs: Any
+    ) -> HttpRequestRegexMatcher:
+        return HttpRequestRegexMatcher(
             method=model.method,
             url_base=model.url_base,
             url_path_pattern=model.url_path_pattern,

{airbyte_cdk-6.33.0.dev0 → airbyte_cdk-6.33.1.dev0}/airbyte_cdk/sources/declarative/partition_routers/list_partition_router.py

@@ -3,7 +3,7 @@
 #
 
 from dataclasses import InitVar, dataclass
-from typing import Any, Iterable, List, Mapping, Optional, Union
+from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Union
 
 from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
 from airbyte_cdk.sources.declarative.partition_routers.partition_router import PartitionRouter
@@ -100,7 +100,9 @@ class ListPartitionRouter(PartitionRouter):
         ):
             slice_value = stream_slice.get(self._cursor_field.eval(self.config))
             if slice_value:
-                return {self.request_option.field_name.eval(self.config): slice_value}  # type: ignore # field_name is always casted to InterpolatedString
+                options: MutableMapping[str, Any] = {}
+                self.request_option.inject_into_request(options, slice_value, self.config)
+                return options
             else:
                 return {}
         else:

{airbyte_cdk-6.33.0.dev0 → airbyte_cdk-6.33.1.dev0}/airbyte_cdk/sources/declarative/partition_routers/substream_partition_router.py

@@ -4,7 +4,7 @@
 import copy
 import logging
 from dataclasses import InitVar, dataclass
-from typing import TYPE_CHECKING, Any, Iterable, List, Mapping, Optional, Union
+from typing import TYPE_CHECKING, Any, Iterable, List, Mapping, MutableMapping, Optional, Union
 
 import dpath
 
@@ -118,7 +118,7 @@ class SubstreamPartitionRouter(PartitionRouter):
     def _get_request_option(
         self, option_type: RequestOptionType, stream_slice: Optional[StreamSlice]
     ) -> Mapping[str, Any]:
-        params = {}
+        params: MutableMapping[str, Any] = {}
         if stream_slice:
             for parent_config in self.parent_stream_configs:
                 if (
@@ -128,13 +128,7 @@ class SubstreamPartitionRouter(PartitionRouter):
                     key = parent_config.partition_field.eval(self.config)  # type: ignore # partition_field is always casted to an interpolated string
                     value = stream_slice.get(key)
                     if value:
-                        params.update(
-                            {
-                                parent_config.request_option.field_name.eval(  # type: ignore [union-attr]
-                                    config=self.config
-                                ): value
-                            }
-                        )
+                        parent_config.request_option.inject_into_request(params, value, self.config)
         return params
 
     def stream_slices(self) -> Iterable[StreamSlice]:
@@ -305,23 +299,33 @@ class SubstreamPartitionRouter(PartitionRouter):
 
     def _migrate_child_state_to_parent_state(self, stream_state: StreamState) -> StreamState:
         """
-        Migrate the child stream state to the parent stream's state format.
+        Migrate the child or global stream state into the parent stream's state format.
+
+        This method converts the child stream state—or, if present, the global state—into a format that is
+        compatible with parent streams that use incremental synchronization. The migration occurs only for
+        parent streams with incremental dependencies. It filters out per-partition states and retains only the
+        global state in the form {cursor_field: cursor_value}.
 
-        This method converts the global or child state into a format compatible with parent
-        streams. The migration occurs only for parent streams with incremental dependencies.
-        The method filters out per-partition states and retains only the global state in the
-        format `{cursor_field: cursor_value}`.
+        The method supports multiple input formats:
+          - A simple global state, e.g.:
+                {"updated_at": "2023-05-27T00:00:00Z"}
+          - A state object that contains a "state" key (which is assumed to hold the global state), e.g.:
+                {"state": {"updated_at": "2023-05-27T00:00:00Z"}, ...}
+            In this case, the migration uses the first value from the "state" dictionary.
+          - Any per-partition state formats or other non-simple structures are ignored during migration.
 
         Args:
             stream_state (StreamState): The state to migrate. Expected formats include:
                 - {"updated_at": "2023-05-27T00:00:00Z"}
-                - {"states": [...] } (ignored during migration)
+                - {"state": {"updated_at": "2023-05-27T00:00:00Z"}, ...}
+                  (In this format, only the first global state value is used, and per-partition states are ignored.)
 
         Returns:
             StreamState: A migrated state for parent streams in the format:
                 {
                     "parent_stream_name": {"parent_stream_cursor": "2023-05-27T00:00:00Z"}
                 }
+            where each parent stream with an incremental dependency is assigned its corresponding cursor value.
 
         Example:
             Input: {"updated_at": "2023-05-27T00:00:00Z"}
@@ -332,11 +336,15 @@ class SubstreamPartitionRouter(PartitionRouter):
         substream_state_values = list(stream_state.values())
         substream_state = substream_state_values[0] if substream_state_values else {}
 
-        # Ignore per-partition states or invalid formats
+        # Ignore per-partition states or invalid formats.
         if isinstance(substream_state, (list, dict)) or len(substream_state_values) != 1:
-            return {}
+            # If a global state is present under the key "state", use its first value.
+            if "state" in stream_state and isinstance(stream_state["state"], dict):
+                substream_state = list(stream_state["state"].values())[0]
+            else:
+                return {}
 
-        # Copy child state to parent streams with incremental dependencies
+        # Build the parent state for all parent streams with incremental dependencies.
         parent_state = {}
         if substream_state:
             for parent_config in self.parent_stream_configs:

{airbyte_cdk-6.33.0.dev0 → airbyte_cdk-6.33.1.dev0}/airbyte_cdk/sources/declarative/requesters/http_requester.py

@@ -202,6 +202,9 @@ class HttpRequester(Requester):
         Raise a ValueError if there's a key collision
         Returned merged mapping otherwise
         """
+
+        is_body_json = requester_method.__name__ == "get_request_body_json"
+
         return combine_mappings(
             [
                 requester_method(
@@ -211,7 +214,8 @@ class HttpRequester(Requester):
                 ),
                 auth_options_method(),
                 extra_options,
-            ]
+            ],
+            allow_same_value_merge=is_body_json,
         )
 
     def _request_headers(

{airbyte_cdk-6.33.0.dev0 → airbyte_cdk-6.33.1.dev0}/airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py

@@ -187,7 +187,7 @@ class DefaultPaginator(Paginator):
     def _get_request_options(
         self, option_type: RequestOptionType, next_page_token: Optional[Mapping[str, Any]]
     ) -> MutableMapping[str, Any]:
-        options = {}
+        options: MutableMapping[str, Any] = {}
 
         token = next_page_token.get("next_page_token") if next_page_token else None
         if (
@@ -196,15 +196,16 @@ class DefaultPaginator(Paginator):
             and isinstance(self.page_token_option, RequestOption)
             and self.page_token_option.inject_into == option_type
         ):
-            options[self.page_token_option.field_name.eval(config=self.config)] = token  # type: ignore # field_name is always cast to an interpolated string
+            self.page_token_option.inject_into_request(options, token, self.config)
+
         if (
             self.page_size_option
             and self.pagination_strategy.get_page_size()
             and self.page_size_option.inject_into == option_type
         ):
-            options[self.page_size_option.field_name.eval(config=self.config)] = (  # type: ignore [union-attr]
-                self.pagination_strategy.get_page_size()
-            )  # type: ignore # field_name is always cast to an interpolated string
+            page_size = self.pagination_strategy.get_page_size()
+            self.page_size_option.inject_into_request(options, page_size, self.config)
+
         return options