airbyte-cdk 6.33.0.dev0__py3-none-any.whl → 6.33.1__py3-none-any.whl

airbyte_cdk/sources/declarative/auth/token.py

@@ -5,7 +5,7 @@
 import base64
 import logging
 from dataclasses import InitVar, dataclass
-from typing import Any, Mapping, Union
+from typing import Any, Mapping, MutableMapping, Union
 
 import requests
 from cachetools import TTLCache, cached
@@ -45,11 +45,6 @@ class ApiKeyAuthenticator(DeclarativeAuthenticator):
     config: Config
     parameters: InitVar[Mapping[str, Any]]
 
-    def __post_init__(self, parameters: Mapping[str, Any]) -> None:
-        self._field_name = InterpolatedString.create(
-            self.request_option.field_name, parameters=parameters
-        )
-
     @property
     def auth_header(self) -> str:
         options = self._get_request_options(RequestOptionType.header)
@@ -60,9 +55,9 @@ class ApiKeyAuthenticator(DeclarativeAuthenticator):
         return self.token_provider.get_token()
 
     def _get_request_options(self, option_type: RequestOptionType) -> Mapping[str, Any]:
-        options = {}
+        options: MutableMapping[str, Any] = {}
         if self.request_option.inject_into == option_type:
-            options[self._field_name.eval(self.config)] = self.token
+            self.request_option.inject_into_request(options, self.token, self.config)
         return options
 
     def get_request_params(self) -> Mapping[str, Any]:

airbyte_cdk/sources/declarative/concurrent_declarative_source.py

@@ -475,10 +475,21 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
             # Also a temporary hack. In the legacy Stream implementation, as part of the read,
             # set_initial_state() is called to instantiate incoming state on the cursor. Although we no
             # longer rely on the legacy low-code cursor for concurrent checkpointing, low-code components
-            # like StopConditionPaginationStrategyDecorator and ClientSideIncrementalRecordFilterDecorator
-            # still rely on a DatetimeBasedCursor that is properly initialized with state.
+            # like StopConditionPaginationStrategyDecorator still rely on a DatetimeBasedCursor that is
+            # properly initialized with state.
             if retriever.cursor:
                 retriever.cursor.set_initial_state(stream_state=stream_state)
+
+            # Similar to above, the ClientSideIncrementalRecordFilterDecorator cursor is a separate instance
+            # from the one initialized on the SimpleRetriever, so it also must also have state initialized
+            # for semi-incremental streams using is_client_side_incremental to filter properly
+            if isinstance(retriever.record_selector, RecordSelector) and isinstance(
+                retriever.record_selector.record_filter, ClientSideIncrementalRecordFilterDecorator
+            ):
+                retriever.record_selector.record_filter._cursor.set_initial_state(
+                    stream_state=stream_state
+                )  # type: ignore  # After non-concurrent cursors are deprecated we can remove these cursor workarounds
+
             # We zero it out here, but since this is a cursor reference, the state is still properly
             # instantiated for the other components that reference it
             retriever.cursor = None

airbyte_cdk/sources/declarative/declarative_component_schema.yaml

@@ -40,12 +40,6 @@ properties:
     "$ref": "#/definitions/Spec"
   concurrency_level:
     "$ref": "#/definitions/ConcurrencyLevel"
-  api_budget:
-    title: API Budget
-    description: Defines how many requests can be made to the API in a given time frame. This field accepts either a generic APIBudget or an HTTP-specific configuration (HTTPAPIBudget) to be applied across all streams.
-    anyOf:
-      - "$ref": "#/definitions/APIBudget"
-      - "$ref": "#/definitions/HTTPAPIBudget"
   metadata:
     type: object
     description: For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.
@@ -800,7 +794,7 @@ definitions:
         description: This option is used to adjust the upper and lower boundaries of each datetime window to beginning and end of the provided target period (day, week, month)
         type: object
         required:
-          - target
+        - target
         properties:
           target:
             title: Target
@@ -1371,207 +1365,6 @@ definitions:
       $parameters:
         type: object
         additional_properties: true
-  APIBudget:
-    title: API Budget
-    description: >
-      A generic API budget configuration that defines the policies (rate limiting rules)
-      and the maximum number of attempts to acquire a call credit. This budget does not automatically
-      update itself based on HTTP response headers.
-    type: object
-    required:
-      - type
-      - policies
-    properties:
-      type:
-        type: string
-        enum: [APIBudget]
-      policies:
-        title: Policies
-        description: List of call rate policies that define how many calls are allowed.
-        type: array
-        items:
-          anyOf:
-            - "$ref": "#/definitions/FixedWindowCallRatePolicy"
-            - "$ref": "#/definitions/MovingWindowCallRatePolicy"
-            - "$ref": "#/definitions/UnlimitedCallRatePolicy"
-      maximum_attempts_to_acquire:
-        title: Maximum Attempts to Acquire
-        description: The maximum number of attempts to acquire a call before giving up.
-        type: integer
-        default: 100000
-    additionalProperties: true
-  HTTPAPIBudget:
-    title: HTTP API Budget
-    description: >
-      An HTTP-specific API budget that extends APIBudget by updating rate limiting information based
-      on HTTP response headers. It extracts available calls and the next reset timestamp from the HTTP responses.
-    type: object
-    required:
-      - type
-      - policies
-    properties:
-      type:
-        type: string
-        enum: [HTTPAPIBudget]
-      policies:
-        title: Policies
-        description: List of call rate policies that define how many calls are allowed.
-        type: array
-        items:
-          anyOf:
-            - "$ref": "#/definitions/FixedWindowCallRatePolicy"
-            - "$ref": "#/definitions/MovingWindowCallRatePolicy"
-            - "$ref": "#/definitions/UnlimitedCallRatePolicy"
-      ratelimit_reset_header:
-        title: Rate Limit Reset Header
-        description: The HTTP response header name that indicates when the rate limit resets.
-        type: string
-        default: "ratelimit-reset"
-      ratelimit_remaining_header:
-        title: Rate Limit Remaining Header
-        description: The HTTP response header name that indicates the number of remaining allowed calls.
-        type: string
-        default: "ratelimit-remaining"
-      status_codes_for_ratelimit_hit:
-        title: Status Codes for Rate Limit Hit
-        description: List of HTTP status codes that indicate a rate limit has been hit.
-        type: array
-        items:
-          type: integer
-        default: [429]
-      maximum_attempts_to_acquire:
-        title: Maximum Attempts to Acquire
-        description: The maximum number of attempts to acquire a call before giving up.
-        type: integer
-        default: 100000
-    additionalProperties: true
-  FixedWindowCallRatePolicy:
-    title: Fixed Window Call Rate Policy
-    description: A policy that allows a fixed number of calls within a specific time window.
-    type: object
-    required:
-      - type
-      - next_reset_ts
-      - period
-      - call_limit
-      - matchers
-    properties:
-      type:
-        type: string
-        enum: [FixedWindowCallRatePolicy]
-      next_reset_ts:
-        title: Next Reset Timestamp
-        description: The timestamp when the rate limit will reset.
-        type: string
-        format: date-time
-      period:
-        title: Period
-        description: The time interval for the rate limit window.
-        type: string
-        format: duration
-      call_limit:
-        title: Call Limit
-        description: The maximum number of calls allowed within the period.
-        type: integer
-      matchers:
-        title: Matchers
-        description: List of matchers that define which requests this policy applies to.
-        type: array
-        items:
-          "$ref": "#/definitions/HttpRequestMatcher"
-    additionalProperties: true
-  MovingWindowCallRatePolicy:
-    title: Moving Window Call Rate Policy
-    description: A policy that allows a fixed number of calls within a moving time window.
-    type: object
-    required:
-      - type
-      - rates
-      - matchers
-    properties:
-      type:
-        type: string
-        enum: [MovingWindowCallRatePolicy]
-      rates:
-        title: Rates
-        description: List of rates that define the call limits for different time intervals.
-        type: array
-        items:
-          "$ref": "#/definitions/Rate"
-      matchers:
-        title: Matchers
-        description: List of matchers that define which requests this policy applies to.
-        type: array
-        items:
-          "$ref": "#/definitions/HttpRequestMatcher"
-    additionalProperties: true
-  UnlimitedCallRatePolicy:
-    title: Unlimited Call Rate Policy
-    description: A policy that allows unlimited calls for specific requests.
-    type: object
-    required:
-      - type
-      - matchers
-    properties:
-      type:
-        type: string
-        enum: [UnlimitedCallRatePolicy]
-      matchers:
-        title: Matchers
-        description: List of matchers that define which requests this policy applies to.
-        type: array
-        items:
-          "$ref": "#/definitions/HttpRequestMatcher"
-    additionalProperties: true
-  Rate:
-    title: Rate
-    description: Defines a rate limit with a specific number of calls allowed within a time interval.
-    type: object
-    required:
-      - limit
-      - interval
-    properties:
-      limit:
-        title: Limit
-        description: The maximum number of calls allowed within the interval.
-        type: integer
-      interval:
-        title: Interval
-        description: The time interval for the rate limit.
-        type: string
-        format: duration
-    additionalProperties: true
-  HttpRequestMatcher:
-    title: HTTP Request Matcher
-    description: >
-      Matches HTTP requests based on method, base URL, URL path pattern, query parameters, and headers.
-      Use `url_base` to specify the scheme and host (without trailing slash) and
-      `url_path_pattern` to apply a regex to the request path.
-    type: object
-    properties:
-      method:
-        title: Method
-        description: The HTTP method to match (e.g., GET, POST).
-        type: string
-      url_base:
-        title: URL Base
-        description: The base URL (scheme and host, e.g. "https://api.example.com") to match.
-        type: string
-      url_path_pattern:
-        title: URL Path Pattern
-        description: A regular expression pattern to match the URL path.
-        type: string
-      params:
-        title: Parameters
-        description: The query parameters to match.
-        type: object
-        additionalProperties: true
-      headers:
-        title: Headers
-        description: The headers to match.
-        type: object
-        additionalProperties: true
-    additionalProperties: true
   DefaultErrorHandler:
     title: Default Error Handler
     description: Component defining how to handle errors. Default behavior includes only retrying server errors (HTTP 5XX) and too many requests (HTTP 429) with an exponential backoff.
@@ -3054,25 +2847,35 @@ definitions:
         enum: [RequestPath]
   RequestOption:
     title: Request Option
-    description: Specifies the key field and where in the request a component's value should be injected.
+    description: Specifies the key field or path and where in the request a component's value should be injected.
     type: object
     required:
       - type
-      - field_name
       - inject_into
     properties:
       type:
         type: string
         enum: [RequestOption]
       field_name:
-        title: Request Option
-        description: Configures which key should be used in the location that the descriptor is being injected into
+        title: Field Name
+        description: Configures which key should be used in the location that the descriptor is being injected into. We hope to eventually deprecate this field in favor of `field_path` for all request_options, but must currently maintain it for backwards compatibility in the Builder.
         type: string
         examples:
           - segment_id
         interpolation_context:
           - config
           - parameters
+      field_path:
+        title: Field Path
+        description: Configures a path to be used for nested structures in JSON body requests (e.g. GraphQL queries)
+        type: array
+        items:
+          type: string
+        examples:
+          - ["data", "viewer", "id"]
+        interpolation_context:
+          - config
+          - parameters
       inject_into:
         title: Inject Into
         description: Configures where the descriptor should be set on the HTTP requests. Note that request parameters that are already encoded in the URL path will not be duplicated.

airbyte_cdk/sources/declarative/incremental/datetime_based_cursor.py

@@ -365,14 +365,15 @@ class DatetimeBasedCursor(DeclarativeCursor):
         options: MutableMapping[str, Any] = {}
         if not stream_slice:
             return options
+
         if self.start_time_option and self.start_time_option.inject_into == option_type:
-            options[self.start_time_option.field_name.eval(config=self.config)] = stream_slice.get(  # type: ignore # field_name is always casted to an interpolated string
-                self._partition_field_start.eval(self.config)
-            )
+            start_time_value = stream_slice.get(self._partition_field_start.eval(self.config))
+            self.start_time_option.inject_into_request(options, start_time_value, self.config)
+
         if self.end_time_option and self.end_time_option.inject_into == option_type:
-            options[self.end_time_option.field_name.eval(config=self.config)] = stream_slice.get(  # type: ignore [union-attr]
-                self._partition_field_end.eval(self.config)
-            )
+            end_time_value = stream_slice.get(self._partition_field_end.eval(self.config))
+            self.end_time_option.inject_into_request(options, end_time_value, self.config)
+
         return options
 
     def should_be_synced(self, record: Record) -> bool:

airbyte_cdk/sources/declarative/manifest_declarative_source.py

@@ -137,10 +137,6 @@ class ManifestDeclarativeSource(DeclarativeSource):
             self._source_config, config
         )
 
-        api_budget_model = self._source_config.get("api_budget")
-        if api_budget_model:
-            self._constructor.set_api_budget(api_budget_model, config)
-
         source_streams = [
             self._constructor.create_component(
                 DeclarativeStreamModel,

airbyte_cdk/sources/declarative/models/declarative_component_schema.py

@@ -3,7 +3,6 @@
 
 from __future__ import annotations
 
-from datetime import datetime, timedelta
 from enum import Enum
 from typing import Any, Dict, List, Literal, Optional, Union
 
@@ -643,45 +642,6 @@ class OAuthAuthenticator(BaseModel):
     parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
 
 
-class Rate(BaseModel):
-    class Config:
-        extra = Extra.allow
-
-    limit: int = Field(
-        ...,
-        description="The maximum number of calls allowed within the interval.",
-        title="Limit",
-    )
-    interval: timedelta = Field(
-        ..., description="The time interval for the rate limit.", title="Interval"
-    )
-
-
-class HttpRequestMatcher(BaseModel):
-    class Config:
-        extra = Extra.allow
-
-    method: Optional[str] = Field(
-        None, description="The HTTP method to match (e.g., GET, POST).", title="Method"
-    )
-    url_base: Optional[str] = Field(
-        None,
-        description='The base URL (scheme and host, e.g. "https://api.example.com") to match.',
-        title="URL Base",
-    )
-    url_path_pattern: Optional[str] = Field(
-        None,
-        description="A regular expression pattern to match the URL path.",
-        title="URL Path Pattern",
-    )
-    params: Optional[Dict[str, Any]] = Field(
-        None, description="The query parameters to match.", title="Parameters"
-    )
-    headers: Optional[Dict[str, Any]] = Field(
-        None, description="The headers to match.", title="Headers"
-    )
-
-
 class DpathExtractor(BaseModel):
     type: Literal["DpathExtractor"]
     field_path: List[str] = Field(
@@ -1240,11 +1200,17 @@ class InjectInto(Enum):
 
 class RequestOption(BaseModel):
     type: Literal["RequestOption"]
-    field_name: str = Field(
-        ...,
-        description="Configures which key should be used in the location that the descriptor is being injected into",
+    field_name: Optional[str] = Field(
+        None,
+        description="Configures which key should be used in the location that the descriptor is being injected into. We hope to eventually deprecate this field in favor of `field_path` for all request_options, but must currently maintain it for backwards compatibility in the Builder.",
         examples=["segment_id"],
-        title="Request Option",
+        title="Field Name",
+    )
+    field_path: Optional[List[str]] = Field(
+        None,
+        description="Configures a path to be used for nested structures in JSON body requests (e.g. GraphQL queries)",
+        examples=[["data", "viewer", "id"]],
+        title="Field Path",
     )
     inject_into: InjectInto = Field(
         ...,
@@ -1618,60 +1584,6 @@ class DatetimeBasedCursor(BaseModel):
     parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
 
 
-class FixedWindowCallRatePolicy(BaseModel):
-    class Config:
-        extra = Extra.allow
-
-    type: Literal["FixedWindowCallRatePolicy"]
-    next_reset_ts: datetime = Field(
-        ...,
-        description="The timestamp when the rate limit will reset.",
-        title="Next Reset Timestamp",
-    )
-    period: timedelta = Field(
-        ..., description="The time interval for the rate limit window.", title="Period"
-    )
-    call_limit: int = Field(
-        ...,
-        description="The maximum number of calls allowed within the period.",
-        title="Call Limit",
-    )
-    matchers: List[HttpRequestMatcher] = Field(
-        ...,
-        description="List of matchers that define which requests this policy applies to.",
-        title="Matchers",
-    )
-
-
-class MovingWindowCallRatePolicy(BaseModel):
-    class Config:
-        extra = Extra.allow
-
-    type: Literal["MovingWindowCallRatePolicy"]
-    rates: List[Rate] = Field(
-        ...,
-        description="List of rates that define the call limits for different time intervals.",
-        title="Rates",
-    )
-    matchers: List[HttpRequestMatcher] = Field(
-        ...,
-        description="List of matchers that define which requests this policy applies to.",
-        title="Matchers",
-    )
-
-
-class UnlimitedCallRatePolicy(BaseModel):
-    class Config:
-        extra = Extra.allow
-
-    type: Literal["UnlimitedCallRatePolicy"]
-    matchers: List[HttpRequestMatcher] = Field(
-        ...,
-        description="List of matchers that define which requests this policy applies to.",
-        title="Matchers",
-    )
-
-
 class DefaultErrorHandler(BaseModel):
     type: Literal["DefaultErrorHandler"]
     backoff_strategies: Optional[
@@ -1803,67 +1715,6 @@ class CompositeErrorHandler(BaseModel):
     parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
 
 
-class APIBudget(BaseModel):
-    class Config:
-        extra = Extra.allow
-
-    type: Literal["APIBudget"]
-    policies: List[
-        Union[
-            FixedWindowCallRatePolicy,
-            MovingWindowCallRatePolicy,
-            UnlimitedCallRatePolicy,
-        ]
-    ] = Field(
-        ...,
-        description="List of call rate policies that define how many calls are allowed.",
-        title="Policies",
-    )
-    maximum_attempts_to_acquire: Optional[int] = Field(
-        100000,
-        description="The maximum number of attempts to acquire a call before giving up.",
-        title="Maximum Attempts to Acquire",
-    )
-
-
-class HTTPAPIBudget(BaseModel):
-    class Config:
-        extra = Extra.allow
-
-    type: Literal["HTTPAPIBudget"]
-    policies: List[
-        Union[
-            FixedWindowCallRatePolicy,
-            MovingWindowCallRatePolicy,
-            UnlimitedCallRatePolicy,
-        ]
-    ] = Field(
-        ...,
-        description="List of call rate policies that define how many calls are allowed.",
-        title="Policies",
-    )
-    ratelimit_reset_header: Optional[str] = Field(
-        "ratelimit-reset",
-        description="The HTTP response header name that indicates when the rate limit resets.",
-        title="Rate Limit Reset Header",
-    )
-    ratelimit_remaining_header: Optional[str] = Field(
-        "ratelimit-remaining",
-        description="The HTTP response header name that indicates the number of remaining allowed calls.",
-        title="Rate Limit Remaining Header",
-    )
-    status_codes_for_ratelimit_hit: Optional[List[int]] = Field(
-        [429],
-        description="List of HTTP status codes that indicate a rate limit has been hit.",
-        title="Status Codes for Rate Limit Hit",
-    )
-    maximum_attempts_to_acquire: Optional[int] = Field(
-        100000,
-        description="The maximum number of attempts to acquire a call before giving up.",
-        title="Maximum Attempts to Acquire",
-    )
-
-
 class ZipfileDecoder(BaseModel):
     class Config:
         extra = Extra.allow
@@ -1897,11 +1748,6 @@ class DeclarativeSource1(BaseModel):
     definitions: Optional[Dict[str, Any]] = None
     spec: Optional[Spec] = None
     concurrency_level: Optional[ConcurrencyLevel] = None
-    api_budget: Optional[Union[APIBudget, HTTPAPIBudget]] = Field(
-        None,
-        description="Defines how many requests can be made to the API in a given time frame. This field accepts either a generic APIBudget or an HTTP-specific configuration (HTTPAPIBudget) to be applied across all streams.",
-        title="API Budget",
-    )
     metadata: Optional[Dict[str, Any]] = Field(
         None,
         description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.",
@@ -1928,11 +1774,6 @@ class DeclarativeSource2(BaseModel):
     definitions: Optional[Dict[str, Any]] = None
     spec: Optional[Spec] = None
     concurrency_level: Optional[ConcurrencyLevel] = None
-    api_budget: Optional[Union[APIBudget, HTTPAPIBudget]] = Field(
-        None,
-        description="Defines how many requests can be made to the API in a given time frame. This field accepts either a generic APIBudget or an HTTP-specific configuration (HTTPAPIBudget) to be applied across all streams.",
-        title="API Budget",
-    )
     metadata: Optional[Dict[str, Any]] = Field(
         None,
         description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.",