airbyte-cdk 6.33.2__tar.gz → 6.33.2.dev1__tar.gz

{airbyte_cdk-6.33.2 → airbyte_cdk-6.33.2.dev1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 6.33.2
+Version: 6.33.2.dev1
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://airbyte.com
 License: MIT

{airbyte_cdk-6.33.2 → airbyte_cdk-6.33.2.dev1}/airbyte_cdk/sources/declarative/auth/oauth.py

@@ -3,7 +3,7 @@
 #
 
 from dataclasses import InitVar, dataclass, field
-from datetime import datetime, timedelta
+from datetime import timedelta
 from typing import Any, List, Mapping, MutableMapping, Optional, Union
 
 from airbyte_cdk.sources.declarative.auth.declarative_authenticator import DeclarativeAuthenticator
@@ -232,13 +232,8 @@ class DeclarativeOauth2Authenticator(AbstractOauth2Authenticator, DeclarativeAut
         return self._refresh_request_headers.eval(self.config)
 
     def get_token_expiry_date(self) -> AirbyteDateTime:
-        if not self._has_access_token_been_initialized():
-            return AirbyteDateTime.from_datetime(datetime.min)
         return self._token_expiry_date  # type: ignore # _token_expiry_date is an AirbyteDateTime. It is never None despite what mypy thinks
 
-    def _has_access_token_been_initialized(self) -> bool:
-        return self._access_token is not None
-
     def set_token_expiry_date(self, value: Union[str, int]) -> None:
         self._token_expiry_date = self._parse_token_expiration_date(value)
 

{airbyte_cdk-6.33.2 → airbyte_cdk-6.33.2.dev1}/airbyte_cdk/sources/declarative/concurrent_declarative_source.py

@@ -3,7 +3,7 @@
 #
 
 import logging
-from typing import Any, Generic, Iterator, List, Mapping, MutableMapping, Optional, Tuple
+from typing import Any, Generic, Iterator, List, Mapping, Optional, Tuple
 
 from airbyte_cdk.models import (
     AirbyteCatalog,
@@ -224,7 +224,6 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
                     stream_state = self._connector_state_manager.get_stream_state(
                         stream_name=declarative_stream.name, namespace=declarative_stream.namespace
                     )
-                    stream_state = self._migrate_state(declarative_stream, stream_state)
 
                     retriever = self._get_retriever(declarative_stream, stream_state)
 
@@ -332,8 +331,6 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
                     stream_state = self._connector_state_manager.get_stream_state(
                         stream_name=declarative_stream.name, namespace=declarative_stream.namespace
                     )
-                    stream_state = self._migrate_state(declarative_stream, stream_state)
-
                     partition_router = declarative_stream.retriever.stream_slicer._partition_router
 
                     perpartition_cursor = (
@@ -524,14 +521,3 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
                 if stream.stream.name not in concurrent_stream_names
             ]
         )
-
-    @staticmethod
-    def _migrate_state(
-        declarative_stream: DeclarativeStream, stream_state: MutableMapping[str, Any]
-    ) -> MutableMapping[str, Any]:
-        for state_migration in declarative_stream.state_migrations:
-            if state_migration.should_migrate(stream_state):
-                # The state variable is expected to be mutable but the migrate method returns an immutable mapping.
-                stream_state = dict(state_migration.migrate(stream_state))
-
-        return stream_state

{airbyte_cdk-6.33.2 → airbyte_cdk-6.33.2.dev1}/airbyte_cdk/sources/declarative/declarative_component_schema.yaml

@@ -40,6 +40,12 @@ properties:
     "$ref": "#/definitions/Spec"
   concurrency_level:
     "$ref": "#/definitions/ConcurrencyLevel"
+  api_budget:
+    title: API Budget
+    description: Defines how many requests can be made to the API in a given time frame. This field accepts either a generic APIBudget or an HTTP-specific configuration (HTTPAPIBudget) to be applied across all streams.
+    anyOf:
+      - "$ref": "#/definitions/APIBudget"
+      - "$ref": "#/definitions/HTTPAPIBudget"
   metadata:
     type: object
     description: For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.
@@ -794,7 +800,7 @@ definitions:
         description: This option is used to adjust the upper and lower boundaries of each datetime window to beginning and end of the provided target period (day, week, month)
         type: object
         required:
-        - target
+          - target
         properties:
           target:
             title: Target
@@ -1365,6 +1371,207 @@ definitions:
       $parameters:
         type: object
         additional_properties: true
+  APIBudget:
+    title: API Budget
+    description: >
+      A generic API budget configuration that defines the policies (rate limiting rules)
+      and the maximum number of attempts to acquire a call credit. This budget does not automatically
+      update itself based on HTTP response headers.
+    type: object
+    required:
+      - type
+      - policies
+    properties:
+      type:
+        type: string
+        enum: [APIBudget]
+      policies:
+        title: Policies
+        description: List of call rate policies that define how many calls are allowed.
+        type: array
+        items:
+          anyOf:
+            - "$ref": "#/definitions/FixedWindowCallRatePolicy"
+            - "$ref": "#/definitions/MovingWindowCallRatePolicy"
+            - "$ref": "#/definitions/UnlimitedCallRatePolicy"
+      maximum_attempts_to_acquire:
+        title: Maximum Attempts to Acquire
+        description: The maximum number of attempts to acquire a call before giving up.
+        type: integer
+        default: 100000
+    additionalProperties: true
+  HTTPAPIBudget:
+    title: HTTP API Budget
+    description: >
+      An HTTP-specific API budget that extends APIBudget by updating rate limiting information based
+      on HTTP response headers. It extracts available calls and the next reset timestamp from the HTTP responses.
+    type: object
+    required:
+      - type
+      - policies
+    properties:
+      type:
+        type: string
+        enum: [HTTPAPIBudget]
+      policies:
+        title: Policies
+        description: List of call rate policies that define how many calls are allowed.
+        type: array
+        items:
+          anyOf:
+            - "$ref": "#/definitions/FixedWindowCallRatePolicy"
+            - "$ref": "#/definitions/MovingWindowCallRatePolicy"
+            - "$ref": "#/definitions/UnlimitedCallRatePolicy"
+      ratelimit_reset_header:
+        title: Rate Limit Reset Header
+        description: The HTTP response header name that indicates when the rate limit resets.
+        type: string
+        default: "ratelimit-reset"
+      ratelimit_remaining_header:
+        title: Rate Limit Remaining Header
+        description: The HTTP response header name that indicates the number of remaining allowed calls.
+        type: string
+        default: "ratelimit-remaining"
+      status_codes_for_ratelimit_hit:
+        title: Status Codes for Rate Limit Hit
+        description: List of HTTP status codes that indicate a rate limit has been hit.
+        type: array
+        items:
+          type: integer
+        default: [429]
+      maximum_attempts_to_acquire:
+        title: Maximum Attempts to Acquire
+        description: The maximum number of attempts to acquire a call before giving up.
+        type: integer
+        default: 100000
+    additionalProperties: true
+  FixedWindowCallRatePolicy:
+    title: Fixed Window Call Rate Policy
+    description: A policy that allows a fixed number of calls within a specific time window.
+    type: object
+    required:
+      - type
+      - next_reset_ts
+      - period
+      - call_limit
+      - matchers
+    properties:
+      type:
+        type: string
+        enum: [FixedWindowCallRatePolicy]
+      next_reset_ts:
+        title: Next Reset Timestamp
+        description: The timestamp when the rate limit will reset.
+        type: string
+        format: date-time
+      period:
+        title: Period
+        description: The time interval for the rate limit window.
+        type: string
+        format: duration
+      call_limit:
+        title: Call Limit
+        description: The maximum number of calls allowed within the period.
+        type: integer
+      matchers:
+        title: Matchers
+        description: List of matchers that define which requests this policy applies to.
+        type: array
+        items:
+          "$ref": "#/definitions/HttpRequestRegexMatcher"
+    additionalProperties: true
+  MovingWindowCallRatePolicy:
+    title: Moving Window Call Rate Policy
+    description: A policy that allows a fixed number of calls within a moving time window.
+    type: object
+    required:
+      - type
+      - rates
+      - matchers
+    properties:
+      type:
+        type: string
+        enum: [MovingWindowCallRatePolicy]
+      rates:
+        title: Rates
+        description: List of rates that define the call limits for different time intervals.
+        type: array
+        items:
+          "$ref": "#/definitions/Rate"
+      matchers:
+        title: Matchers
+        description: List of matchers that define which requests this policy applies to.
+        type: array
+        items:
+          "$ref": "#/definitions/HttpRequestRegexMatcher"
+    additionalProperties: true
+  UnlimitedCallRatePolicy:
+    title: Unlimited Call Rate Policy
+    description: A policy that allows unlimited calls for specific requests.
+    type: object
+    required:
+      - type
+      - matchers
+    properties:
+      type:
+        type: string
+        enum: [UnlimitedCallRatePolicy]
+      matchers:
+        title: Matchers
+        description: List of matchers that define which requests this policy applies to.
+        type: array
+        items:
+          "$ref": "#/definitions/HttpRequestRegexMatcher"
+    additionalProperties: true
+  Rate:
+    title: Rate
+    description: Defines a rate limit with a specific number of calls allowed within a time interval.
+    type: object
+    required:
+      - limit
+      - interval
+    properties:
+      limit:
+        title: Limit
+        description: The maximum number of calls allowed within the interval.
+        type: integer
+      interval:
+        title: Interval
+        description: The time interval for the rate limit.
+        type: string
+        format: duration
+    additionalProperties: true
+  HttpRequestRegexMatcher:
+    title: HTTP Request Matcher
+    description: >
+      Matches HTTP requests based on method, base URL, URL path pattern, query parameters, and headers.
+      Use `url_base` to specify the scheme and host (without trailing slash) and
+      `url_path_pattern` to apply a regex to the request path.
+    type: object
+    properties:
+      method:
+        title: Method
+        description: The HTTP method to match (e.g., GET, POST).
+        type: string
+      url_base:
+        title: URL Base
+        description: The base URL (scheme and host, e.g. "https://api.example.com") to match.
+        type: string
+      url_path_pattern:
+        title: URL Path Pattern
+        description: A regular expression pattern to match the URL path.
+        type: string
+      params:
+        title: Parameters
+        description: The query parameters to match.
+        type: object
+        additionalProperties: true
+      headers:
+        title: Headers
+        description: The headers to match.
+        type: object
+        additionalProperties: true
+    additionalProperties: true
   DefaultErrorHandler:
     title: Default Error Handler
     description: Component defining how to handle errors. Default behavior includes only retrying server errors (HTTP 5XX) and too many requests (HTTP 429) with an exponential backoff.

{airbyte_cdk-6.33.2 → airbyte_cdk-6.33.2.dev1}/airbyte_cdk/sources/declarative/incremental/concurrent_partition_cursor.py

@@ -58,7 +58,8 @@ class ConcurrentPerPartitionCursor(Cursor):
     CurrentPerPartitionCursor expects the state of the ConcurrentCursor to follow the format {cursor_field: cursor_value}.
     """
 
-    DEFAULT_MAX_PARTITIONS_NUMBER = 10000
+    DEFAULT_MAX_PARTITIONS_NUMBER = 25_000
+    SWITCH_TO_GLOBAL_LIMIT = 1000
     _NO_STATE: Mapping[str, Any] = {}
     _NO_CURSOR_STATE: Mapping[str, Any] = {}
     _GLOBAL_STATE_KEY = "state"
@@ -99,7 +100,7 @@ class ConcurrentPerPartitionCursor(Cursor):
         self._new_global_cursor: Optional[StreamState] = None
         self._lookback_window: int = 0
         self._parent_state: Optional[StreamState] = None
-        self._over_limit: int = 0
+        self._number_of_partitions: int = 0
         self._use_global_cursor: bool = False
         self._partition_serializer = PerPartitionKeySerializer()
 
@@ -233,8 +234,8 @@ class ConcurrentPerPartitionCursor(Cursor):
           or removed due to being the oldest.
         """
         with self._lock:
+            self._number_of_partitions += 1
             while len(self._cursor_per_partition) > self.DEFAULT_MAX_PARTITIONS_NUMBER - 1:
-                self._over_limit += 1
                 # Try removing finished partitions first
                 for partition_key in list(self._cursor_per_partition.keys()):
                     if (
@@ -245,7 +246,7 @@ class ConcurrentPerPartitionCursor(Cursor):
                             partition_key
                         )  # Remove the oldest partition
                         logger.warning(
-                            f"The maximum number of partitions has been reached. Dropping the oldest finished partition: {oldest_partition}. Over limit: {self._over_limit}."
+                            f"The maximum number of partitions has been reached. Dropping the oldest finished partition: {oldest_partition}. Over limit: {self._number_of_partitions}."
                         )
                         break
                 else:
@@ -254,7 +255,7 @@ class ConcurrentPerPartitionCursor(Cursor):
                         1
                     ]  # Remove the oldest partition
                     logger.warning(
-                        f"The maximum number of partitions has been reached. Dropping the oldest partition: {oldest_partition}. Over limit: {self._over_limit}."
+                        f"The maximum number of partitions has been reached. Dropping the oldest partition: {oldest_partition}. Over limit: {self._number_of_partitions}."
                     )
 
     def _set_initial_state(self, stream_state: StreamState) -> None:
@@ -355,6 +356,10 @@ class ConcurrentPerPartitionCursor(Cursor):
 
     def observe(self, record: Record) -> None:
         if not self._use_global_cursor and self.limit_reached():
+            logger.info(
+                f"Exceeded the 'SWITCH_TO_GLOBAL_LIMIT' of {self.SWITCH_TO_GLOBAL_LIMIT}. "
+                f"Switching to global cursor for {self._stream_name}."
+            )
             self._use_global_cursor = True
 
         if not record.associated_slice:
@@ -397,4 +402,4 @@ class ConcurrentPerPartitionCursor(Cursor):
         return cursor
 
     def limit_reached(self) -> bool:
-        return self._over_limit > self.DEFAULT_MAX_PARTITIONS_NUMBER
+        return self._number_of_partitions > self.SWITCH_TO_GLOBAL_LIMIT

{airbyte_cdk-6.33.2 → airbyte_cdk-6.33.2.dev1}/airbyte_cdk/sources/declarative/manifest_declarative_source.py

@@ -137,6 +137,10 @@ class ManifestDeclarativeSource(DeclarativeSource):
             self._source_config, config
         )
 
+        api_budget_model = self._source_config.get("api_budget")
+        if api_budget_model:
+            self._constructor.set_api_budget(api_budget_model, config)
+
         source_streams = [
             self._constructor.create_component(
                 DeclarativeStreamModel,

{airbyte_cdk-6.33.2 → airbyte_cdk-6.33.2.dev1}/airbyte_cdk/sources/declarative/models/declarative_component_schema.py

@@ -3,6 +3,7 @@
 
 from __future__ import annotations
 
+from datetime import datetime, timedelta
 from enum import Enum
 from typing import Any, Dict, List, Literal, Optional, Union
 
@@ -642,6 +643,45 @@ class OAuthAuthenticator(BaseModel):
     parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
 
 
+class Rate(BaseModel):
+    class Config:
+        extra = Extra.allow
+
+    limit: int = Field(
+        ...,
+        description="The maximum number of calls allowed within the interval.",
+        title="Limit",
+    )
+    interval: timedelta = Field(
+        ..., description="The time interval for the rate limit.", title="Interval"
+    )
+
+
+class HttpRequestRegexMatcher(BaseModel):
+    class Config:
+        extra = Extra.allow
+
+    method: Optional[str] = Field(
+        None, description="The HTTP method to match (e.g., GET, POST).", title="Method"
+    )
+    url_base: Optional[str] = Field(
+        None,
+        description='The base URL (scheme and host, e.g. "https://api.example.com") to match.',
+        title="URL Base",
+    )
+    url_path_pattern: Optional[str] = Field(
+        None,
+        description="A regular expression pattern to match the URL path.",
+        title="URL Path Pattern",
+    )
+    params: Optional[Dict[str, Any]] = Field(
+        None, description="The query parameters to match.", title="Parameters"
+    )
+    headers: Optional[Dict[str, Any]] = Field(
+        None, description="The headers to match.", title="Headers"
+    )
+
+
 class DpathExtractor(BaseModel):
     type: Literal["DpathExtractor"]
     field_path: List[str] = Field(
@@ -1584,6 +1624,60 @@ class DatetimeBasedCursor(BaseModel):
     parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
 
 
+class FixedWindowCallRatePolicy(BaseModel):
+    class Config:
+        extra = Extra.allow
+
+    type: Literal["FixedWindowCallRatePolicy"]
+    next_reset_ts: datetime = Field(
+        ...,
+        description="The timestamp when the rate limit will reset.",
+        title="Next Reset Timestamp",
+    )
+    period: timedelta = Field(
+        ..., description="The time interval for the rate limit window.", title="Period"
+    )
+    call_limit: int = Field(
+        ...,
+        description="The maximum number of calls allowed within the period.",
+        title="Call Limit",
+    )
+    matchers: List[HttpRequestRegexMatcher] = Field(
+        ...,
+        description="List of matchers that define which requests this policy applies to.",
+        title="Matchers",
+    )
+
+
+class MovingWindowCallRatePolicy(BaseModel):
+    class Config:
+        extra = Extra.allow
+
+    type: Literal["MovingWindowCallRatePolicy"]
+    rates: List[Rate] = Field(
+        ...,
+        description="List of rates that define the call limits for different time intervals.",
+        title="Rates",
+    )
+    matchers: List[HttpRequestRegexMatcher] = Field(
+        ...,
+        description="List of matchers that define which requests this policy applies to.",
+        title="Matchers",
+    )
+
+
+class UnlimitedCallRatePolicy(BaseModel):
+    class Config:
+        extra = Extra.allow
+
+    type: Literal["UnlimitedCallRatePolicy"]
+    matchers: List[HttpRequestRegexMatcher] = Field(
+        ...,
+        description="List of matchers that define which requests this policy applies to.",
+        title="Matchers",
+    )
+
+
 class DefaultErrorHandler(BaseModel):
     type: Literal["DefaultErrorHandler"]
     backoff_strategies: Optional[
@@ -1715,6 +1809,67 @@ class CompositeErrorHandler(BaseModel):
     parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
 
 
+class APIBudget(BaseModel):
+    class Config:
+        extra = Extra.allow
+
+    type: Literal["APIBudget"]
+    policies: List[
+        Union[
+            FixedWindowCallRatePolicy,
+            MovingWindowCallRatePolicy,
+            UnlimitedCallRatePolicy,
+        ]
+    ] = Field(
+        ...,
+        description="List of call rate policies that define how many calls are allowed.",
+        title="Policies",
+    )
+    maximum_attempts_to_acquire: Optional[int] = Field(
+        100000,
+        description="The maximum number of attempts to acquire a call before giving up.",
+        title="Maximum Attempts to Acquire",
+    )
+
+
+class HTTPAPIBudget(BaseModel):
+    class Config:
+        extra = Extra.allow
+
+    type: Literal["HTTPAPIBudget"]
+    policies: List[
+        Union[
+            FixedWindowCallRatePolicy,
+            MovingWindowCallRatePolicy,
+            UnlimitedCallRatePolicy,
+        ]
+    ] = Field(
+        ...,
+        description="List of call rate policies that define how many calls are allowed.",
+        title="Policies",
+    )
+    ratelimit_reset_header: Optional[str] = Field(
+        "ratelimit-reset",
+        description="The HTTP response header name that indicates when the rate limit resets.",
+        title="Rate Limit Reset Header",
+    )
+    ratelimit_remaining_header: Optional[str] = Field(
+        "ratelimit-remaining",
+        description="The HTTP response header name that indicates the number of remaining allowed calls.",
+        title="Rate Limit Remaining Header",
+    )
+    status_codes_for_ratelimit_hit: Optional[List[int]] = Field(
+        [429],
+        description="List of HTTP status codes that indicate a rate limit has been hit.",
+        title="Status Codes for Rate Limit Hit",
+    )
+    maximum_attempts_to_acquire: Optional[int] = Field(
+        100000,
+        description="The maximum number of attempts to acquire a call before giving up.",
+        title="Maximum Attempts to Acquire",
+    )
+
+
 class ZipfileDecoder(BaseModel):
     class Config:
         extra = Extra.allow
@@ -1748,6 +1903,11 @@ class DeclarativeSource1(BaseModel):
     definitions: Optional[Dict[str, Any]] = None
     spec: Optional[Spec] = None
     concurrency_level: Optional[ConcurrencyLevel] = None
+    api_budget: Optional[Union[APIBudget, HTTPAPIBudget]] = Field(
+        None,
+        description="Defines how many requests can be made to the API in a given time frame. This field accepts either a generic APIBudget or an HTTP-specific configuration (HTTPAPIBudget) to be applied across all streams.",
+        title="API Budget",
+    )
     metadata: Optional[Dict[str, Any]] = Field(
         None,
         description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.",
@@ -1774,6 +1934,11 @@ class DeclarativeSource2(BaseModel):
     definitions: Optional[Dict[str, Any]] = None
     spec: Optional[Spec] = None
     concurrency_level: Optional[ConcurrencyLevel] = None
+    api_budget: Optional[Union[APIBudget, HTTPAPIBudget]] = Field(
+        None,
+        description="Defines how many requests can be made to the API in a given time frame. This field accepts either a generic APIBudget or an HTTP-specific configuration (HTTPAPIBudget) to be applied across all streams.",
+        title="API Budget",
+    )
     metadata: Optional[Dict[str, Any]] = Field(
         None,
         description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.",