airbyte-cdk 6.32.0__py3-none-any.whl → 6.33.0.dev0__py3-none-any.whl

airbyte_cdk/sources/declarative/declarative_component_schema.yaml

@@ -40,6 +40,12 @@ properties:
     "$ref": "#/definitions/Spec"
   concurrency_level:
     "$ref": "#/definitions/ConcurrencyLevel"
+  api_budget:
+    title: API Budget
+    description: Defines how many requests can be made to the API in a given time frame. This field accepts either a generic APIBudget or an HTTP-specific configuration (HTTPAPIBudget) to be applied across all streams.
+    anyOf:
+      - "$ref": "#/definitions/APIBudget"
+      - "$ref": "#/definitions/HTTPAPIBudget"
   metadata:
     type: object
     description: For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.
@@ -794,7 +800,7 @@ definitions:
         description: This option is used to adjust the upper and lower boundaries of each datetime window to beginning and end of the provided target period (day, week, month)
         type: object
         required:
-        - target
+          - target
         properties:
           target:
             title: Target
@@ -1365,6 +1371,207 @@ definitions:
       $parameters:
         type: object
         additional_properties: true
+  APIBudget:
+    title: API Budget
+    description: >
+      A generic API budget configuration that defines the policies (rate limiting rules)
+      and the maximum number of attempts to acquire a call credit. This budget does not automatically
+      update itself based on HTTP response headers.
+    type: object
+    required:
+      - type
+      - policies
+    properties:
+      type:
+        type: string
+        enum: [APIBudget]
+      policies:
+        title: Policies
+        description: List of call rate policies that define how many calls are allowed.
+        type: array
+        items:
+          anyOf:
+            - "$ref": "#/definitions/FixedWindowCallRatePolicy"
+            - "$ref": "#/definitions/MovingWindowCallRatePolicy"
+            - "$ref": "#/definitions/UnlimitedCallRatePolicy"
+      maximum_attempts_to_acquire:
+        title: Maximum Attempts to Acquire
+        description: The maximum number of attempts to acquire a call before giving up.
+        type: integer
+        default: 100000
+    additionalProperties: true
+  HTTPAPIBudget:
+    title: HTTP API Budget
+    description: >
+      An HTTP-specific API budget that extends APIBudget by updating rate limiting information based
+      on HTTP response headers. It extracts available calls and the next reset timestamp from the HTTP responses.
+    type: object
+    required:
+      - type
+      - policies
+    properties:
+      type:
+        type: string
+        enum: [HTTPAPIBudget]
+      policies:
+        title: Policies
+        description: List of call rate policies that define how many calls are allowed.
+        type: array
+        items:
+          anyOf:
+            - "$ref": "#/definitions/FixedWindowCallRatePolicy"
+            - "$ref": "#/definitions/MovingWindowCallRatePolicy"
+            - "$ref": "#/definitions/UnlimitedCallRatePolicy"
+      ratelimit_reset_header:
+        title: Rate Limit Reset Header
+        description: The HTTP response header name that indicates when the rate limit resets.
+        type: string
+        default: "ratelimit-reset"
+      ratelimit_remaining_header:
+        title: Rate Limit Remaining Header
+        description: The HTTP response header name that indicates the number of remaining allowed calls.
+        type: string
+        default: "ratelimit-remaining"
+      status_codes_for_ratelimit_hit:
+        title: Status Codes for Rate Limit Hit
+        description: List of HTTP status codes that indicate a rate limit has been hit.
+        type: array
+        items:
+          type: integer
+        default: [429]
+      maximum_attempts_to_acquire:
+        title: Maximum Attempts to Acquire
+        description: The maximum number of attempts to acquire a call before giving up.
+        type: integer
+        default: 100000
+    additionalProperties: true
+  FixedWindowCallRatePolicy:
+    title: Fixed Window Call Rate Policy
+    description: A policy that allows a fixed number of calls within a specific time window.
+    type: object
+    required:
+      - type
+      - next_reset_ts
+      - period
+      - call_limit
+      - matchers
+    properties:
+      type:
+        type: string
+        enum: [FixedWindowCallRatePolicy]
+      next_reset_ts:
+        title: Next Reset Timestamp
+        description: The timestamp when the rate limit will reset.
+        type: string
+        format: date-time
+      period:
+        title: Period
+        description: The time interval for the rate limit window.
+        type: string
+        format: duration
+      call_limit:
+        title: Call Limit
+        description: The maximum number of calls allowed within the period.
+        type: integer
+      matchers:
+        title: Matchers
+        description: List of matchers that define which requests this policy applies to.
+        type: array
+        items:
+          "$ref": "#/definitions/HttpRequestMatcher"
+    additionalProperties: true
+  MovingWindowCallRatePolicy:
+    title: Moving Window Call Rate Policy
+    description: A policy that allows a fixed number of calls within a moving time window.
+    type: object
+    required:
+      - type
+      - rates
+      - matchers
+    properties:
+      type:
+        type: string
+        enum: [MovingWindowCallRatePolicy]
+      rates:
+        title: Rates
+        description: List of rates that define the call limits for different time intervals.
+        type: array
+        items:
+          "$ref": "#/definitions/Rate"
+      matchers:
+        title: Matchers
+        description: List of matchers that define which requests this policy applies to.
+        type: array
+        items:
+          "$ref": "#/definitions/HttpRequestMatcher"
+    additionalProperties: true
+  UnlimitedCallRatePolicy:
+    title: Unlimited Call Rate Policy
+    description: A policy that allows unlimited calls for specific requests.
+    type: object
+    required:
+      - type
+      - matchers
+    properties:
+      type:
+        type: string
+        enum: [UnlimitedCallRatePolicy]
+      matchers:
+        title: Matchers
+        description: List of matchers that define which requests this policy applies to.
+        type: array
+        items:
+          "$ref": "#/definitions/HttpRequestMatcher"
+    additionalProperties: true
+  Rate:
+    title: Rate
+    description: Defines a rate limit with a specific number of calls allowed within a time interval.
+    type: object
+    required:
+      - limit
+      - interval
+    properties:
+      limit:
+        title: Limit
+        description: The maximum number of calls allowed within the interval.
+        type: integer
+      interval:
+        title: Interval
+        description: The time interval for the rate limit.
+        type: string
+        format: duration
+    additionalProperties: true
+  HttpRequestMatcher:
+    title: HTTP Request Matcher
+    description: >
+      Matches HTTP requests based on method, base URL, URL path pattern, query parameters, and headers.
+      Use `url_base` to specify the scheme and host (without trailing slash) and
+      `url_path_pattern` to apply a regex to the request path.
+    type: object
+    properties:
+      method:
+        title: Method
+        description: The HTTP method to match (e.g., GET, POST).
+        type: string
+      url_base:
+        title: URL Base
+        description: The base URL (scheme and host, e.g. "https://api.example.com") to match.
+        type: string
+      url_path_pattern:
+        title: URL Path Pattern
+        description: A regular expression pattern to match the URL path.
+        type: string
+      params:
+        title: Parameters
+        description: The query parameters to match.
+        type: object
+        additionalProperties: true
+      headers:
+        title: Headers
+        description: The headers to match.
+        type: object
+        additionalProperties: true
+    additionalProperties: true
   DefaultErrorHandler:
     title: Default Error Handler
     description: Component defining how to handle errors. Default behavior includes only retrying server errors (HTTP 5XX) and too many requests (HTTP 429) with an exponential backoff.

airbyte_cdk/sources/declarative/manifest_declarative_source.py

@@ -137,6 +137,10 @@ class ManifestDeclarativeSource(DeclarativeSource):
             self._source_config, config
         )
 
+        api_budget_model = self._source_config.get("api_budget")
+        if api_budget_model:
+            self._constructor.set_api_budget(api_budget_model, config)
+
         source_streams = [
             self._constructor.create_component(
                 DeclarativeStreamModel,

airbyte_cdk/sources/declarative/models/declarative_component_schema.py

@@ -3,6 +3,7 @@
 
 from __future__ import annotations
 
+from datetime import datetime, timedelta
 from enum import Enum
 from typing import Any, Dict, List, Literal, Optional, Union
 
@@ -642,6 +643,45 @@ class OAuthAuthenticator(BaseModel):
     parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
 
 
+class Rate(BaseModel):
+    class Config:
+        extra = Extra.allow
+
+    limit: int = Field(
+        ...,
+        description="The maximum number of calls allowed within the interval.",
+        title="Limit",
+    )
+    interval: timedelta = Field(
+        ..., description="The time interval for the rate limit.", title="Interval"
+    )
+
+
+class HttpRequestMatcher(BaseModel):
+    class Config:
+        extra = Extra.allow
+
+    method: Optional[str] = Field(
+        None, description="The HTTP method to match (e.g., GET, POST).", title="Method"
+    )
+    url_base: Optional[str] = Field(
+        None,
+        description='The base URL (scheme and host, e.g. "https://api.example.com") to match.',
+        title="URL Base",
+    )
+    url_path_pattern: Optional[str] = Field(
+        None,
+        description="A regular expression pattern to match the URL path.",
+        title="URL Path Pattern",
+    )
+    params: Optional[Dict[str, Any]] = Field(
+        None, description="The query parameters to match.", title="Parameters"
+    )
+    headers: Optional[Dict[str, Any]] = Field(
+        None, description="The headers to match.", title="Headers"
+    )
+
+
 class DpathExtractor(BaseModel):
     type: Literal["DpathExtractor"]
     field_path: List[str] = Field(
@@ -1578,6 +1618,60 @@ class DatetimeBasedCursor(BaseModel):
     parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
 
 
+class FixedWindowCallRatePolicy(BaseModel):
+    class Config:
+        extra = Extra.allow
+
+    type: Literal["FixedWindowCallRatePolicy"]
+    next_reset_ts: datetime = Field(
+        ...,
+        description="The timestamp when the rate limit will reset.",
+        title="Next Reset Timestamp",
+    )
+    period: timedelta = Field(
+        ..., description="The time interval for the rate limit window.", title="Period"
+    )
+    call_limit: int = Field(
+        ...,
+        description="The maximum number of calls allowed within the period.",
+        title="Call Limit",
+    )
+    matchers: List[HttpRequestMatcher] = Field(
+        ...,
+        description="List of matchers that define which requests this policy applies to.",
+        title="Matchers",
+    )
+
+
+class MovingWindowCallRatePolicy(BaseModel):
+    class Config:
+        extra = Extra.allow
+
+    type: Literal["MovingWindowCallRatePolicy"]
+    rates: List[Rate] = Field(
+        ...,
+        description="List of rates that define the call limits for different time intervals.",
+        title="Rates",
+    )
+    matchers: List[HttpRequestMatcher] = Field(
+        ...,
+        description="List of matchers that define which requests this policy applies to.",
+        title="Matchers",
+    )
+
+
+class UnlimitedCallRatePolicy(BaseModel):
+    class Config:
+        extra = Extra.allow
+
+    type: Literal["UnlimitedCallRatePolicy"]
+    matchers: List[HttpRequestMatcher] = Field(
+        ...,
+        description="List of matchers that define which requests this policy applies to.",
+        title="Matchers",
+    )
+
+
 class DefaultErrorHandler(BaseModel):
     type: Literal["DefaultErrorHandler"]
     backoff_strategies: Optional[
@@ -1709,6 +1803,67 @@ class CompositeErrorHandler(BaseModel):
     parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
 
 
+class APIBudget(BaseModel):
+    class Config:
+        extra = Extra.allow
+
+    type: Literal["APIBudget"]
+    policies: List[
+        Union[
+            FixedWindowCallRatePolicy,
+            MovingWindowCallRatePolicy,
+            UnlimitedCallRatePolicy,
+        ]
+    ] = Field(
+        ...,
+        description="List of call rate policies that define how many calls are allowed.",
+        title="Policies",
+    )
+    maximum_attempts_to_acquire: Optional[int] = Field(
+        100000,
+        description="The maximum number of attempts to acquire a call before giving up.",
+        title="Maximum Attempts to Acquire",
+    )
+
+
+class HTTPAPIBudget(BaseModel):
+    class Config:
+        extra = Extra.allow
+
+    type: Literal["HTTPAPIBudget"]
+    policies: List[
+        Union[
+            FixedWindowCallRatePolicy,
+            MovingWindowCallRatePolicy,
+            UnlimitedCallRatePolicy,
+        ]
+    ] = Field(
+        ...,
+        description="List of call rate policies that define how many calls are allowed.",
+        title="Policies",
+    )
+    ratelimit_reset_header: Optional[str] = Field(
+        "ratelimit-reset",
+        description="The HTTP response header name that indicates when the rate limit resets.",
+        title="Rate Limit Reset Header",
+    )
+    ratelimit_remaining_header: Optional[str] = Field(
+        "ratelimit-remaining",
+        description="The HTTP response header name that indicates the number of remaining allowed calls.",
+        title="Rate Limit Remaining Header",
+    )
+    status_codes_for_ratelimit_hit: Optional[List[int]] = Field(
+        [429],
+        description="List of HTTP status codes that indicate a rate limit has been hit.",
+        title="Status Codes for Rate Limit Hit",
+    )
+    maximum_attempts_to_acquire: Optional[int] = Field(
+        100000,
+        description="The maximum number of attempts to acquire a call before giving up.",
+        title="Maximum Attempts to Acquire",
+    )
+
+
 class ZipfileDecoder(BaseModel):
     class Config:
         extra = Extra.allow
@@ -1742,6 +1897,11 @@ class DeclarativeSource1(BaseModel):
     definitions: Optional[Dict[str, Any]] = None
     spec: Optional[Spec] = None
     concurrency_level: Optional[ConcurrencyLevel] = None
+    api_budget: Optional[Union[APIBudget, HTTPAPIBudget]] = Field(
+        None,
+        description="Defines how many requests can be made to the API in a given time frame. This field accepts either a generic APIBudget or an HTTP-specific configuration (HTTPAPIBudget) to be applied across all streams.",
+        title="API Budget",
+    )
     metadata: Optional[Dict[str, Any]] = Field(
         None,
         description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.",
@@ -1768,6 +1928,11 @@ class DeclarativeSource2(BaseModel):
     definitions: Optional[Dict[str, Any]] = None
     spec: Optional[Spec] = None
     concurrency_level: Optional[ConcurrencyLevel] = None
+    api_budget: Optional[Union[APIBudget, HTTPAPIBudget]] = Field(
+        None,
+        description="Defines how many requests can be made to the API in a given time frame. This field accepts either a generic APIBudget or an HTTP-specific configuration (HTTPAPIBudget) to be applied across all streams.",
+        title="API Budget",
+    )
     metadata: Optional[Dict[str, Any]] = Field(
         None,
         description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.",

airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py

@@ -112,6 +112,9 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     AddFields as AddFieldsModel,
 )
+from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
+    APIBudget as APIBudgetModel,
+)
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     ApiKeyAuthenticator as ApiKeyAuthenticatorModel,
 )
@@ -226,6 +229,9 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     ExponentialBackoffStrategy as ExponentialBackoffStrategyModel,
 )
+from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
+    FixedWindowCallRatePolicy as FixedWindowCallRatePolicyModel,
+)
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     FlattenFields as FlattenFieldsModel,
 )
@@ -235,12 +241,18 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     GzipParser as GzipParserModel,
 )
+from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
+    HTTPAPIBudget as HTTPAPIBudgetModel,
+)
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     HttpComponentsResolver as HttpComponentsResolverModel,
 )
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     HttpRequester as HttpRequesterModel,
 )
+from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
+    HttpRequestMatcher as HttpRequestMatcherModel,
+)
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     HttpResponseFilter as HttpResponseFilterModel,
 )
@@ -295,6 +307,9 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     MinMaxDatetime as MinMaxDatetimeModel,
 )
+from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
+    MovingWindowCallRatePolicy as MovingWindowCallRatePolicyModel,
+)
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     NoAuth as NoAuthModel,
 )
@@ -313,6 +328,9 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     ParentStreamConfig as ParentStreamConfigModel,
 )
+from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
+    Rate as RateModel,
+)
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     RecordFilter as RecordFilterModel,
 )
@@ -356,6 +374,9 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     TypesMap as TypesMapModel,
 )
+from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
+    UnlimitedCallRatePolicy as UnlimitedCallRatePolicyModel,
+)
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import ValueType
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     WaitTimeFromHeader as WaitTimeFromHeaderModel,
@@ -469,6 +490,15 @@ from airbyte_cdk.sources.message import (
     MessageRepository,
     NoopMessageRepository,
 )
+from airbyte_cdk.sources.streams.call_rate import (
+    APIBudget,
+    FixedWindowCallRatePolicy,
+    HttpAPIBudget,
+    HttpRequestMatcher,
+    MovingWindowCallRatePolicy,
+    Rate,
+    UnlimitedCallRatePolicy,
+)
 from airbyte_cdk.sources.streams.concurrent.clamping import (
     ClampingEndProvider,
     ClampingStrategy,
@@ -520,6 +550,7 @@ class ModelToComponentFactory:
             self._evaluate_log_level(emit_connector_builder_messages)
         )
         self._connector_state_manager = connector_state_manager or ConnectorStateManager()
+        self._api_budget: Optional[Union[APIBudget, HttpAPIBudget]] = None
 
     def _init_mappings(self) -> None:
         self.PYDANTIC_MODEL_TO_CONSTRUCTOR: Mapping[Type[BaseModel], Callable[..., Any]] = {
@@ -607,6 +638,13 @@ class ModelToComponentFactory:
             StreamConfigModel: self.create_stream_config,
             ComponentMappingDefinitionModel: self.create_components_mapping_definition,
             ZipfileDecoderModel: self.create_zipfile_decoder,
+            APIBudgetModel: self.create_api_budget,
+            HTTPAPIBudgetModel: self.create_http_api_budget,
+            FixedWindowCallRatePolicyModel: self.create_fixed_window_call_rate_policy,
+            MovingWindowCallRatePolicyModel: self.create_moving_window_call_rate_policy,
+            UnlimitedCallRatePolicyModel: self.create_unlimited_call_rate_policy,
+            RateModel: self.create_rate,
+            HttpRequestMatcherModel: self.create_http_request_matcher,
         }
 
         # Needed for the case where we need to perform a second parse on the fields of a custom component
@@ -1911,6 +1949,8 @@ class ModelToComponentFactory:
             )
         )
 
+        api_budget = self._api_budget
+
         request_options_provider = InterpolatedRequestOptionsProvider(
             request_body_data=model.request_body_data,
             request_body_json=model.request_body_json,
@@ -1931,6 +1971,7 @@ class ModelToComponentFactory:
             path=model.path,
             authenticator=authenticator,
             error_handler=error_handler,
+            api_budget=api_budget,
             http_method=HttpMethod[model.http_method.value],
             request_options_provider=request_options_provider,
             config=config,
@@ -2919,3 +2960,103 @@ class ModelToComponentFactory:
             return isinstance(parser.inner_parser, JsonParser)
         else:
             return False
+
+    def create_api_budget(self, model: APIBudgetModel, config: Config, **kwargs: Any) -> APIBudget:
+        policies = [
+            self._create_component_from_model(model=policy, config=config)
+            for policy in model.policies
+        ]
+
+        return APIBudget(
+            policies=policies,
+            maximum_attempts_to_acquire=model.maximum_attempts_to_acquire or 100000,
+        )
+
+    def create_http_api_budget(
+        self, model: HTTPAPIBudgetModel, config: Config, **kwargs: Any
+    ) -> HttpAPIBudget:
+        policies = [
+            self._create_component_from_model(model=policy, config=config)
+            for policy in model.policies
+        ]
+
+        return HttpAPIBudget(
+            policies=policies,
+            maximum_attempts_to_acquire=model.maximum_attempts_to_acquire or 100000,
+            ratelimit_reset_header=model.ratelimit_reset_header or "ratelimit-reset",
+            ratelimit_remaining_header=model.ratelimit_remaining_header or "ratelimit-remaining",
+            status_codes_for_ratelimit_hit=model.status_codes_for_ratelimit_hit or (429,),
+        )
+
+    def create_fixed_window_call_rate_policy(
+        self, model: FixedWindowCallRatePolicyModel, config: Config, **kwargs: Any
+    ) -> FixedWindowCallRatePolicy:
+        matchers = [
+            self._create_component_from_model(model=matcher, config=config)
+            for matcher in model.matchers
+        ]
+        return FixedWindowCallRatePolicy(
+            next_reset_ts=model.next_reset_ts,
+            period=parse_duration(model.period),
+            call_limit=model.call_limit,
+            matchers=matchers,
+        )
+
+    def create_moving_window_call_rate_policy(
+        self, model: MovingWindowCallRatePolicyModel, config: Config, **kwargs: Any
+    ) -> MovingWindowCallRatePolicy:
+        rates = [
+            self._create_component_from_model(model=rate, config=config) for rate in model.rates
+        ]
+        matchers = [
+            self._create_component_from_model(model=matcher, config=config)
+            for matcher in model.matchers
+        ]
+        return MovingWindowCallRatePolicy(
+            rates=rates,
+            matchers=matchers,
+        )
+
+    def create_unlimited_call_rate_policy(
+        self, model: UnlimitedCallRatePolicyModel, config: Config, **kwargs: Any
+    ) -> UnlimitedCallRatePolicy:
+        matchers = [
+            self._create_component_from_model(model=matcher, config=config)
+            for matcher in model.matchers
+        ]
+
+        return UnlimitedCallRatePolicy(
+            matchers=matchers,
+        )
+
+    def create_rate(self, model: RateModel, config: Config, **kwargs: Any) -> Rate:
+        return Rate(
+            limit=model.limit,
+            interval=model.interval,
+        )
+
+    def create_http_request_matcher(
+        self, model: HttpRequestMatcherModel, config: Config, **kwargs: Any
+    ) -> HttpRequestMatcher:
+        return HttpRequestMatcher(
+            method=model.method,
+            url_base=model.url_base,
+            url_path_pattern=model.url_path_pattern,
+            params=model.params,
+            headers=model.headers,
+        )
+
+    def set_api_budget(self, component_definition: ComponentDefinition, config: Config) -> None:
+        model_str = component_definition.get("type")
+        if model_str == "APIBudget":
+            # Annotate model_type as a type that is a subclass of BaseModel
+            model_type: Union[Type[APIBudgetModel], Type[HTTPAPIBudgetModel]] = APIBudgetModel
+        elif model_str == "HTTPAPIBudget":
+            model_type = HTTPAPIBudgetModel
+        else:
+            raise ValueError(f"Unknown API Budget type: {model_str}")
+
+        # create_component expects a type[BaseModel] and returns an instance of that model.
+        self._api_budget = self.create_component(
+            model_type=model_type, component_definition=component_definition, config=config
+        )

airbyte_cdk/sources/declarative/requesters/http_requester.py

@@ -22,6 +22,7 @@ from airbyte_cdk.sources.declarative.requesters.request_options.interpolated_req
 )
 from airbyte_cdk.sources.declarative.requesters.requester import HttpMethod, Requester
 from airbyte_cdk.sources.message import MessageRepository, NoopMessageRepository
+from airbyte_cdk.sources.streams.call_rate import APIBudget
 from airbyte_cdk.sources.streams.http import HttpClient
 from airbyte_cdk.sources.streams.http.error_handlers import ErrorHandler
 from airbyte_cdk.sources.types import Config, StreamSlice, StreamState
@@ -55,6 +56,7 @@ class HttpRequester(Requester):
     http_method: Union[str, HttpMethod] = HttpMethod.GET
     request_options_provider: Optional[InterpolatedRequestOptionsProvider] = None
     error_handler: Optional[ErrorHandler] = None
+    api_budget: Optional[APIBudget] = None
     disable_retries: bool = False
     message_repository: MessageRepository = NoopMessageRepository()
     use_cache: bool = False
@@ -91,6 +93,7 @@ class HttpRequester(Requester):
             name=self.name,
             logger=self.logger,
             error_handler=self.error_handler,
+            api_budget=self.api_budget,
             authenticator=self._authenticator,
             use_cache=self.use_cache,
             backoff_strategy=backoff_strategies,

airbyte_cdk/sources/streams/call_rate.py

@@ -6,10 +6,12 @@ import abc
 import dataclasses
 import datetime
 import logging
+import re
 import time
+from dataclasses import InitVar, dataclass, field
 from datetime import timedelta
 from threading import RLock
-from typing import TYPE_CHECKING, Any, Mapping, Optional
+from typing import TYPE_CHECKING, Any, Mapping, Optional, Union
 from urllib import parse
 
 import requests
@@ -98,43 +100,55 @@ class RequestMatcher(abc.ABC):
 
 
 class HttpRequestMatcher(RequestMatcher):
-    """Simple implementation of RequestMatcher for http requests case"""
+    """
+    Extended RequestMatcher for HTTP requests that supports matching on:
+      - HTTP method (case-insensitive)
+      - URL base (scheme + netloc) optionally
+      - URL path pattern (a regex applied to the path portion of the URL)
+      - Query parameters (must be present)
+      - Headers (header names compared case-insensitively)
+    """
 
     def __init__(
         self,
         method: Optional[str] = None,
-        url: Optional[str] = None,
+        url_base: Optional[str] = None,
+        url_path_pattern: Optional[str] = None,
         params: Optional[Mapping[str, Any]] = None,
         headers: Optional[Mapping[str, Any]] = None,
     ):
-        """Constructor
-
-        :param method:
-        :param url:
-        :param params:
-        :param headers:
         """
-        self._method = method
-        self._url = url
+        :param method: HTTP method (e.g. "GET", "POST"); compared case-insensitively.
+        :param url_base: Base URL (scheme://host) that must match.
+        :param url_path_pattern: A regex pattern that will be applied to the path portion of the URL.
+        :param params: Dictionary of query parameters that must be present in the request.
+        :param headers: Dictionary of headers that must be present (header keys are compared case-insensitively).
+        """
+        self._method = method.upper() if method else None
+
+        # Normalize the url_base if provided: remove trailing slash.
+        self._url_base = url_base.rstrip("/") if url_base else None
+
+        # Compile the URL path pattern if provided.
+        self._url_path_pattern = re.compile(url_path_pattern) if url_path_pattern else None
+
+        # Normalize query parameters to strings.
         self._params = {str(k): str(v) for k, v in (params or {}).items()}
-        self._headers = {str(k): str(v) for k, v in (headers or {}).items()}
+
+        # Normalize header keys to lowercase.
+        self._headers = {str(k).lower(): str(v) for k, v in (headers or {}).items()}
 
     @staticmethod
     def _match_dict(obj: Mapping[str, Any], pattern: Mapping[str, Any]) -> bool:
-        """Check that all elements from pattern dict present and have the same values in obj dict
-
-        :param obj:
-        :param pattern:
-        :return:
-        """
+        """Check that every key/value in the pattern exists in the object."""
         return pattern.items() <= obj.items()
 
     def __call__(self, request: Any) -> bool:
         """
-
-        :param request:
-        :return: True if matches the provided request object, False - otherwise
+        :param request: A requests.Request or requests.PreparedRequest instance.
+        :return: True if the request matches all provided criteria; False otherwise.
         """
+        # Prepare the request (if needed) and extract the URL details.
         if isinstance(request, requests.Request):
             prepared_request = request.prepare()
         elif isinstance(request, requests.PreparedRequest):
@@ -142,21 +156,40 @@ class HttpRequestMatcher(RequestMatcher):
         else:
             return False
 
-        if self._method is not None:
-            if prepared_request.method != self._method:
+        # Check HTTP method.
+        if self._method is not None and prepared_request.method is not None:
+            if prepared_request.method.upper() != self._method:
                 return False
-        if self._url is not None and prepared_request.url is not None:
-            url_without_params = prepared_request.url.split("?")[0]
-            if url_without_params != self._url:
+
+        # Parse the URL.
+        parsed_url = parse.urlsplit(prepared_request.url)
+        # Reconstruct the base: scheme://netloc
+        request_url_base = f"{str(parsed_url.scheme)}://{str(parsed_url.netloc)}"
+        # The path (without query parameters)
+        request_path = str(parsed_url.path).rstrip("/")
+
+        # If a base URL is provided, check that it matches.
+        if self._url_base is not None:
+            if request_url_base != self._url_base:
                 return False
-        if self._params is not None:
-            parsed_url = parse.urlsplit(prepared_request.url)
-            params = dict(parse.parse_qsl(str(parsed_url.query)))
-            if not self._match_dict(params, self._params):
+
+        # If a URL path pattern is provided, ensure the path matches the regex.
+        if self._url_path_pattern is not None:
+            if not self._url_path_pattern.search(request_path):
                 return False
-        if self._headers is not None:
-            if not self._match_dict(prepared_request.headers, self._headers):
+
+        # Check query parameters.
+        if self._params:
+            query_params = dict(parse.parse_qsl(str(parsed_url.query)))
+            if not self._match_dict(query_params, self._params):
                 return False
+
+        # Check headers (normalize keys to lower-case).
+        if self._headers:
+            req_headers = {k.lower(): v for k, v in prepared_request.headers.items()}
+            if not self._match_dict(req_headers, self._headers):
+                return False
+
         return True
 
 
@@ -399,24 +432,17 @@ class AbstractAPIBudget(abc.ABC):
         """
 
 
+@dataclass
 class APIBudget(AbstractAPIBudget):
-    """Default APIBudget implementation"""
-
-    def __init__(
-        self, policies: list[AbstractCallRatePolicy], maximum_attempts_to_acquire: int = 100000
-    ) -> None:
-        """Constructor
-
-        :param policies: list of policies in this budget
-        :param maximum_attempts_to_acquire: number of attempts before throwing hit ratelimit exception, we put some big number here
-         to avoid situations when many threads compete with each other for a few lots over a significant amount of time
-        """
+    """
+    Default APIBudget implementation.
+    """
 
-        self._policies = policies
-        self._maximum_attempts_to_acquire = maximum_attempts_to_acquire
+    policies: list[AbstractCallRatePolicy]
+    maximum_attempts_to_acquire: int = 100000
 
     def get_matching_policy(self, request: Any) -> Optional[AbstractCallRatePolicy]:
-        for policy in self._policies:
+        for policy in self.policies:
             if policy.matches(request):
                 return policy
         return None
@@ -437,7 +463,7 @@ class APIBudget(AbstractAPIBudget):
         policy = self.get_matching_policy(request)
         if policy:
             self._do_acquire(request=request, policy=policy, block=block, timeout=timeout)
-        elif self._policies:
+        elif self.policies:
             logger.info("no policies matched with requests, allow call by default")
 
     def update_from_response(self, request: Any, response: Any) -> None:
@@ -460,7 +486,7 @@ class APIBudget(AbstractAPIBudget):
         """
         last_exception = None
         # sometimes we spend all budget before a second attempt, so we have few more here
-        for attempt in range(1, self._maximum_attempts_to_acquire):
+        for attempt in range(1, self.maximum_attempts_to_acquire):
             try:
                 policy.try_acquire(request, weight=1)
                 return
@@ -484,31 +510,18 @@ class APIBudget(AbstractAPIBudget):
 
         if last_exception:
             logger.info(
-                "we used all %s attempts to acquire and failed", self._maximum_attempts_to_acquire
+                "we used all %s attempts to acquire and failed", self.maximum_attempts_to_acquire
             )
             raise last_exception
 
 
+@dataclass
 class HttpAPIBudget(APIBudget):
     """Implementation of AbstractAPIBudget for HTTP"""
 
-    def __init__(
-        self,
-        ratelimit_reset_header: str = "ratelimit-reset",
-        ratelimit_remaining_header: str = "ratelimit-remaining",
-        status_codes_for_ratelimit_hit: tuple[int] = (429,),
-        **kwargs: Any,
-    ):
-        """Constructor
-
-        :param ratelimit_reset_header: name of the header that has a timestamp of the next reset of call budget
-        :param ratelimit_remaining_header: name of the header that has the number of calls left
-        :param status_codes_for_ratelimit_hit: list of HTTP status codes that signal about rate limit being hit
-        """
-        self._ratelimit_reset_header = ratelimit_reset_header
-        self._ratelimit_remaining_header = ratelimit_remaining_header
-        self._status_codes_for_ratelimit_hit = status_codes_for_ratelimit_hit
-        super().__init__(**kwargs)
+    ratelimit_reset_header: str = "ratelimit-reset"
+    ratelimit_remaining_header: str = "ratelimit-remaining"
+    status_codes_for_ratelimit_hit: Union[tuple[int], list[int]] = (429,)
 
     def update_from_response(self, request: Any, response: Any) -> None:
         policy = self.get_matching_policy(request)
@@ -523,17 +536,17 @@ class HttpAPIBudget(APIBudget):
     def get_reset_ts_from_response(
         self, response: requests.Response
     ) -> Optional[datetime.datetime]:
-        if response.headers.get(self._ratelimit_reset_header):
+        if response.headers.get(self.ratelimit_reset_header):
             return datetime.datetime.fromtimestamp(
-                int(response.headers[self._ratelimit_reset_header])
+                int(response.headers[self.ratelimit_reset_header])
             )
         return None
 
     def get_calls_left_from_response(self, response: requests.Response) -> Optional[int]:
-        if response.headers.get(self._ratelimit_remaining_header):
-            return int(response.headers[self._ratelimit_remaining_header])
+        if response.headers.get(self.ratelimit_remaining_header):
+            return int(response.headers[self.ratelimit_remaining_header])
 
-        if response.status_code in self._status_codes_for_ratelimit_hit:
+        if response.status_code in self.status_codes_for_ratelimit_hit:
             return 0
 
         return None

{airbyte_cdk-6.32.0.dist-info → airbyte_cdk-6.33.0.dev0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 6.32.0
+Version: 6.33.0.dev0
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://airbyte.com
 License: MIT

{airbyte_cdk-6.32.0.dist-info → airbyte_cdk-6.33.0.dev0.dist-info}/RECORD

@@ -67,7 +67,7 @@ airbyte_cdk/sources/declarative/concurrent_declarative_source.py,sha256=x5a_Wv0c
 airbyte_cdk/sources/declarative/datetime/__init__.py,sha256=l9LG7Qm6e5r_qgqfVKnx3mXYtg1I9MmMjomVIPfU4XA,177
 airbyte_cdk/sources/declarative/datetime/datetime_parser.py,sha256=SX9JjdesN1edN2WVUVMzU_ptqp2QB1OnsnjZ4mwcX7w,2579
 airbyte_cdk/sources/declarative/datetime/min_max_datetime.py,sha256=0BHBtDNQZfvwM45-tY5pNlTcKAFSGGNxemoi0Jic-0E,5785
-airbyte_cdk/sources/declarative/declarative_component_schema.yaml,sha256=43cBrshQer-KiNwjlA6peDyfrST88VQIZmfgdAZTiLc,139874
+airbyte_cdk/sources/declarative/declarative_component_schema.yaml,sha256=MOzhkZUsmcFiSP1ES_eEQyAta_ms5zG5IEA9XwooR8M,147035
 airbyte_cdk/sources/declarative/declarative_source.py,sha256=nF7wBqFd3AQmEKAm4CnIo29CJoQL562cJGSCeL8U8bA,1531
 airbyte_cdk/sources/declarative/declarative_stream.py,sha256=venZjfpvtqr3oFSuvMBWtn4h9ayLhD4L65ACuXCDZ64,10445
 airbyte_cdk/sources/declarative/decoders/__init__.py,sha256=KSpQetKGqPCv-38QgcVJ5kzM5nzbFldTSsYDCS3Xf0Y,1035
@@ -104,18 +104,18 @@ airbyte_cdk/sources/declarative/interpolation/interpolated_string.py,sha256=LYEZ
 airbyte_cdk/sources/declarative/interpolation/interpolation.py,sha256=-V5UddGm69UKEB6o_O1EIES9kfY8FV_X4Ji8w1yOuSA,981
 airbyte_cdk/sources/declarative/interpolation/jinja.py,sha256=BtsY_jtT4MihFqeQgc05HXj3Ndt-e2ESQgGwbg3Sdxc,6430
 airbyte_cdk/sources/declarative/interpolation/macros.py,sha256=Y5AWYxbJTUtJ_Jm7DV9qrZDiymFR9LST7fBt4piT2-U,4585
-airbyte_cdk/sources/declarative/manifest_declarative_source.py,sha256=26qMXRugdPAd3zyYRH6YpNi--TorGZVOtxzY5O6muL0,16912
+airbyte_cdk/sources/declarative/manifest_declarative_source.py,sha256=TN6GCgLXaWDONTaJwQ3A5ELqC-sxwKz-UYSraJYB-dI,17078
 airbyte_cdk/sources/declarative/migrations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 airbyte_cdk/sources/declarative/migrations/legacy_to_per_partition_state_migration.py,sha256=iemy3fKLczcU0-Aor7tx5jcT6DRedKMqyK7kCOp01hg,3924
 airbyte_cdk/sources/declarative/migrations/state_migration.py,sha256=KWPjealMLKSMtajXgkdGgKg7EmTLR-CqqD7UIh0-eDU,794
 airbyte_cdk/sources/declarative/models/__init__.py,sha256=nUFxNCiKeYRVXuZEKA7GD-lTHxsiKcQ8FitZjKhPIvE,100
-airbyte_cdk/sources/declarative/models/declarative_component_schema.py,sha256=UJ9sYWqoohaT0mr0ALxOP74t0TTJRhfZMHBoqViaoVU,98273
+airbyte_cdk/sources/declarative/models/declarative_component_schema.py,sha256=gRRQoZ4VAuyxtBALvHQnGMF9rOLgPSLSBp0yzWW-w4Y,103695
 airbyte_cdk/sources/declarative/parsers/__init__.py,sha256=ZnqYNxHsKCgO38IwB34RQyRMXTs4GTvlRi3ImKnIioo,61
 airbyte_cdk/sources/declarative/parsers/custom_code_compiler.py,sha256=958MMX6_ZOJUlDDdNr9Krosgi2bCKGx2Z765M2Woz18,5505
 airbyte_cdk/sources/declarative/parsers/custom_exceptions.py,sha256=Rir9_z3Kcd5Es0-LChrzk-0qubAsiK_RSEnLmK2OXm8,553
 airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py,sha256=CXwTfD3wSQq3okcqwigpprbHhSURUokh4GK2OmOyKC8,9132
 airbyte_cdk/sources/declarative/parsers/manifest_reference_resolver.py,sha256=IWUOdF03o-aQn0Occo1BJCxU0Pz-QILk5L67nzw2thw,6803
-airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py,sha256=f8gzijhJtUEJqNZvZihaFyBqrn8xqxjYrncgiXeKZY8,128557
+airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py,sha256=ee3BYsYxpGH3D6fLb8BWpGG1HFHcAgFRsCJXUO98nbA,134255
 airbyte_cdk/sources/declarative/partition_routers/__init__.py,sha256=HJ-Syp3p7RpyR_OK0X_a2kSyISfu3W-PKrRI16iY0a8,957
 airbyte_cdk/sources/declarative/partition_routers/async_job_partition_router.py,sha256=VelO7zKqKtzMJ35jyFeg0ypJLQC0plqqIBNXoBW1G2E,3001
 airbyte_cdk/sources/declarative/partition_routers/cartesian_product_stream_slicer.py,sha256=c5cuVFM6NFkuQqG8Z5IwkBuwDrvXZN1CunUOM_L0ezg,6892
@@ -139,7 +139,7 @@ airbyte_cdk/sources/declarative/requesters/error_handlers/default_http_response_
 airbyte_cdk/sources/declarative/requesters/error_handlers/error_handler.py,sha256=Tan66odx8VHzfdyyXMQkXz2pJYksllGqvxmpoajgcK4,669
 airbyte_cdk/sources/declarative/requesters/error_handlers/http_response_filter.py,sha256=E-fQbt4ShfxZVoqfnmOx69C6FUPWZz8BIqI3DN9Kcjs,7935
 airbyte_cdk/sources/declarative/requesters/http_job_repository.py,sha256=3GtOefPH08evlSUxaILkiKLTHbIspFY4qd5B3ZqNE60,10063
-airbyte_cdk/sources/declarative/requesters/http_requester.py,sha256=RqYPkgJFAWfcZBTc-JBcGHPm4JL1ZQOhs9GKU4MP2eE,14723
+airbyte_cdk/sources/declarative/requesters/http_requester.py,sha256=F1lydK00pbImprvauxzsFiy_IpQI59iErqtuf2vES1Q,14866
 airbyte_cdk/sources/declarative/requesters/paginators/__init__.py,sha256=uArbKs9JKNCt7t9tZoeWwjDpyI1HoPp29FNW0JzvaEM,644
 airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py,sha256=FnSl3qPvv5wD6ieAI2Ic5c4dqBk-3fRe4tCaWzq3YwM,11840
 airbyte_cdk/sources/declarative/requesters/paginators/no_pagination.py,sha256=j6j9QRPaTbKQ2N661RFVKthhkWiodEp6ut0tKeEd0Ng,2019
@@ -249,7 +249,7 @@ airbyte_cdk/sources/message/repository.py,sha256=SG7avgti_-dj8FcRHTTrhgLLGJbElv1
 airbyte_cdk/sources/source.py,sha256=KIBBH5VLEb8BZ8B9aROlfaI6OLoJqKDPMJ10jkAR7nk,3611
 airbyte_cdk/sources/streams/__init__.py,sha256=8fzTKpRTnSx5PggXgQPKJzHNZUV2BCA40N-dI6JM1xI,256
 airbyte_cdk/sources/streams/availability_strategy.py,sha256=_RU4JITrxMEN36g1RDHMu0iSw0I_3yWGfo5N8_YRvOg,3247
-airbyte_cdk/sources/streams/call_rate.py,sha256=Um_Ny8R7WZ2B0PWoxr-wrWPsgc5we7HrHalaMcozuVs,21052
+airbyte_cdk/sources/streams/call_rate.py,sha256=sf_AvWe6R_CqGkn1dnwQiDROBEvRESrPUv-zQeQ91Es,21779
 airbyte_cdk/sources/streams/checkpoint/__init__.py,sha256=3oy7Hd4ivVWTZlN6dKAf4Fv_G7U5iZrvhO9hT871UIo,712
 airbyte_cdk/sources/streams/checkpoint/checkpoint_reader.py,sha256=6HMT2NI-FQuaW0nt95NcyWrt5rZN4gF-Arx0sxdgbv4,15221
 airbyte_cdk/sources/streams/checkpoint/cursor.py,sha256=3e-3c-54k8U7Awno7DMmAD9ndbnl9OM48EnbEgeDUO0,3499
@@ -351,9 +351,9 @@ airbyte_cdk/utils/slice_hasher.py,sha256=EDxgROHDbfG-QKQb59m7h_7crN1tRiawdf5uU7G
 airbyte_cdk/utils/spec_schema_transformations.py,sha256=-5HTuNsnDBAhj-oLeQXwpTGA0HdcjFOf2zTEMUTTg_Y,816
 airbyte_cdk/utils/stream_status_utils.py,sha256=ZmBoiy5HVbUEHAMrUONxZvxnvfV9CesmQJLDTAIWnWw,1171
 airbyte_cdk/utils/traced_exception.py,sha256=C8uIBuCL_E4WnBAOPSxBicD06JAldoN9fGsQDp463OY,6292
-airbyte_cdk-6.32.0.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
-airbyte_cdk-6.32.0.dist-info/LICENSE_SHORT,sha256=aqF6D1NcESmpn-cqsxBtszTEnHKnlsp8L4x9wAh3Nxg,55
-airbyte_cdk-6.32.0.dist-info/METADATA,sha256=y_ghboeo7rvFlmHSnjuS0EuII2y0kvZzRcW2tEN1oqE,6010
-airbyte_cdk-6.32.0.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-airbyte_cdk-6.32.0.dist-info/entry_points.txt,sha256=fj-e3PAQvsxsQzyyq8UkG1k8spunWnD4BAH2AwlR6NM,95
-airbyte_cdk-6.32.0.dist-info/RECORD,,
+airbyte_cdk-6.33.0.dev0.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
+airbyte_cdk-6.33.0.dev0.dist-info/LICENSE_SHORT,sha256=aqF6D1NcESmpn-cqsxBtszTEnHKnlsp8L4x9wAh3Nxg,55
+airbyte_cdk-6.33.0.dev0.dist-info/METADATA,sha256=U-Epy8I3SzZzuU46_xkRtIpomk90rBI5BSigUTU4Sa4,6015
+airbyte_cdk-6.33.0.dev0.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+airbyte_cdk-6.33.0.dev0.dist-info/entry_points.txt,sha256=fj-e3PAQvsxsQzyyq8UkG1k8spunWnD4BAH2AwlR6NM,95
+airbyte_cdk-6.33.0.dev0.dist-info/RECORD,,