airbyte-cdk 6.33.3__py3-none-any.whl → 6.33.5__py3-none-any.whl

airbyte_cdk/sources/declarative/declarative_component_schema.yaml

@@ -40,6 +40,8 @@ properties:
     "$ref": "#/definitions/Spec"
   concurrency_level:
     "$ref": "#/definitions/ConcurrencyLevel"
+  api_budget:
+    "$ref": "#/definitions/HTTPAPIBudget"
   metadata:
     type: object
     description: For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.
@@ -794,7 +796,7 @@ definitions:
         description: This option is used to adjust the upper and lower boundaries of each datetime window to beginning and end of the provided target period (day, week, month)
         type: object
         required:
-        - target
+          - target
         properties:
           target:
             title: Target
@@ -1365,6 +1367,170 @@ definitions:
       $parameters:
         type: object
         additional_properties: true
+  HTTPAPIBudget:
+    title: HTTP API Budget
+    description: >
+      Defines how many requests can be made to the API in a given time frame. `HTTPAPIBudget` extracts the remaining
+      call count and the reset time from HTTP response headers using the header names provided by
+      `ratelimit_remaining_header` and `ratelimit_reset_header`. Only requests using `HttpRequester`
+      are rate-limited; custom components that bypass `HttpRequester` are not covered by this budget.
+    type: object
+    required:
+      - type
+      - policies
+    properties:
+      type:
+        type: string
+        enum: [HTTPAPIBudget]
+      policies:
+        title: Policies
+        description: List of call rate policies that define how many calls are allowed.
+        type: array
+        items:
+          anyOf:
+            - "$ref": "#/definitions/FixedWindowCallRatePolicy"
+            - "$ref": "#/definitions/MovingWindowCallRatePolicy"
+            - "$ref": "#/definitions/UnlimitedCallRatePolicy"
+      ratelimit_reset_header:
+        title: Rate Limit Reset Header
+        description: The HTTP response header name that indicates when the rate limit resets.
+        type: string
+        default: "ratelimit-reset"
+      ratelimit_remaining_header:
+        title: Rate Limit Remaining Header
+        description: The HTTP response header name that indicates the number of remaining allowed calls.
+        type: string
+        default: "ratelimit-remaining"
+      status_codes_for_ratelimit_hit:
+        title: Status Codes for Rate Limit Hit
+        description: List of HTTP status codes that indicate a rate limit has been hit.
+        type: array
+        items:
+          type: integer
+        default: [429]
+    additionalProperties: true
+  FixedWindowCallRatePolicy:
+    title: Fixed Window Call Rate Policy
+    description: A policy that allows a fixed number of calls within a specific time window.
+    type: object
+    required:
+      - type
+      - period
+      - call_limit
+      - matchers
+    properties:
+      type:
+        type: string
+        enum: [FixedWindowCallRatePolicy]
+      period:
+        title: Period
+        description: The time interval for the rate limit window.
+        type: string
+      call_limit:
+        title: Call Limit
+        description: The maximum number of calls allowed within the period.
+        type: integer
+      matchers:
+        title: Matchers
+        description: List of matchers that define which requests this policy applies to.
+        type: array
+        items:
+          "$ref": "#/definitions/HttpRequestRegexMatcher"
+    additionalProperties: true
+  MovingWindowCallRatePolicy:
+    title: Moving Window Call Rate Policy
+    description: A policy that allows a fixed number of calls within a moving time window.
+    type: object
+    required:
+      - type
+      - rates
+      - matchers
+    properties:
+      type:
+        type: string
+        enum: [MovingWindowCallRatePolicy]
+      rates:
+        title: Rates
+        description: List of rates that define the call limits for different time intervals.
+        type: array
+        items:
+          "$ref": "#/definitions/Rate"
+      matchers:
+        title: Matchers
+        description: List of matchers that define which requests this policy applies to.
+        type: array
+        items:
+          "$ref": "#/definitions/HttpRequestRegexMatcher"
+    additionalProperties: true
+  UnlimitedCallRatePolicy:
+    title: Unlimited Call Rate Policy
+    description: A policy that allows unlimited calls for specific requests.
+    type: object
+    required:
+      - type
+      - matchers
+    properties:
+      type:
+        type: string
+        enum: [UnlimitedCallRatePolicy]
+      matchers:
+        title: Matchers
+        description: List of matchers that define which requests this policy applies to.
+        type: array
+        items:
+          "$ref": "#/definitions/HttpRequestRegexMatcher"
+    additionalProperties: true
+  Rate:
+    title: Rate
+    description: Defines a rate limit with a specific number of calls allowed within a time interval.
+    type: object
+    required:
+      - limit
+      - interval
+    properties:
+      limit:
+        title: Limit
+        description: The maximum number of calls allowed within the interval.
+        type: integer
+      interval:
+        title: Interval
+        description: The time interval for the rate limit.
+        type: string
+        examples:
+          - "PT1H"
+          - "P1D"
+    additionalProperties: true
+  HttpRequestRegexMatcher:
+    title: HTTP Request Matcher
+    description: >
+      Matches HTTP requests based on method, base URL, URL path pattern, query parameters, and headers.
+      Use `url_base` to specify the scheme and host (without trailing slash) and
+      `url_path_pattern` to apply a regex to the request path.
+    type: object
+    properties:
+      method:
+        title: Method
+        description: The HTTP method to match (e.g., GET, POST).
+        type: string
+      url_base:
+        title: URL Base
+        description: The base URL (scheme and host, e.g. "https://api.example.com") to match.
+        type: string
+      url_path_pattern:
+        title: URL Path Pattern
+        description: A regular expression pattern to match the URL path.
+        type: string
+      params:
+        title: Parameters
+        description: The query parameters to match.
+        type: object
+        additionalProperties: true
+      headers:
+        title: Headers
+        description: The headers to match.
+        type: object
+        additionalProperties: true
+    additionalProperties: true
   DefaultErrorHandler:
     title: Default Error Handler
     description: Component defining how to handle errors. Default behavior includes only retrying server errors (HTTP 5XX) and too many requests (HTTP 429) with an exponential backoff.

airbyte_cdk/sources/declarative/extractors/record_selector.py

@@ -41,6 +41,7 @@ class RecordSelector(HttpSelector):
     _name: Union[InterpolatedString, str] = field(init=False, repr=False, default="")
     record_filter: Optional[RecordFilter] = None
     transformations: List[RecordTransformation] = field(default_factory=lambda: [])
+    transform_before_filtering: bool = False
 
     def __post_init__(self, parameters: Mapping[str, Any]) -> None:
         self._parameters = parameters
@@ -104,9 +105,17 @@ class RecordSelector(HttpSelector):
         Until we decide to move this logic away from the selector, we made this method public so that users like AsyncJobRetriever could
         share the logic of doing transformations on a set of records.
         """
-        filtered_data = self._filter(all_data, stream_state, stream_slice, next_page_token)
-        transformed_data = self._transform(filtered_data, stream_state, stream_slice)
-        normalized_data = self._normalize_by_schema(transformed_data, schema=records_schema)
+        if self.transform_before_filtering:
+            transformed_data = self._transform(all_data, stream_state, stream_slice)
+            transformed_filtered_data = self._filter(
+                transformed_data, stream_state, stream_slice, next_page_token
+            )
+        else:
+            filtered_data = self._filter(all_data, stream_state, stream_slice, next_page_token)
+            transformed_filtered_data = self._transform(filtered_data, stream_state, stream_slice)
+        normalized_data = self._normalize_by_schema(
+            transformed_filtered_data, schema=records_schema
+        )
         for data in normalized_data:
             yield Record(data=data, stream_name=self.name, associated_slice=stream_slice)
 

airbyte_cdk/sources/declarative/manifest_declarative_source.py

@@ -137,6 +137,10 @@ class ManifestDeclarativeSource(DeclarativeSource):
             self._source_config, config
         )
 
+        api_budget_model = self._source_config.get("api_budget")
+        if api_budget_model:
+            self._constructor.set_api_budget(api_budget_model, config)
+
         source_streams = [
             self._constructor.create_component(
                 DeclarativeStreamModel,

airbyte_cdk/sources/declarative/models/declarative_component_schema.py

@@ -642,6 +642,48 @@ class OAuthAuthenticator(BaseModel):
     parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
 
 
+class Rate(BaseModel):
+    class Config:
+        extra = Extra.allow
+
+    limit: int = Field(
+        ...,
+        description="The maximum number of calls allowed within the interval.",
+        title="Limit",
+    )
+    interval: str = Field(
+        ...,
+        description="The time interval for the rate limit.",
+        examples=["PT1H", "P1D"],
+        title="Interval",
+    )
+
+
+class HttpRequestRegexMatcher(BaseModel):
+    class Config:
+        extra = Extra.allow
+
+    method: Optional[str] = Field(
+        None, description="The HTTP method to match (e.g., GET, POST).", title="Method"
+    )
+    url_base: Optional[str] = Field(
+        None,
+        description='The base URL (scheme and host, e.g. "https://api.example.com") to match.',
+        title="URL Base",
+    )
+    url_path_pattern: Optional[str] = Field(
+        None,
+        description="A regular expression pattern to match the URL path.",
+        title="URL Path Pattern",
+    )
+    params: Optional[Dict[str, Any]] = Field(
+        None, description="The query parameters to match.", title="Parameters"
+    )
+    headers: Optional[Dict[str, Any]] = Field(
+        None, description="The headers to match.", title="Headers"
+    )
+
+
 class DpathExtractor(BaseModel):
     type: Literal["DpathExtractor"]
     field_path: List[str] = Field(
@@ -1565,6 +1607,55 @@ class DatetimeBasedCursor(BaseModel):
     parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
 
 
+class FixedWindowCallRatePolicy(BaseModel):
+    class Config:
+        extra = Extra.allow
+
+    type: Literal["FixedWindowCallRatePolicy"]
+    period: str = Field(
+        ..., description="The time interval for the rate limit window.", title="Period"
+    )
+    call_limit: int = Field(
+        ...,
+        description="The maximum number of calls allowed within the period.",
+        title="Call Limit",
+    )
+    matchers: List[HttpRequestRegexMatcher] = Field(
+        ...,
+        description="List of matchers that define which requests this policy applies to.",
+        title="Matchers",
+    )
+
+
+class MovingWindowCallRatePolicy(BaseModel):
+    class Config:
+        extra = Extra.allow
+
+    type: Literal["MovingWindowCallRatePolicy"]
+    rates: List[Rate] = Field(
+        ...,
+        description="List of rates that define the call limits for different time intervals.",
+        title="Rates",
+    )
+    matchers: List[HttpRequestRegexMatcher] = Field(
+        ...,
+        description="List of matchers that define which requests this policy applies to.",
+        title="Matchers",
+    )
+
+
+class UnlimitedCallRatePolicy(BaseModel):
+    class Config:
+        extra = Extra.allow
+
+    type: Literal["UnlimitedCallRatePolicy"]
+    matchers: List[HttpRequestRegexMatcher] = Field(
+        ...,
+        description="List of matchers that define which requests this policy applies to.",
+        title="Matchers",
+    )
+
+
 class DefaultErrorHandler(BaseModel):
     type: Literal["DefaultErrorHandler"]
     backoff_strategies: Optional[
@@ -1696,6 +1787,39 @@ class CompositeErrorHandler(BaseModel):
     parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
 
 
+class HTTPAPIBudget(BaseModel):
+    class Config:
+        extra = Extra.allow
+
+    type: Literal["HTTPAPIBudget"]
+    policies: List[
+        Union[
+            FixedWindowCallRatePolicy,
+            MovingWindowCallRatePolicy,
+            UnlimitedCallRatePolicy,
+        ]
+    ] = Field(
+        ...,
+        description="List of call rate policies that define how many calls are allowed.",
+        title="Policies",
+    )
+    ratelimit_reset_header: Optional[str] = Field(
+        "ratelimit-reset",
+        description="The HTTP response header name that indicates when the rate limit resets.",
+        title="Rate Limit Reset Header",
+    )
+    ratelimit_remaining_header: Optional[str] = Field(
+        "ratelimit-remaining",
+        description="The HTTP response header name that indicates the number of remaining allowed calls.",
+        title="Rate Limit Remaining Header",
+    )
+    status_codes_for_ratelimit_hit: Optional[List[int]] = Field(
+        [429],
+        description="List of HTTP status codes that indicate a rate limit has been hit.",
+        title="Status Codes for Rate Limit Hit",
+    )
+
+
 class ZipfileDecoder(BaseModel):
     class Config:
         extra = Extra.allow
@@ -1724,6 +1848,7 @@ class DeclarativeSource1(BaseModel):
     definitions: Optional[Dict[str, Any]] = None
     spec: Optional[Spec] = None
     concurrency_level: Optional[ConcurrencyLevel] = None
+    api_budget: Optional[HTTPAPIBudget] = None
     metadata: Optional[Dict[str, Any]] = Field(
         None,
         description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.",
@@ -1750,6 +1875,7 @@ class DeclarativeSource2(BaseModel):
     definitions: Optional[Dict[str, Any]] = None
     spec: Optional[Spec] = None
     concurrency_level: Optional[ConcurrencyLevel] = None
+    api_budget: Optional[HTTPAPIBudget] = None
     metadata: Optional[Dict[str, Any]] = Field(
         None,
         description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.",

airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py

@@ -221,18 +221,27 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     ExponentialBackoffStrategy as ExponentialBackoffStrategyModel,
 )
+from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
+    FixedWindowCallRatePolicy as FixedWindowCallRatePolicyModel,
+)
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     FlattenFields as FlattenFieldsModel,
 )
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     GzipDecoder as GzipDecoderModel,
 )
+from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
+    HTTPAPIBudget as HTTPAPIBudgetModel,
+)
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     HttpComponentsResolver as HttpComponentsResolverModel,
 )
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     HttpRequester as HttpRequesterModel,
 )
+from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
+    HttpRequestRegexMatcher as HttpRequestRegexMatcherModel,
+)
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     HttpResponseFilter as HttpResponseFilterModel,
 )
@@ -281,6 +290,9 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     MinMaxDatetime as MinMaxDatetimeModel,
 )
+from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
+    MovingWindowCallRatePolicy as MovingWindowCallRatePolicyModel,
+)
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     NoAuth as NoAuthModel,
 )
@@ -299,6 +311,9 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     ParentStreamConfig as ParentStreamConfigModel,
 )
+from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
+    Rate as RateModel,
+)
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     RecordFilter as RecordFilterModel,
 )
@@ -342,6 +357,9 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     TypesMap as TypesMapModel,
 )
+from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
+    UnlimitedCallRatePolicy as UnlimitedCallRatePolicyModel,
+)
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import ValueType
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     WaitTimeFromHeader as WaitTimeFromHeaderModel,
@@ -455,6 +473,15 @@ from airbyte_cdk.sources.message import (
     MessageRepository,
     NoopMessageRepository,
 )
+from airbyte_cdk.sources.streams.call_rate import (
+    APIBudget,
+    FixedWindowCallRatePolicy,
+    HttpAPIBudget,
+    HttpRequestRegexMatcher,
+    MovingWindowCallRatePolicy,
+    Rate,
+    UnlimitedCallRatePolicy,
+)
 from airbyte_cdk.sources.streams.concurrent.clamping import (
     ClampingEndProvider,
     ClampingStrategy,
@@ -506,6 +533,7 @@ class ModelToComponentFactory:
             self._evaluate_log_level(emit_connector_builder_messages)
         )
         self._connector_state_manager = connector_state_manager or ConnectorStateManager()
+        self._api_budget: Optional[Union[APIBudget, HttpAPIBudget]] = None
 
     def _init_mappings(self) -> None:
         self.PYDANTIC_MODEL_TO_CONSTRUCTOR: Mapping[Type[BaseModel], Callable[..., Any]] = {
@@ -590,6 +618,12 @@ class ModelToComponentFactory:
             StreamConfigModel: self.create_stream_config,
             ComponentMappingDefinitionModel: self.create_components_mapping_definition,
             ZipfileDecoderModel: self.create_zipfile_decoder,
+            HTTPAPIBudgetModel: self.create_http_api_budget,
+            FixedWindowCallRatePolicyModel: self.create_fixed_window_call_rate_policy,
+            MovingWindowCallRatePolicyModel: self.create_moving_window_call_rate_policy,
+            UnlimitedCallRatePolicyModel: self.create_unlimited_call_rate_policy,
+            RateModel: self.create_rate,
+            HttpRequestRegexMatcherModel: self.create_http_request_matcher,
         }
 
         # Needed for the case where we need to perform a second parse on the fields of a custom component
@@ -1902,6 +1936,8 @@ class ModelToComponentFactory:
             )
         )
 
+        api_budget = self._api_budget
+
         request_options_provider = InterpolatedRequestOptionsProvider(
             request_body_data=model.request_body_data,
             request_body_json=model.request_body_json,
@@ -1922,6 +1958,7 @@ class ModelToComponentFactory:
             path=model.path,
             authenticator=authenticator,
             error_handler=error_handler,
+            api_budget=api_budget,
             http_method=HttpMethod[model.http_method.value],
             request_options_provider=request_options_provider,
             config=config,
@@ -2378,6 +2415,8 @@ class ModelToComponentFactory:
             if model.record_filter
             else None
         )
+
+        transform_before_filtering = False
         if client_side_incremental_sync:
             record_filter = ClientSideIncrementalRecordFilterDecorator(
                 config=config,
@@ -2387,6 +2426,8 @@ class ModelToComponentFactory:
                 else None,
                 **client_side_incremental_sync,
             )
+            transform_before_filtering = True
+
         schema_normalization = (
             TypeTransformer(SCHEMA_TRANSFORMER_TYPE_MAPPING[model.schema_normalization])
             if isinstance(model.schema_normalization, SchemaNormalizationModel)
@@ -2401,6 +2442,7 @@ class ModelToComponentFactory:
             transformations=transformations or [],
             schema_normalization=schema_normalization,
             parameters=model.parameters or {},
+            transform_before_filtering=transform_before_filtering,
         )
 
     @staticmethod
@@ -2921,3 +2963,84 @@ class ModelToComponentFactory:
             return isinstance(parser.inner_parser, JsonParser)
         else:
             return False
+
+    def create_http_api_budget(
+        self, model: HTTPAPIBudgetModel, config: Config, **kwargs: Any
+    ) -> HttpAPIBudget:
+        policies = [
+            self._create_component_from_model(model=policy, config=config)
+            for policy in model.policies
+        ]
+
+        return HttpAPIBudget(
+            policies=policies,
+            ratelimit_reset_header=model.ratelimit_reset_header or "ratelimit-reset",
+            ratelimit_remaining_header=model.ratelimit_remaining_header or "ratelimit-remaining",
+            status_codes_for_ratelimit_hit=model.status_codes_for_ratelimit_hit or [429],
+        )
+
+    def create_fixed_window_call_rate_policy(
+        self, model: FixedWindowCallRatePolicyModel, config: Config, **kwargs: Any
+    ) -> FixedWindowCallRatePolicy:
+        matchers = [
+            self._create_component_from_model(model=matcher, config=config)
+            for matcher in model.matchers
+        ]
+
+        # Set the initial reset timestamp to 10 days from now.
+        # This value will be updated by the first request.
+        return FixedWindowCallRatePolicy(
+            next_reset_ts=datetime.datetime.now() + datetime.timedelta(days=10),
+            period=parse_duration(model.period),
+            call_limit=model.call_limit,
+            matchers=matchers,
+        )
+
+    def create_moving_window_call_rate_policy(
+        self, model: MovingWindowCallRatePolicyModel, config: Config, **kwargs: Any
+    ) -> MovingWindowCallRatePolicy:
+        rates = [
+            self._create_component_from_model(model=rate, config=config) for rate in model.rates
+        ]
+        matchers = [
+            self._create_component_from_model(model=matcher, config=config)
+            for matcher in model.matchers
+        ]
+        return MovingWindowCallRatePolicy(
+            rates=rates,
+            matchers=matchers,
+        )
+
+    def create_unlimited_call_rate_policy(
+        self, model: UnlimitedCallRatePolicyModel, config: Config, **kwargs: Any
+    ) -> UnlimitedCallRatePolicy:
+        matchers = [
+            self._create_component_from_model(model=matcher, config=config)
+            for matcher in model.matchers
+        ]
+
+        return UnlimitedCallRatePolicy(
+            matchers=matchers,
+        )
+
+    def create_rate(self, model: RateModel, config: Config, **kwargs: Any) -> Rate:
+        return Rate(
+            limit=model.limit,
+            interval=parse_duration(model.interval),
+        )
+
+    def create_http_request_matcher(
+        self, model: HttpRequestRegexMatcherModel, config: Config, **kwargs: Any
+    ) -> HttpRequestRegexMatcher:
+        return HttpRequestRegexMatcher(
+            method=model.method,
+            url_base=model.url_base,
+            url_path_pattern=model.url_path_pattern,
+            params=model.params,
+            headers=model.headers,
+        )
+
+    def set_api_budget(self, component_definition: ComponentDefinition, config: Config) -> None:
+        self._api_budget = self.create_component(
+            model_type=HTTPAPIBudgetModel, component_definition=component_definition, config=config
+        )

airbyte_cdk/sources/declarative/requesters/http_requester.py

@@ -22,6 +22,7 @@ from airbyte_cdk.sources.declarative.requesters.request_options.interpolated_req
 )
 from airbyte_cdk.sources.declarative.requesters.requester import HttpMethod, Requester
 from airbyte_cdk.sources.message import MessageRepository, NoopMessageRepository
+from airbyte_cdk.sources.streams.call_rate import APIBudget
 from airbyte_cdk.sources.streams.http import HttpClient
 from airbyte_cdk.sources.streams.http.error_handlers import ErrorHandler
 from airbyte_cdk.sources.types import Config, StreamSlice, StreamState
@@ -55,6 +56,7 @@ class HttpRequester(Requester):
     http_method: Union[str, HttpMethod] = HttpMethod.GET
     request_options_provider: Optional[InterpolatedRequestOptionsProvider] = None
     error_handler: Optional[ErrorHandler] = None
+    api_budget: Optional[APIBudget] = None
     disable_retries: bool = False
     message_repository: MessageRepository = NoopMessageRepository()
     use_cache: bool = False
@@ -91,6 +93,7 @@ class HttpRequester(Requester):
             name=self.name,
             logger=self.logger,
             error_handler=self.error_handler,
+            api_budget=self.api_budget,
             authenticator=self._authenticator,
             use_cache=self.use_cache,
             backoff_strategy=backoff_strategies,