airbyte-cdk 6.34.0.dev1__py3-none-any.whl → 6.34.1.dev0__py3-none-any.whl

airbyte_cdk/sources/declarative/extractors/record_selector.py

@@ -41,7 +41,6 @@ class RecordSelector(HttpSelector):
     _name: Union[InterpolatedString, str] = field(init=False, repr=False, default="")
     record_filter: Optional[RecordFilter] = None
     transformations: List[RecordTransformation] = field(default_factory=lambda: [])
-    transform_before_filtering: bool = False
 
     def __post_init__(self, parameters: Mapping[str, Any]) -> None:
         self._parameters = parameters
@@ -105,17 +104,9 @@ class RecordSelector(HttpSelector):
         Until we decide to move this logic away from the selector, we made this method public so that users like AsyncJobRetriever could
         share the logic of doing transformations on a set of records.
         """
-        if self.transform_before_filtering:
-            transformed_data = self._transform(all_data, stream_state, stream_slice)
-            transformed_filtered_data = self._filter(
-                transformed_data, stream_state, stream_slice, next_page_token
-            )
-        else:
-            filtered_data = self._filter(all_data, stream_state, stream_slice, next_page_token)
-            transformed_filtered_data = self._transform(filtered_data, stream_state, stream_slice)
-        normalized_data = self._normalize_by_schema(
-            transformed_filtered_data, schema=records_schema
-        )
+        filtered_data = self._filter(all_data, stream_state, stream_slice, next_page_token)
+        transformed_data = self._transform(filtered_data, stream_state, stream_slice)
+        normalized_data = self._normalize_by_schema(transformed_data, schema=records_schema)
         for data in normalized_data:
             yield Record(data=data, stream_name=self.name, associated_slice=stream_slice)
 

airbyte_cdk/sources/declarative/incremental/concurrent_partition_cursor.py

@@ -5,7 +5,6 @@
 import copy
 import logging
 import threading
-import time
 from collections import OrderedDict
 from copy import deepcopy
 from datetime import timedelta
@@ -59,8 +58,7 @@ class ConcurrentPerPartitionCursor(Cursor):
     CurrentPerPartitionCursor expects the state of the ConcurrentCursor to follow the format {cursor_field: cursor_value}.
     """
 
-    DEFAULT_MAX_PARTITIONS_NUMBER = 25_000
-    SWITCH_TO_GLOBAL_LIMIT = 10_000
+    DEFAULT_MAX_PARTITIONS_NUMBER = 10000
     _NO_STATE: Mapping[str, Any] = {}
     _NO_CURSOR_STATE: Mapping[str, Any] = {}
     _GLOBAL_STATE_KEY = "state"
@@ -101,11 +99,9 @@ class ConcurrentPerPartitionCursor(Cursor):
         self._new_global_cursor: Optional[StreamState] = None
         self._lookback_window: int = 0
         self._parent_state: Optional[StreamState] = None
-        self._number_of_partitions: int = 0
+        self._over_limit: int = 0
         self._use_global_cursor: bool = False
         self._partition_serializer = PerPartitionKeySerializer()
-        # Track the last time a state message was emitted
-        self._last_emission_time: float = 0.0
 
         self._set_initial_state(stream_state)
 
@@ -145,16 +141,21 @@ class ConcurrentPerPartitionCursor(Cursor):
             raise ValueError("stream_slice cannot be None")
 
         partition_key = self._to_partition_key(stream_slice.partition)
+        self._cursor_per_partition[partition_key].close_partition(partition=partition)
         with self._lock:
             self._semaphore_per_partition[partition_key].acquire()
-            if not self._use_global_cursor:
-                self._cursor_per_partition[partition_key].close_partition(partition=partition)
-                cursor = self._cursor_per_partition[partition_key]
+            cursor = self._cursor_per_partition[partition_key]
+            if (
+                partition_key in self._finished_partitions
+                and self._semaphore_per_partition[partition_key]._value == 0
+            ):
                 if (
-                    partition_key in self._finished_partitions
-                    and self._semaphore_per_partition[partition_key]._value == 0
+                    self._new_global_cursor is None
+                    or self._new_global_cursor[self.cursor_field.cursor_field_key]
+                    < cursor.state[self.cursor_field.cursor_field_key]
                 ):
-                    self._update_global_cursor(cursor.state[self.cursor_field.cursor_field_key])
+                    self._new_global_cursor = copy.deepcopy(cursor.state)
+            if not self._use_global_cursor:
                 self._emit_state_message()
 
     def ensure_at_least_one_state_emitted(self) -> None:
@@ -168,23 +169,9 @@ class ConcurrentPerPartitionCursor(Cursor):
             self._global_cursor = self._new_global_cursor
             self._lookback_window = self._timer.finish()
             self._parent_state = self._partition_router.get_stream_state()
-        self._emit_state_message(throttle=False)
+        self._emit_state_message()
 
-    def _throttle_state_message(self) -> Optional[float]:
-        """
-        Throttles the state message emission to once every 60 seconds.
-        """
-        current_time = time.time()
-        if current_time - self._last_emission_time <= 60:
-            return None
-        return current_time
-
-    def _emit_state_message(self, throttle: bool = True) -> None:
-        if throttle:
-            current_time = self._throttle_state_message()
-            if current_time is None:
-                return
-            self._last_emission_time = current_time
+    def _emit_state_message(self) -> None:
         self._connector_state_manager.update_state_for_stream(
             self._stream_name,
             self._stream_namespace,
@@ -215,7 +202,6 @@ class ConcurrentPerPartitionCursor(Cursor):
                 self._lookback_window if self._global_cursor else 0,
             )
             with self._lock:
-                self._number_of_partitions += 1
                 self._cursor_per_partition[self._to_partition_key(partition.partition)] = cursor
             self._semaphore_per_partition[self._to_partition_key(partition.partition)] = (
                 threading.Semaphore(0)
@@ -246,15 +232,9 @@ class ConcurrentPerPartitionCursor(Cursor):
         - Logs a warning each time a partition is removed, indicating whether it was finished
           or removed due to being the oldest.
         """
-        if not self._use_global_cursor and self.limit_reached():
-            logger.info(
-                f"Exceeded the 'SWITCH_TO_GLOBAL_LIMIT' of {self.SWITCH_TO_GLOBAL_LIMIT}. "
-                f"Switching to global cursor for {self._stream_name}."
-            )
-            self._use_global_cursor = True
-
         with self._lock:
             while len(self._cursor_per_partition) > self.DEFAULT_MAX_PARTITIONS_NUMBER - 1:
+                self._over_limit += 1
                 # Try removing finished partitions first
                 for partition_key in list(self._cursor_per_partition.keys()):
                     if (
@@ -265,7 +245,7 @@ class ConcurrentPerPartitionCursor(Cursor):
                             partition_key
                         )  # Remove the oldest partition
                         logger.warning(
-                            f"The maximum number of partitions has been reached. Dropping the oldest finished partition: {oldest_partition}. Over limit: {self._number_of_partitions - self.DEFAULT_MAX_PARTITIONS_NUMBER}."
+                            f"The maximum number of partitions has been reached. Dropping the oldest finished partition: {oldest_partition}. Over limit: {self._over_limit}."
                         )
                         break
                 else:
@@ -274,7 +254,7 @@ class ConcurrentPerPartitionCursor(Cursor):
                         1
                     ]  # Remove the oldest partition
                     logger.warning(
-                        f"The maximum number of partitions has been reached. Dropping the oldest partition: {oldest_partition}. Over limit: {self._number_of_partitions - self.DEFAULT_MAX_PARTITIONS_NUMBER}."
+                        f"The maximum number of partitions has been reached. Dropping the oldest partition: {oldest_partition}. Over limit: {self._over_limit}."
                     )
 
     def _set_initial_state(self, stream_state: StreamState) -> None:
@@ -334,7 +314,6 @@ class ConcurrentPerPartitionCursor(Cursor):
             self._lookback_window = int(stream_state.get("lookback_window", 0))
 
             for state in stream_state.get(self._PERPARTITION_STATE_KEY, []):
-                self._number_of_partitions += 1
                 self._cursor_per_partition[self._to_partition_key(state["partition"])] = (
                     self._create_cursor(state["cursor"])
                 )
@@ -375,26 +354,16 @@ class ConcurrentPerPartitionCursor(Cursor):
             self._new_global_cursor = deepcopy(fixed_global_state)
 
     def observe(self, record: Record) -> None:
+        if not self._use_global_cursor and self.limit_reached():
+            self._use_global_cursor = True
+
         if not record.associated_slice:
             raise ValueError(
                 "Invalid state as stream slices that are emitted should refer to an existing cursor"
             )
-
-        record_cursor = self._connector_state_converter.output_format(
-            self._connector_state_converter.parse_value(self._cursor_field.extract_value(record))
-        )
-        self._update_global_cursor(record_cursor)
-        if not self._use_global_cursor:
-            self._cursor_per_partition[
-                self._to_partition_key(record.associated_slice.partition)
-            ].observe(record)
-
-    def _update_global_cursor(self, value: Any) -> None:
-        if (
-            self._new_global_cursor is None
-            or self._new_global_cursor[self.cursor_field.cursor_field_key] < value
-        ):
-            self._new_global_cursor = {self.cursor_field.cursor_field_key: copy.deepcopy(value)}
+        self._cursor_per_partition[
+            self._to_partition_key(record.associated_slice.partition)
+        ].observe(record)
 
     def _to_partition_key(self, partition: Mapping[str, Any]) -> str:
         return self._partition_serializer.to_partition_key(partition)
@@ -428,4 +397,4 @@ class ConcurrentPerPartitionCursor(Cursor):
         return cursor
 
     def limit_reached(self) -> bool:
-        return self._number_of_partitions > self.SWITCH_TO_GLOBAL_LIMIT
+        return self._over_limit > self.DEFAULT_MAX_PARTITIONS_NUMBER

airbyte_cdk/sources/declarative/incremental/datetime_based_cursor.py

@@ -21,7 +21,6 @@ from airbyte_cdk.sources.declarative.requesters.request_option import (
 )
 from airbyte_cdk.sources.message import MessageRepository
 from airbyte_cdk.sources.types import Config, Record, StreamSlice, StreamState
-from airbyte_cdk.utils.mapping_helpers import _validate_component_request_option_paths
 
 
 @dataclass
@@ -123,10 +122,6 @@ class DatetimeBasedCursor(DeclarativeCursor):
         if not self.cursor_datetime_formats:
             self.cursor_datetime_formats = [self.datetime_format]
 
-        _validate_component_request_option_paths(
-            self.config, self.start_time_option, self.end_time_option
-        )
-
     def get_stream_state(self) -> StreamState:
         return {self.cursor_field.eval(self.config): self._cursor} if self._cursor else {}  # type: ignore  # cursor_field is converted to an InterpolatedString in __post_init__
 
@@ -370,15 +365,14 @@ class DatetimeBasedCursor(DeclarativeCursor):
         options: MutableMapping[str, Any] = {}
         if not stream_slice:
             return options
-
         if self.start_time_option and self.start_time_option.inject_into == option_type:
-            start_time_value = stream_slice.get(self._partition_field_start.eval(self.config))
-            self.start_time_option.inject_into_request(options, start_time_value, self.config)
-
+            options[self.start_time_option.field_name.eval(config=self.config)] = stream_slice.get(  # type: ignore # field_name is always casted to an interpolated string
+                self._partition_field_start.eval(self.config)
+            )
         if self.end_time_option and self.end_time_option.inject_into == option_type:
-            end_time_value = stream_slice.get(self._partition_field_end.eval(self.config))
-            self.end_time_option.inject_into_request(options, end_time_value, self.config)
-
+            options[self.end_time_option.field_name.eval(config=self.config)] = stream_slice.get(  # type: ignore [union-attr]
+                self._partition_field_end.eval(self.config)
+            )
         return options
 
     def should_be_synced(self, record: Record) -> bool:

airbyte_cdk/sources/declarative/manifest_declarative_source.py

@@ -137,10 +137,6 @@ class ManifestDeclarativeSource(DeclarativeSource):
             self._source_config, config
         )
 
-        api_budget_model = self._source_config.get("api_budget")
-        if api_budget_model:
-            self._constructor.set_api_budget(api_budget_model, config)
-
         source_streams = [
             self._constructor.create_component(
                 DeclarativeStreamModel,
@@ -369,11 +365,6 @@ class ManifestDeclarativeSource(DeclarativeSource):
                 # Ensure that each stream is created with a unique name
                 name = dynamic_stream.get("name")
 
-                if not isinstance(name, str):
-                    raise ValueError(
-                        f"Expected stream name {name} to be a string, got {type(name)}."
-                    )
-
                 if name in seen_dynamic_streams:
                     error_message = f"Dynamic streams list contains a duplicate name: {name}. Please contact Airbyte Support."
                     failure_type = FailureType.system_error

airbyte_cdk/sources/declarative/models/declarative_component_schema.py

@@ -642,48 +642,6 @@ class OAuthAuthenticator(BaseModel):
     parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
 
 
-class Rate(BaseModel):
-    class Config:
-        extra = Extra.allow
-
-    limit: int = Field(
-        ...,
-        description="The maximum number of calls allowed within the interval.",
-        title="Limit",
-    )
-    interval: str = Field(
-        ...,
-        description="The time interval for the rate limit.",
-        examples=["PT1H", "P1D"],
-        title="Interval",
-    )
-
-
-class HttpRequestRegexMatcher(BaseModel):
-    class Config:
-        extra = Extra.allow
-
-    method: Optional[str] = Field(
-        None, description="The HTTP method to match (e.g., GET, POST).", title="Method"
-    )
-    url_base: Optional[str] = Field(
-        None,
-        description='The base URL (scheme and host, e.g. "https://api.example.com") to match.',
-        title="URL Base",
-    )
-    url_path_pattern: Optional[str] = Field(
-        None,
-        description="A regular expression pattern to match the URL path.",
-        title="URL Path Pattern",
-    )
-    params: Optional[Dict[str, Any]] = Field(
-        None, description="The query parameters to match.", title="Parameters"
-    )
-    headers: Optional[Dict[str, Any]] = Field(
-        None, description="The headers to match.", title="Headers"
-    )
-
-
 class DpathExtractor(BaseModel):
     type: Literal["DpathExtractor"]
     field_path: List[str] = Field(
@@ -929,6 +887,15 @@ class CustomDecoder(BaseModel):
     parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
 
 
+class GzipJsonDecoder(BaseModel):
+    class Config:
+        extra = Extra.allow
+
+    type: Literal["GzipJsonDecoder"]
+    encoding: Optional[str] = "utf-8"
+    parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
+
+
 class MinMaxDatetime(BaseModel):
     type: Literal["MinMaxDatetime"]
     datetime: str = Field(
@@ -1233,17 +1200,11 @@ class InjectInto(Enum):
 
 class RequestOption(BaseModel):
     type: Literal["RequestOption"]
-    field_name: Optional[str] = Field(
-        None,
-        description="Configures which key should be used in the location that the descriptor is being injected into. We hope to eventually deprecate this field in favor of `field_path` for all request_options, but must currently maintain it for backwards compatibility in the Builder.",
+    field_name: str = Field(
+        ...,
+        description="Configures which key should be used in the location that the descriptor is being injected into",
         examples=["segment_id"],
-        title="Field Name",
-    )
-    field_path: Optional[List[str]] = Field(
-        None,
-        description="Configures a path to be used for nested structures in JSON body requests (e.g. GraphQL queries)",
-        examples=[["data", "viewer", "id"]],
-        title="Field Path",
+        title="Request Option",
     )
     inject_into: InjectInto = Field(
         ...,
@@ -1307,8 +1268,18 @@ class LegacySessionTokenAuthenticator(BaseModel):
     parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
 
 
-class CsvDecoder(BaseModel):
-    type: Literal["CsvDecoder"]
+class JsonParser(BaseModel):
+    type: Literal["JsonParser"]
+    encoding: Optional[str] = "utf-8"
+
+
+class JsonLineParser(BaseModel):
+    type: Literal["JsonLineParser"]
+    encoding: Optional[str] = "utf-8"
+
+
+class CsvParser(BaseModel):
+    type: Literal["CsvParser"]
     encoding: Optional[str] = "utf-8"
     delimiter: Optional[str] = ","
 
@@ -1607,55 +1578,6 @@ class DatetimeBasedCursor(BaseModel):
     parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
 
 
-class FixedWindowCallRatePolicy(BaseModel):
-    class Config:
-        extra = Extra.allow
-
-    type: Literal["FixedWindowCallRatePolicy"]
-    period: str = Field(
-        ..., description="The time interval for the rate limit window.", title="Period"
-    )
-    call_limit: int = Field(
-        ...,
-        description="The maximum number of calls allowed within the period.",
-        title="Call Limit",
-    )
-    matchers: List[HttpRequestRegexMatcher] = Field(
-        ...,
-        description="List of matchers that define which requests this policy applies to.",
-        title="Matchers",
-    )
-
-
-class MovingWindowCallRatePolicy(BaseModel):
-    class Config:
-        extra = Extra.allow
-
-    type: Literal["MovingWindowCallRatePolicy"]
-    rates: List[Rate] = Field(
-        ...,
-        description="List of rates that define the call limits for different time intervals.",
-        title="Rates",
-    )
-    matchers: List[HttpRequestRegexMatcher] = Field(
-        ...,
-        description="List of matchers that define which requests this policy applies to.",
-        title="Matchers",
-    )
-
-
-class UnlimitedCallRatePolicy(BaseModel):
-    class Config:
-        extra = Extra.allow
-
-    type: Literal["UnlimitedCallRatePolicy"]
-    matchers: List[HttpRequestRegexMatcher] = Field(
-        ...,
-        description="List of matchers that define which requests this policy applies to.",
-        title="Matchers",
-    )
-
-
 class DefaultErrorHandler(BaseModel):
     type: Literal["DefaultErrorHandler"]
     backoff_strategies: Optional[
@@ -1752,9 +1674,9 @@ class RecordSelector(BaseModel):
     parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
 
 
-class GzipDecoder(BaseModel):
-    type: Literal["GzipDecoder"]
-    decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder]
+class GzipParser(BaseModel):
+    type: Literal["GzipParser"]
+    inner_parser: Union[JsonLineParser, CsvParser, JsonParser]
 
 
 class Spec(BaseModel):
@@ -1787,51 +1709,23 @@ class CompositeErrorHandler(BaseModel):
     parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
 
 
-class HTTPAPIBudget(BaseModel):
-    class Config:
-        extra = Extra.allow
-
-    type: Literal["HTTPAPIBudget"]
-    policies: List[
-        Union[
-            FixedWindowCallRatePolicy,
-            MovingWindowCallRatePolicy,
-            UnlimitedCallRatePolicy,
-        ]
-    ] = Field(
-        ...,
-        description="List of call rate policies that define how many calls are allowed.",
-        title="Policies",
-    )
-    ratelimit_reset_header: Optional[str] = Field(
-        "ratelimit-reset",
-        description="The HTTP response header name that indicates when the rate limit resets.",
-        title="Rate Limit Reset Header",
-    )
-    ratelimit_remaining_header: Optional[str] = Field(
-        "ratelimit-remaining",
-        description="The HTTP response header name that indicates the number of remaining allowed calls.",
-        title="Rate Limit Remaining Header",
-    )
-    status_codes_for_ratelimit_hit: Optional[List[int]] = Field(
-        [429],
-        description="List of HTTP status codes that indicate a rate limit has been hit.",
-        title="Status Codes for Rate Limit Hit",
-    )
-
-
 class ZipfileDecoder(BaseModel):
     class Config:
         extra = Extra.allow
 
     type: Literal["ZipfileDecoder"]
-    decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder] = Field(
+    parser: Union[GzipParser, JsonParser, JsonLineParser, CsvParser] = Field(
         ...,
         description="Parser to parse the decompressed data from the zipfile(s).",
         title="Parser",
     )
 
 
+class CompositeRawDecoder(BaseModel):
+    type: Literal["CompositeRawDecoder"]
+    parser: Union[GzipParser, JsonParser, JsonLineParser, CsvParser]
+
+
 class DeclarativeSource1(BaseModel):
     class Config:
         extra = Extra.forbid
@@ -1848,7 +1742,6 @@ class DeclarativeSource1(BaseModel):
     definitions: Optional[Dict[str, Any]] = None
     spec: Optional[Spec] = None
     concurrency_level: Optional[ConcurrencyLevel] = None
-    api_budget: Optional[HTTPAPIBudget] = None
     metadata: Optional[Dict[str, Any]] = Field(
         None,
         description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.",
@@ -1875,7 +1768,6 @@ class DeclarativeSource2(BaseModel):
     definitions: Optional[Dict[str, Any]] = None
     spec: Optional[Spec] = None
     concurrency_level: Optional[ConcurrencyLevel] = None
-    api_budget: Optional[HTTPAPIBudget] = None
     metadata: Optional[Dict[str, Any]] = Field(
         None,
         description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.",
@@ -2035,7 +1927,7 @@ class SessionTokenAuthenticator(BaseModel):
         description="Authentication method to use for requests sent to the API, specifying how to inject the session token.",
         title="Data Request Authentication",
     )
-    decoder: Optional[Union[JsonDecoder, XmlDecoder]] = Field(
+    decoder: Optional[Union[JsonDecoder, XmlDecoder, CompositeRawDecoder]] = Field(
         None, description="Component used to decode the response.", title="Decoder"
     )
     parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
@@ -2235,12 +2127,12 @@ class SimpleRetriever(BaseModel):
     decoder: Optional[
         Union[
             CustomDecoder,
-            CsvDecoder,
-            GzipDecoder,
             JsonDecoder,
             JsonlDecoder,
             IterableDecoder,
             XmlDecoder,
+            GzipJsonDecoder,
+            CompositeRawDecoder,
             ZipfileDecoder,
         ]
     ] = Field(
@@ -2313,12 +2205,12 @@ class AsyncRetriever(BaseModel):
     decoder: Optional[
         Union[
             CustomDecoder,
-            CsvDecoder,
-            GzipDecoder,
             JsonDecoder,
             JsonlDecoder,
             IterableDecoder,
             XmlDecoder,
+            GzipJsonDecoder,
+            CompositeRawDecoder,
             ZipfileDecoder,
         ]
     ] = Field(
@@ -2329,12 +2221,12 @@ class AsyncRetriever(BaseModel):
     download_decoder: Optional[
         Union[
             CustomDecoder,
-            CsvDecoder,
-            GzipDecoder,
             JsonDecoder,
             JsonlDecoder,
             IterableDecoder,
             XmlDecoder,
+            GzipJsonDecoder,
+            CompositeRawDecoder,
             ZipfileDecoder,
         ]
     ] = Field(
@@ -2379,7 +2271,6 @@ class DynamicDeclarativeStream(BaseModel):
 
 
 ComplexFieldType.update_forward_refs()
-GzipDecoder.update_forward_refs()
 CompositeErrorHandler.update_forward_refs()
 DeclarativeSource1.update_forward_refs()
 DeclarativeSource2.update_forward_refs()