airbyte-cdk 6.36.2__py3-none-any.whl → 6.37.0.dev1__py3-none-any.whl

airbyte_cdk/sources/declarative/models/declarative_component_schema.py

@@ -646,7 +646,7 @@ class Rate(BaseModel):
     class Config:
         extra = Extra.allow
 
-    limit: Union[int, str] = Field(
+    limit: int = Field(
         ...,
         description="The maximum number of calls allowed within the interval.",
         title="Limit",
@@ -2225,7 +2225,15 @@ class SimpleRetriever(BaseModel):
             CustomPartitionRouter,
             ListPartitionRouter,
             SubstreamPartitionRouter,
-            List[Union[CustomPartitionRouter, ListPartitionRouter, SubstreamPartitionRouter]],
+            GroupingPartitionRouter,
+            List[
+                Union[
+                    CustomPartitionRouter,
+                    ListPartitionRouter,
+                    SubstreamPartitionRouter,
+                    GroupingPartitionRouter,
+                ]
+            ],
         ]
     ] = Field(
         [],
@@ -2303,7 +2311,15 @@ class AsyncRetriever(BaseModel):
             CustomPartitionRouter,
             ListPartitionRouter,
             SubstreamPartitionRouter,
-            List[Union[CustomPartitionRouter, ListPartitionRouter, SubstreamPartitionRouter]],
+            GroupingPartitionRouter,
+            List[
+                Union[
+                    CustomPartitionRouter,
+                    ListPartitionRouter,
+                    SubstreamPartitionRouter,
+                    GroupingPartitionRouter,
+                ]
+            ],
         ]
     ] = Field(
         [],
@@ -2355,6 +2371,29 @@ class SubstreamPartitionRouter(BaseModel):
     parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
 
 
+class GroupingPartitionRouter(BaseModel):
+    type: Literal["GroupingPartitionRouter"]
+    group_size: int = Field(
+        ...,
+        description="The number of partitions to include in each group. This determines how many partition values are batched together in a single slice.",
+        examples=[10, 50],
+        title="Group Size",
+    )
+    underlying_partition_router: Union[
+        CustomPartitionRouter, ListPartitionRouter, SubstreamPartitionRouter
+    ] = Field(
+        ...,
+        description="The partition router whose output will be grouped. This can be any valid partition router component.",
+        title="Underlying Partition Router",
+    )
+    deduplicate: Optional[bool] = Field(
+        True,
+        description="If true, ensures that partitions are unique within each group by removing duplicates based on the partition key.",
+        title="Deduplicate Partitions",
+    )
+    parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
+
+
 class HttpComponentsResolver(BaseModel):
     type: Literal["HttpComponentsResolver"]
     retriever: Union[AsyncRetriever, CustomRetriever, SimpleRetriever] = Field(

airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py

@@ -227,6 +227,9 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     FlattenFields as FlattenFieldsModel,
 )
+from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
+    GroupingPartitionRouter as GroupingPartitionRouterModel,
+)
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     GzipDecoder as GzipDecoderModel,
 )
@@ -379,6 +382,7 @@ from airbyte_cdk.sources.declarative.parsers.custom_code_compiler import (
 )
 from airbyte_cdk.sources.declarative.partition_routers import (
     CartesianProductStreamSlicer,
+    GroupingPartitionRouter,
     ListPartitionRouter,
     PartitionRouter,
     SinglePartitionRouter,
@@ -624,6 +628,7 @@ class ModelToComponentFactory:
             UnlimitedCallRatePolicyModel: self.create_unlimited_call_rate_policy,
             RateModel: self.create_rate,
             HttpRequestRegexMatcherModel: self.create_http_request_matcher,
+            GroupingPartitionRouterModel: self.create_grouping_partition_router,
         }
 
         # Needed for the case where we need to perform a second parse on the fields of a custom component
@@ -2091,10 +2096,10 @@ class ModelToComponentFactory:
     def create_json_decoder(model: JsonDecoderModel, config: Config, **kwargs: Any) -> Decoder:
         return JsonDecoder(parameters={})
 
-    def create_csv_decoder(self, model: CsvDecoderModel, config: Config, **kwargs: Any) -> Decoder:
+    @staticmethod
+    def create_csv_decoder(model: CsvDecoderModel, config: Config, **kwargs: Any) -> Decoder:
         return CompositeRawDecoder(
-            parser=ModelToComponentFactory._get_parser(model, config),
-            stream_response=False if self._emit_connector_builder_messages else True,
+            parser=ModelToComponentFactory._get_parser(model, config), stream_response=True
         )
 
     @staticmethod
@@ -2103,12 +2108,10 @@ class ModelToComponentFactory:
             parser=ModelToComponentFactory._get_parser(model, config), stream_response=True
         )
 
-    def create_gzip_decoder(
-        self, model: GzipDecoderModel, config: Config, **kwargs: Any
-    ) -> Decoder:
+    @staticmethod
+    def create_gzip_decoder(model: GzipDecoderModel, config: Config, **kwargs: Any) -> Decoder:
         return CompositeRawDecoder(
-            parser=ModelToComponentFactory._get_parser(model, config),
-            stream_response=False if self._emit_connector_builder_messages else True,
+            parser=ModelToComponentFactory._get_parser(model, config), stream_response=True
         )
 
     @staticmethod
@@ -2629,47 +2632,6 @@ class ModelToComponentFactory:
         transformations: List[RecordTransformation],
         **kwargs: Any,
     ) -> AsyncRetriever:
-        def _get_download_retriever() -> SimpleRetrieverTestReadDecorator | SimpleRetriever:
-            record_selector = RecordSelector(
-                extractor=download_extractor,
-                name=name,
-                record_filter=None,
-                transformations=transformations,
-                schema_normalization=TypeTransformer(TransformConfig.NoTransform),
-                config=config,
-                parameters={},
-            )
-            paginator = (
-                self._create_component_from_model(
-                    model=model.download_paginator, decoder=decoder, config=config, url_base=""
-                )
-                if model.download_paginator
-                else NoPagination(parameters={})
-            )
-            maximum_number_of_slices = self._limit_slices_fetched or 5
-
-            if self._limit_slices_fetched or self._emit_connector_builder_messages:
-                return SimpleRetrieverTestReadDecorator(
-                    requester=download_requester,
-                    record_selector=record_selector,
-                    primary_key=None,
-                    name=job_download_components_name,
-                    paginator=paginator,
-                    config=config,
-                    parameters={},
-                    maximum_number_of_slices=maximum_number_of_slices,
-                )
-
-            return SimpleRetriever(
-                requester=download_requester,
-                record_selector=record_selector,
-                primary_key=None,
-                name=job_download_components_name,
-                paginator=paginator,
-                config=config,
-                parameters={},
-            )
-
         decoder = (
             self._create_component_from_model(model=model.decoder, config=config)
             if model.decoder
@@ -2723,7 +2685,29 @@ class ModelToComponentFactory:
             config=config,
             name=job_download_components_name,
         )
-        download_retriever = _get_download_retriever()
+        download_retriever = SimpleRetriever(
+            requester=download_requester,
+            record_selector=RecordSelector(
+                extractor=download_extractor,
+                name=name,
+                record_filter=None,
+                transformations=transformations,
+                schema_normalization=TypeTransformer(TransformConfig.NoTransform),
+                config=config,
+                parameters={},
+            ),
+            primary_key=None,
+            name=job_download_components_name,
+            paginator=(
+                self._create_component_from_model(
+                    model=model.download_paginator, decoder=decoder, config=config, url_base=""
+                )
+                if model.download_paginator
+                else NoPagination(parameters={})
+            ),
+            config=config,
+            parameters={},
+        )
         abort_requester = (
             self._create_component_from_model(
                 model=model.abort_requester,
@@ -3045,9 +3029,8 @@ class ModelToComponentFactory:
         )
 
     def create_rate(self, model: RateModel, config: Config, **kwargs: Any) -> Rate:
-        interpolated_limit = InterpolatedString.create(str(model.limit), parameters={})
         return Rate(
-            limit=int(interpolated_limit.eval(config=config)),
+            limit=model.limit,
             interval=parse_duration(model.interval),
         )
 
@@ -3066,3 +3049,31 @@ class ModelToComponentFactory:
         self._api_budget = self.create_component(
             model_type=HTTPAPIBudgetModel, component_definition=component_definition, config=config
         )
+
+    def create_grouping_partition_router(
+        self, model: GroupingPartitionRouterModel, config: Config, **kwargs: Any
+    ) -> GroupingPartitionRouter:
+        underlying_router = self._create_component_from_model(
+            model=model.underlying_partition_router, config=config
+        )
+        if model.group_size < 1:
+            raise ValueError(f"Group size must be greater than 0, got {model.group_size}")
+
+        if not isinstance(underlying_router, PartitionRouter):
+            raise ValueError(
+                f"Underlying partition router must be a PartitionRouter subclass, got {type(underlying_router)}"
+            )
+
+        if isinstance(underlying_router, SubstreamPartitionRouter):
+            if any(
+                parent_config.request_option
+                for parent_config in underlying_router.parent_stream_configs
+            ):
+                raise ValueError("Request options are not supported for GroupingPartitionRouter.")
+
+        return GroupingPartitionRouter(
+            group_size=model.group_size,
+            underlying_partition_router=underlying_router,
+            deduplicate=model.deduplicate if model.deduplicate is not None else True,
+            config=config,
+        )

airbyte_cdk/sources/declarative/partition_routers/__init__.py

@@ -8,6 +8,9 @@ from airbyte_cdk.sources.declarative.partition_routers.async_job_partition_route
 from airbyte_cdk.sources.declarative.partition_routers.cartesian_product_stream_slicer import (
     CartesianProductStreamSlicer,
 )
+from airbyte_cdk.sources.declarative.partition_routers.grouping_partition_router import (
+    GroupingPartitionRouter,
+)
 from airbyte_cdk.sources.declarative.partition_routers.list_partition_router import (
     ListPartitionRouter,
 )
@@ -22,6 +25,7 @@ from airbyte_cdk.sources.declarative.partition_routers.substream_partition_route
 __all__ = [
     "AsyncJobPartitionRouter",
     "CartesianProductStreamSlicer",
+    "GroupingPartitionRouter",
     "ListPartitionRouter",
     "SinglePartitionRouter",
     "SubstreamPartitionRouter",

airbyte_cdk/sources/declarative/partition_routers/grouping_partition_router.py

@@ -0,0 +1,136 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+from dataclasses import dataclass
+from typing import Any, Iterable, Mapping, Optional
+
+from airbyte_cdk.sources.declarative.partition_routers.partition_router import PartitionRouter
+from airbyte_cdk.sources.types import Config, StreamSlice, StreamState
+
+
+@dataclass
+class GroupingPartitionRouter(PartitionRouter):
+    """
+    A partition router that groups partitions from an underlying partition router into batches of a specified size.
+    This is useful for APIs that support filtering by multiple partition keys in a single request.
+
+    Attributes:
+        group_size (int): The number of partitions to include in each group.
+        underlying_partition_router (PartitionRouter): The partition router whose output will be grouped.
+        deduplicate (bool): If True, ensures unique partitions within each group by removing duplicates based on the partition key.
+        config (Config): The connector configuration.
+        parameters (Mapping[str, Any]): Additional parameters for interpolation and configuration.
+    """
+
+    group_size: int
+    underlying_partition_router: PartitionRouter
+    config: Config
+    deduplicate: bool = True
+
+    def stream_slices(self) -> Iterable[StreamSlice]:
+        """
+        Lazily groups partitions from the underlying partition router into batches of size `group_size`.
+
+        This method processes partitions one at a time from the underlying router, maintaining a batch buffer.
+        When the buffer reaches `group_size` or the underlying router is exhausted, it yields a grouped slice.
+        If deduplication is enabled, it tracks seen partition keys to ensure uniqueness within the current batch.
+
+        Yields:
+            Iterable[StreamSlice]: An iterable of StreamSlice objects, where each slice contains a batch of partition values.
+        """
+        batch = []
+        seen_keys = set()
+
+        # Iterate over partitions lazily from the underlying router
+        for partition in self.underlying_partition_router.stream_slices():
+            # Extract the partition key (assuming single key-value pair, e.g., {"board_ids": value})
+            key = next(iter(partition.partition.values()), None)
+
+            # Skip duplicates if deduplication is enabled
+            if self.deduplicate and key in seen_keys:
+                continue
+
+            # Add partition to the batch
+            batch.append(partition)
+            if self.deduplicate:
+                seen_keys.add(key)
+
+            # Yield the batch when it reaches the group_size
+            if len(batch) == self.group_size:
+                yield self._create_grouped_slice(batch)
+                batch = []  # Reset the batch
+
+        # Yield any remaining partitions if the batch isn't empty
+        if batch:
+            yield self._create_grouped_slice(batch)
+
+    def _create_grouped_slice(self, batch: list[StreamSlice]) -> StreamSlice:
+        """
+        Creates a grouped StreamSlice from a batch of partitions, aggregating extra fields into a dictionary with list values.
+
+        Args:
+            batch (list[StreamSlice]): A list of StreamSlice objects to group.
+
+        Returns:
+            StreamSlice: A single StreamSlice with combined partition and extra field values.
+        """
+        # Combine partition values into a single dict with lists
+        grouped_partition = {
+            key: [p.partition.get(key) for p in batch] for key in batch[0].partition.keys()
+        }
+
+        # Aggregate extra fields into a dict with list values
+        extra_fields_dict = (
+            {
+                key: [p.extra_fields.get(key) for p in batch]
+                for key in set().union(*(p.extra_fields.keys() for p in batch if p.extra_fields))
+            }
+            if any(p.extra_fields for p in batch)
+            else {}
+        )
+        return StreamSlice(
+            partition=grouped_partition,
+            cursor_slice={},  # Cursor is managed by the underlying router or incremental sync
+            extra_fields=extra_fields_dict,
+        )
+
+    def get_request_params(
+        self,
+        stream_state: Optional[StreamState] = None,
+        stream_slice: Optional[StreamSlice] = None,
+        next_page_token: Optional[Mapping[str, Any]] = None,
+    ) -> Mapping[str, Any]:
+        return {}
+
+    def get_request_headers(
+        self,
+        stream_state: Optional[StreamState] = None,
+        stream_slice: Optional[StreamSlice] = None,
+        next_page_token: Optional[Mapping[str, Any]] = None,
+    ) -> Mapping[str, Any]:
+        return {}
+
+    def get_request_body_data(
+        self,
+        stream_state: Optional[StreamState] = None,
+        stream_slice: Optional[StreamSlice] = None,
+        next_page_token: Optional[Mapping[str, Any]] = None,
+    ) -> Mapping[str, Any]:
+        return {}
+
+    def get_request_body_json(
+        self,
+        stream_state: Optional[StreamState] = None,
+        stream_slice: Optional[StreamSlice] = None,
+        next_page_token: Optional[Mapping[str, Any]] = None,
+    ) -> Mapping[str, Any]:
+        return {}
+
+    def set_initial_state(self, stream_state: StreamState) -> None:
+        """Delegate state initialization to the underlying partition router."""
+        self.underlying_partition_router.set_initial_state(stream_state)
+
+    def get_stream_state(self) -> Optional[Mapping[str, StreamState]]:
+        """Delegate state retrieval to the underlying partition router."""
+        return self.underlying_partition_router.get_stream_state()

airbyte_cdk/sources/declarative/requesters/http_job_repository.py

@@ -23,7 +23,6 @@ from airbyte_cdk.sources.declarative.extractors.response_to_file_extractor impor
 )
 from airbyte_cdk.sources.declarative.requesters.requester import Requester
 from airbyte_cdk.sources.declarative.retrievers.simple_retriever import SimpleRetriever
-from airbyte_cdk.sources.http_logger import format_http_message
 from airbyte_cdk.sources.types import Record, StreamSlice
 from airbyte_cdk.utils import AirbyteTracedException
 
@@ -72,15 +71,7 @@ class AsyncHttpJobRepository(AsyncJobRepository):
         """
 
         polling_response: Optional[requests.Response] = self.polling_requester.send_request(
-            stream_slice=stream_slice,
-            log_formatter=lambda polling_response: format_http_message(
-                response=polling_response,
-                title="Async Job -- Polling",
-                description="Poll the status of the server-side async job.",
-                stream_name=None,
-                is_auxiliary=True,
-                type="ASYNC_POLL",
-            ),
+            stream_slice=stream_slice
         )
         if polling_response is None:
             raise AirbyteTracedException(
@@ -127,17 +118,8 @@ class AsyncHttpJobRepository(AsyncJobRepository):
         """
 
         response: Optional[requests.Response] = self.creation_requester.send_request(
-            stream_slice=stream_slice,
-            log_formatter=lambda response: format_http_message(
-                response=response,
-                title="Async Job -- Create",
-                description="Create the server-side async job.",
-                stream_name=None,
-                is_auxiliary=True,
-                type="ASYNC_CREATE",
-            ),
+            stream_slice=stream_slice
         )
-
         if not response:
             raise AirbyteTracedException(
                 internal_message="Always expect a response or an exception from creation_requester",
@@ -235,33 +217,13 @@ class AsyncHttpJobRepository(AsyncJobRepository):
         if not self.abort_requester:
             return
 
-        abort_response = self.abort_requester.send_request(
-            stream_slice=self._get_create_job_stream_slice(job),
-            log_formatter=lambda abort_response: format_http_message(
-                response=abort_response,
-                title="Async Job -- Abort",
-                description="Abort the running server-side async job.",
-                stream_name=None,
-                is_auxiliary=True,
-                type="ASYNC_ABORT",
-            ),
-        )
+        self.abort_requester.send_request(stream_slice=self._get_create_job_stream_slice(job))
 
     def delete(self, job: AsyncJob) -> None:
         if not self.delete_requester:
             return
 
-        delete_job_reponse = self.delete_requester.send_request(
-            stream_slice=self._get_create_job_stream_slice(job),
-            log_formatter=lambda delete_job_reponse: format_http_message(
-                response=delete_job_reponse,
-                title="Async Job -- Delete",
-                description="Delete the specified job from the list of Jobs.",
-                stream_name=None,
-                is_auxiliary=True,
-                type="ASYNC_DELETE",
-            ),
-        )
+        self.delete_requester.send_request(stream_slice=self._get_create_job_stream_slice(job))
         self._clean_up_job(job.api_job_id())
 
     def _clean_up_job(self, job_id: str) -> None:

airbyte_cdk/sources/declarative/retrievers/async_retriever.py

@@ -1,12 +1,13 @@
 # Copyright (c) 2024 Airbyte, Inc., all rights reserved.
 
 
-from dataclasses import InitVar, dataclass, field
+from dataclasses import InitVar, dataclass
 from typing import Any, Iterable, Mapping, Optional
 
 from typing_extensions import deprecated
 
 from airbyte_cdk.sources.declarative.async_job.job import AsyncJob
+from airbyte_cdk.sources.declarative.async_job.job_orchestrator import AsyncPartition
 from airbyte_cdk.sources.declarative.extractors.record_selector import RecordSelector
 from airbyte_cdk.sources.declarative.partition_routers.async_job_partition_router import (
     AsyncJobPartitionRouter,
@@ -15,7 +16,6 @@ from airbyte_cdk.sources.declarative.retrievers.retriever import Retriever
 from airbyte_cdk.sources.source import ExperimentalClassWarning
 from airbyte_cdk.sources.streams.core import StreamData
 from airbyte_cdk.sources.types import Config, StreamSlice, StreamState
-from airbyte_cdk.sources.utils.slice_logger import AlwaysLogSliceLogger
 
 
 @deprecated(
@@ -28,10 +28,6 @@ class AsyncRetriever(Retriever):
     parameters: InitVar[Mapping[str, Any]]
     record_selector: RecordSelector
     stream_slicer: AsyncJobPartitionRouter
-    slice_logger: AlwaysLogSliceLogger = field(
-        init=False,
-        default_factory=lambda: AlwaysLogSliceLogger(),
-    )
 
     def __post_init__(self, parameters: Mapping[str, Any]) -> None:
         self._parameters = parameters
@@ -79,16 +75,13 @@ class AsyncRetriever(Retriever):
         return stream_slice.extra_fields.get("jobs", []) if stream_slice else []
 
     def stream_slices(self) -> Iterable[Optional[StreamSlice]]:
-        yield from self.stream_slicer.stream_slices()
+        return self.stream_slicer.stream_slices()
 
     def read_records(
         self,
         records_schema: Mapping[str, Any],
         stream_slice: Optional[StreamSlice] = None,
     ) -> Iterable[StreamData]:
-        # emit the slice_descriptor log message, for connector builder TestRead
-        yield self.slice_logger.create_slice_log_message(stream_slice.cursor_slice)  # type: ignore
-
         stream_state: StreamState = self._get_stream_state()
         jobs: Iterable[AsyncJob] = self._validate_and_get_stream_slice_jobs(stream_slice)
         records: Iterable[Mapping[str, Any]] = self.stream_slicer.fetch_records(jobs)

airbyte_cdk/sources/http_logger.py

@@ -15,14 +15,11 @@ def format_http_message(
     description: str,
     stream_name: Optional[str],
     is_auxiliary: bool | None = None,
-    type: Optional[str] = None,
 ) -> LogMessage:
-    request_type: str = type if type else "HTTP"
     request = response.request
     log_message = {
         "http": {
             "title": title,
-            "type": request_type,
             "description": description,
             "request": {
                 "method": request.method,

airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py

@@ -396,7 +396,6 @@ class AbstractOauth2Authenticator(AuthBase):
                     "Obtains access token",
                     self._NO_STREAM_NAME,
                     is_auxiliary=True,
-                    type="AUTH",
                 ),
             )
 

{airbyte_cdk-6.36.2.dist-info → airbyte_cdk-6.37.0.dev1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 6.36.2
+Version: 6.37.0.dev1
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://airbyte.com
 License: MIT