airbyte-cdk 6.37.0.dev0__tar.gz → 6.37.0.dev1__tar.gz

{airbyte_cdk-6.37.0.dev0 → airbyte_cdk-6.37.0.dev1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 6.37.0.dev0
+Version: 6.37.0.dev1
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://airbyte.com
 License: MIT

{airbyte_cdk-6.37.0.dev0 → airbyte_cdk-6.37.0.dev1}/airbyte_cdk/connector_builder/models.py

@@ -21,6 +21,20 @@ class HttpRequest:
     body: Optional[str] = None
 
 
+@dataclass
+class StreamReadPages:
+    records: List[object]
+    request: Optional[HttpRequest] = None
+    response: Optional[HttpResponse] = None
+
+
+@dataclass
+class StreamReadSlices:
+    pages: List[StreamReadPages]
+    slice_descriptor: Optional[Dict[str, Any]]
+    state: Optional[List[Dict[str, Any]]] = None
+
+
 @dataclass
 class LogMessage:
     message: str
@@ -32,27 +46,11 @@ class LogMessage:
 @dataclass
 class AuxiliaryRequest:
     title: str
-    type: str
     description: str
     request: HttpRequest
     response: HttpResponse
 
 
-@dataclass
-class StreamReadPages:
-    records: List[object]
-    request: Optional[HttpRequest] = None
-    response: Optional[HttpResponse] = None
-
-
-@dataclass
-class StreamReadSlices:
-    pages: List[StreamReadPages]
-    slice_descriptor: Optional[Dict[str, Any]]
-    state: Optional[List[Dict[str, Any]]] = None
-    auxiliary_requests: Optional[List[AuxiliaryRequest]] = None
-
-
 @dataclass
 class StreamRead(object):
     logs: List[LogMessage]

{airbyte_cdk-6.37.0.dev0 → airbyte_cdk-6.37.0.dev1}/airbyte_cdk/connector_builder/test_reader/helpers.py

@@ -28,7 +28,7 @@ from airbyte_cdk.utils.schema_inferrer import (
     SchemaInferrer,
 )
 
-from .types import ASYNC_AUXILIARY_REQUEST_TYPES, LOG_MESSAGES_OUTPUT_TYPE
+from .types import LOG_MESSAGES_OUTPUT_TYPE
 
 # -------
 # Parsers
@@ -226,8 +226,7 @@ def should_close_page(
         at_least_one_page_in_group
         and is_log_message(message)
         and (
-            is_page_http_request(json_message)
-            or message.log.message.startswith(SliceLogger.SLICE_LOG_PREFIX)  # type: ignore[union-attr] # AirbyteMessage with MessageType.LOG has log.message
+            is_page_http_request(json_message) or message.log.message.startswith("slice:")  # type: ignore[union-attr] # AirbyteMessage with MessageType.LOG has log.message
         )
     )
 
@@ -331,10 +330,6 @@ def is_auxiliary_http_request(message: Optional[Dict[str, Any]]) -> bool:
     return is_http_log(message) and message.get("http", {}).get("is_auxiliary", False)
 
 
-def is_async_auxiliary_request(message: AuxiliaryRequest) -> bool:
-    return message.type in ASYNC_AUXILIARY_REQUEST_TYPES
-
-
 def is_log_message(message: AirbyteMessage) -> bool:
     """
     Determines whether the provided message is of type LOG.
@@ -418,7 +413,6 @@ def handle_current_slice(
     current_slice_pages: List[StreamReadPages],
     current_slice_descriptor: Optional[Dict[str, Any]] = None,
     latest_state_message: Optional[Dict[str, Any]] = None,
-    auxiliary_requests: Optional[List[AuxiliaryRequest]] = None,
 ) -> StreamReadSlices:
     """
     Handles the current slice by packaging its pages, descriptor, and state into a StreamReadSlices instance.
@@ -427,7 +421,6 @@ def handle_current_slice(
         current_slice_pages (List[StreamReadPages]): The pages to be included in the slice.
         current_slice_descriptor (Optional[Dict[str, Any]]): Descriptor for the current slice, optional.
         latest_state_message (Optional[Dict[str, Any]]): The latest state message, optional.
-        auxiliary_requests (Optional[List[AuxiliaryRequest]]): The auxiliary requests to include, optional.
 
     Returns:
         StreamReadSlices: An object containing the current slice's pages, descriptor, and state.
@@ -436,7 +429,6 @@ def handle_current_slice(
         pages=current_slice_pages,
         slice_descriptor=current_slice_descriptor,
         state=[latest_state_message] if latest_state_message else [],
-        auxiliary_requests=auxiliary_requests if auxiliary_requests else [],
     )
 
 
@@ -494,24 +486,29 @@ def handle_auxiliary_request(json_message: Dict[str, JsonType]) -> AuxiliaryRequ
     Raises:
         ValueError: If any of the "airbyte_cdk", "stream", or "http" fields is not a dictionary.
     """
+    airbyte_cdk = json_message.get("airbyte_cdk", {})
+
+    if not isinstance(airbyte_cdk, dict):
+        raise ValueError(
+            f"Expected airbyte_cdk to be a dict, got {airbyte_cdk} of type {type(airbyte_cdk)}"
+        )
+
+    stream = airbyte_cdk.get("stream", {})
 
-    airbyte_cdk = get_airbyte_cdk_from_message(json_message)
-    stream = get_stream_from_airbyte_cdk(airbyte_cdk)
-    title_prefix = get_auxiliary_request_title_prefix(stream)
-    http = get_http_property_from_message(json_message)
-    request_type = get_auxiliary_request_type(stream, http)
+    if not isinstance(stream, dict):
+        raise ValueError(f"Expected stream to be a dict, got {stream} of type {type(stream)}")
 
-    title = title_prefix + str(http.get("title", None))
-    description = str(http.get("description", None))
-    request = create_request_from_log_message(json_message)
-    response = create_response_from_log_message(json_message)
+    title_prefix = "Parent stream: " if stream.get("is_substream", False) else ""
+    http = json_message.get("http", {})
+
+    if not isinstance(http, dict):
+        raise ValueError(f"Expected http to be a dict, got {http} of type {type(http)}")
 
     return AuxiliaryRequest(
-        title=title,
-        type=request_type,
-        description=description,
-        request=request,
-        response=response,
+        title=title_prefix + str(http.get("title", None)),
+        description=str(http.get("description", None)),
+        request=create_request_from_log_message(json_message),
+        response=create_response_from_log_message(json_message),
     )
 
 
@@ -561,8 +558,7 @@ def handle_log_message(
         at_least_one_page_in_group,
         current_page_request,
         current_page_response,
-        auxiliary_request,
-        log_message,
+        auxiliary_request or log_message,
     )
 
 
@@ -593,97 +589,3 @@ def handle_record_message(
         datetime_format_inferrer.accumulate(message.record)  # type: ignore
 
     return records_count
-
-
-# -------
-# Reusable Getters
-# -------
-
-
-def get_airbyte_cdk_from_message(json_message: Dict[str, JsonType]) -> dict:  # type: ignore
-    """
-    Retrieves the "airbyte_cdk" dictionary from the provided JSON message.
-
-    This function validates that the extracted "airbyte_cdk" is of type dict,
-    raising a ValueError if the validation fails.
-
-    Parameters:
-        json_message (Dict[str, JsonType]): A dictionary representing the JSON message.
-
-    Returns:
-        dict: The "airbyte_cdk" dictionary extracted from the JSON message.
-
-    Raises:
-        ValueError: If the "airbyte_cdk" field is not a dictionary.
-    """
-    airbyte_cdk = json_message.get("airbyte_cdk", {})
-
-    if not isinstance(airbyte_cdk, dict):
-        raise ValueError(
-            f"Expected airbyte_cdk to be a dict, got {airbyte_cdk} of type {type(airbyte_cdk)}"
-        )
-
-    return airbyte_cdk
-
-
-def get_stream_from_airbyte_cdk(airbyte_cdk: dict) -> dict:  # type: ignore
-    """
-    Retrieves the "stream" dictionary from the provided "airbyte_cdk" dictionary.
-
-    This function ensures that the extracted "stream" is of type dict,
-    raising a ValueError if the validation fails.
-
-    Parameters:
-        airbyte_cdk (dict): The dictionary representing the Airbyte CDK data.
-
-    Returns:
-        dict: The "stream" dictionary extracted from the Airbyte CDK data.
-
-    Raises:
-        ValueError: If the "stream" field is not a dictionary.
-    """
-
-    stream = airbyte_cdk.get("stream", {})
-
-    if not isinstance(stream, dict):
-        raise ValueError(f"Expected stream to be a dict, got {stream} of type {type(stream)}")
-
-    return stream
-
-
-def get_auxiliary_request_title_prefix(stream: dict) -> str:  # type: ignore
-    """
-    Generates a title prefix based on the stream type.
-    """
-    return "Parent stream: " if stream.get("is_substream", False) else ""
-
-
-def get_http_property_from_message(json_message: Dict[str, JsonType]) -> dict:  # type: ignore
-    """
-    Retrieves the "http" dictionary from the provided JSON message.
-
-    This function validates that the extracted "http" is of type dict,
-    raising a ValueError if the validation fails.
-
-    Parameters:
-        json_message (Dict[str, JsonType]): A dictionary representing the JSON message.
-
-    Returns:
-        dict: The "http" dictionary extracted from the JSON message.
-
-    Raises:
-        ValueError: If the "http" field is not a dictionary.
-    """
-    http = json_message.get("http", {})
-
-    if not isinstance(http, dict):
-        raise ValueError(f"Expected http to be a dict, got {http} of type {type(http)}")
-
-    return http
-
-
-def get_auxiliary_request_type(stream: dict, http: dict) -> str:  # type: ignore
-    """
-    Determines the type of the auxiliary request based on the stream and HTTP properties.
-    """
-    return "PARENT_STREAM" if stream.get("is_substream", False) else str(http.get("type", None))

{airbyte_cdk-6.37.0.dev0 → airbyte_cdk-6.37.0.dev1}/airbyte_cdk/connector_builder/test_reader/message_grouper.py

@@ -6,7 +6,6 @@
 from typing import Any, Dict, Iterator, List, Mapping, Optional
 
 from airbyte_cdk.connector_builder.models import (
-    AuxiliaryRequest,
     HttpRequest,
     HttpResponse,
     StreamReadPages,
@@ -25,7 +24,6 @@ from .helpers import (
     handle_current_slice,
     handle_log_message,
     handle_record_message,
-    is_async_auxiliary_request,
     is_config_update_message,
     is_log_message,
     is_record_message,
@@ -91,7 +89,6 @@ def get_message_groups(
     current_page_request: Optional[HttpRequest] = None
     current_page_response: Optional[HttpResponse] = None
     latest_state_message: Optional[Dict[str, Any]] = None
-    slice_auxiliary_requests: List[AuxiliaryRequest] = []
 
     while records_count < limit and (message := next(messages, None)):
         json_message = airbyte_message_to_json(message)
@@ -109,7 +106,6 @@ def get_message_groups(
                 current_slice_pages,
                 current_slice_descriptor,
                 latest_state_message,
-                slice_auxiliary_requests,
             )
             current_slice_descriptor = parse_slice_description(message.log.message)  # type: ignore
             current_slice_pages = []
@@ -122,8 +118,7 @@ def get_message_groups(
                 at_least_one_page_in_group,
                 current_page_request,
                 current_page_response,
-                auxiliary_request,
-                log_message,
+                log_or_auxiliary_request,
             ) = handle_log_message(
                 message,
                 json_message,
@@ -131,15 +126,8 @@ def get_message_groups(
                 current_page_request,
                 current_page_response,
             )
-
-            if auxiliary_request:
-                if is_async_auxiliary_request(auxiliary_request):
-                    slice_auxiliary_requests.append(auxiliary_request)
-                else:
-                    yield auxiliary_request
-
-            if log_message:
-                yield log_message
+            if log_or_auxiliary_request:
+                yield log_or_auxiliary_request
         elif is_trace_with_error(message):
             if message.trace is not None:
                 yield message.trace
@@ -169,5 +157,4 @@ def get_message_groups(
                 current_slice_pages,
                 current_slice_descriptor,
                 latest_state_message,
-                slice_auxiliary_requests,
             )

{airbyte_cdk-6.37.0.dev0 → airbyte_cdk-6.37.0.dev1}/airbyte_cdk/connector_builder/test_reader/types.py

@@ -71,13 +71,5 @@ LOG_MESSAGES_OUTPUT_TYPE = tuple[
     bool,
     HttpRequest | None,
     HttpResponse | None,
-    AuxiliaryRequest | None,
-    AirbyteLogMessage | None,
-]
-
-ASYNC_AUXILIARY_REQUEST_TYPES = [
-    "ASYNC_CREATE",
-    "ASYNC_POLL",
-    "ASYNC_ABORT",
-    "ASYNC_DELETE",
+    AuxiliaryRequest | AirbyteLogMessage | None,
 ]

{airbyte_cdk-6.37.0.dev0 → airbyte_cdk-6.37.0.dev1}/airbyte_cdk/entrypoint.py

@@ -37,8 +37,8 @@ from airbyte_cdk.sources import Source
 from airbyte_cdk.sources.connector_state_manager import HashableStreamDescriptor
 from airbyte_cdk.sources.utils.schema_helpers import check_config_against_spec_or_exit, split_config
 
-from airbyte_cdk.utils import PrintBuffer, is_cloud_environment, message_utils  # add PrintBuffer back once fixed
-# from airbyte_cdk.utils import is_cloud_environment, message_utils
+# from airbyte_cdk.utils import PrintBuffer, is_cloud_environment, message_utils  # add PrintBuffer back once fixed
+from airbyte_cdk.utils import is_cloud_environment, message_utils
 from airbyte_cdk.utils.airbyte_secrets_utils import get_secrets, update_secrets
 from airbyte_cdk.utils.constants import ENV_REQUEST_CACHE_PATH
 from airbyte_cdk.utils.traced_exception import AirbyteTracedException
@@ -337,11 +337,11 @@ def launch(source: Source, args: List[str]) -> None:
     parsed_args = source_entrypoint.parse_args(args)
     # temporarily removes the PrintBuffer because we're seeing weird print behavior for concurrent syncs
     # Refer to: https://github.com/airbytehq/oncall/issues/6235
-    with PrintBuffer():
-        for message in source_entrypoint.run(parsed_args):
-            # simply printing is creating issues for concurrent CDK as Python uses different two instructions to print: one for the message and
-            # the other for the break line. Adding `\n` to the message ensure that both are printed at the same time
-            print(f"{message}\n", end="", flush=True)
+    # with PrintBuffer():
+    for message in source_entrypoint.run(parsed_args):
+        # simply printing is creating issues for concurrent CDK as Python uses different two instructions to print: one for the message and
+        # the other for the break line. Adding `\n` to the message ensure that both are printed at the same time
+        print(f"{message}\n", end="", flush=True)
 
 
 def _init_internal_request_filter() -> None:

{airbyte_cdk-6.37.0.dev0 → airbyte_cdk-6.37.0.dev1}/airbyte_cdk/sources/declarative/auth/token_provider.py

@@ -58,7 +58,6 @@ class SessionTokenProvider(TokenProvider):
                 "Obtains session token",
                 None,
                 is_auxiliary=True,
-                type="AUTH",
             ),
         )
         if response is None:

{airbyte_cdk-6.37.0.dev0 → airbyte_cdk-6.37.0.dev1}/airbyte_cdk/sources/declarative/concurrent_declarative_source.py

@@ -44,7 +44,6 @@ from airbyte_cdk.sources.declarative.types import ConnectionDefinition
 from airbyte_cdk.sources.source import TState
 from airbyte_cdk.sources.streams import Stream
 from airbyte_cdk.sources.streams.concurrent.abstract_stream import AbstractStream
-from airbyte_cdk.sources.streams.concurrent.abstract_stream_facade import AbstractStreamFacade
 from airbyte_cdk.sources.streams.concurrent.availability_strategy import (
     AlwaysAvailableAvailabilityStrategy,
 )
@@ -119,12 +118,6 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
             message_repository=self.message_repository,
         )
 
-    # TODO: Remove this. This property is necessary to safely migrate Stripe during the transition state.
-    @property
-    def is_partially_declarative(self) -> bool:
-        """This flag used to avoid unexpected AbstractStreamFacade processing as concurrent streams."""
-        return False
-
     def read(
         self,
         logger: logging.Logger,
@@ -376,14 +369,6 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
                     )
                 else:
                     synchronous_streams.append(declarative_stream)
-            # TODO: Remove this. This check is necessary to safely migrate Stripe during the transition state.
-            # Condition below needs to ensure that concurrent support is not lost for sources that already support
-            # it before migration, but now are only partially migrated to declarative implementation (e.g., Stripe).
-            elif (
-                isinstance(declarative_stream, AbstractStreamFacade)
-                and self.is_partially_declarative
-            ):
-                concurrent_streams.append(declarative_stream.get_underlying_stream())
             else:
                 synchronous_streams.append(declarative_stream)
 

{airbyte_cdk-6.37.0.dev0 → airbyte_cdk-6.37.0.dev1}/airbyte_cdk/sources/declarative/declarative_component_schema.yaml

@@ -1490,11 +1490,7 @@ definitions:
       limit:
         title: Limit
         description: The maximum number of calls allowed within the interval.
-        anyOf:
-          - type: integer
-          - type: string
-        interpolation_context:
-          - config
+        type: integer
       interval:
         title: Interval
         description: The time interval for the rate limit.
@@ -3134,12 +3130,14 @@ definitions:
           - "$ref": "#/definitions/CustomPartitionRouter"
           - "$ref": "#/definitions/ListPartitionRouter"
           - "$ref": "#/definitions/SubstreamPartitionRouter"
+          - "$ref": "#/definitions/GroupingPartitionRouter"
           - type: array
             items:
               anyOf:
                 - "$ref": "#/definitions/CustomPartitionRouter"
                 - "$ref": "#/definitions/ListPartitionRouter"
                 - "$ref": "#/definitions/SubstreamPartitionRouter"
+                - "$ref": "#/definitions/GroupingPartitionRouter"
       decoder:
         title: Decoder
         description: Component decoding the response so records can be extracted.
@@ -3294,12 +3292,14 @@ definitions:
           - "$ref": "#/definitions/CustomPartitionRouter"
           - "$ref": "#/definitions/ListPartitionRouter"
           - "$ref": "#/definitions/SubstreamPartitionRouter"
+          - "$ref": "#/definitions/GroupingPartitionRouter"
           - type: array
             items:
               anyOf:
                 - "$ref": "#/definitions/CustomPartitionRouter"
                 - "$ref": "#/definitions/ListPartitionRouter"
                 - "$ref": "#/definitions/SubstreamPartitionRouter"
+                - "$ref": "#/definitions/GroupingPartitionRouter"
       decoder:
         title: Decoder
         description: Component decoding the response so records can be extracted.
@@ -3416,6 +3416,44 @@ definitions:
       $parameters:
         type: object
         additionalProperties: true
+  GroupingPartitionRouter:
+    title: Grouping Partition Router
+    description: >
+      A decorator on top of a partition router that groups partitions into batches of a specified size.
+      This is useful for APIs that support filtering by multiple partition keys in a single request.
+      Note that per-partition incremental syncs may not work as expected because the grouping
+      of partitions might change between syncs, potentially leading to inconsistent state tracking.
+    type: object
+    required:
+      - type
+      - group_size
+      - underlying_partition_router
+    properties:
+      type:
+        type: string
+        enum: [GroupingPartitionRouter]
+      group_size:
+        title: Group Size
+        description: The number of partitions to include in each group. This determines how many partition values are batched together in a single slice.
+        type: integer
+        examples:
+          - 10
+          - 50
+      underlying_partition_router:
+        title: Underlying Partition Router
+        description: The partition router whose output will be grouped. This can be any valid partition router component.
+        anyOf:
+          - "$ref": "#/definitions/CustomPartitionRouter"
+          - "$ref": "#/definitions/ListPartitionRouter"
+          - "$ref": "#/definitions/SubstreamPartitionRouter"
+      deduplicate:
+        title: Deduplicate Partitions
+        description: If true, ensures that partitions are unique within each group by removing duplicates based on the partition key.
+        type: boolean
+        default: true
+      $parameters:
+        type: object
+        additionalProperties: true
   WaitUntilTimeFromHeader:
     title: Wait Until Time Defined In Response Header
     description: Extract time at which we can retry the request from response header and wait for the difference between now and that time.

{airbyte_cdk-6.37.0.dev0 → airbyte_cdk-6.37.0.dev1}/airbyte_cdk/sources/declarative/decoders/composite_raw_decoder.py

@@ -107,16 +107,6 @@ class CsvParser(Parser):
     encoding: Optional[str] = "utf-8"
     delimiter: Optional[str] = ","
 
-    def _get_delimiter(self) -> Optional[str]:
-        """
-        Get delimiter from the configuration. Check for the escape character and decode it.
-        """
-        if self.delimiter is not None:
-            if self.delimiter.startswith("\\"):
-                self.delimiter = self.delimiter.encode("utf-8").decode("unicode_escape")
-
-        return self.delimiter
-
     def parse(
         self,
         data: BufferedIOBase,
@@ -125,9 +115,8 @@ class CsvParser(Parser):
         Parse CSV data from decompressed bytes.
         """
         text_data = TextIOWrapper(data, encoding=self.encoding)  # type: ignore
-        reader = csv.DictReader(text_data, delimiter=self._get_delimiter() or ",")
-        for row in reader:
-            yield row
+        reader = csv.DictReader(text_data, delimiter=self.delimiter or ",")
+        yield from reader
 
 
 @dataclass