airbyte-cdk 6.34.1.dev0__py3-none-any.whl → 6.34.1.dev1__py3-none-any.whl

airbyte_cdk/sources/streams/permissions/identities_stream.py

@@ -0,0 +1,75 @@
+#
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+#
+
+import traceback
+from abc import ABC, abstractmethod
+from typing import Any, Dict, Iterable, List, Mapping, MutableMapping, Optional
+
+from airbyte_protocol_dataclasses.models import SyncMode
+
+from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, Level
+from airbyte_cdk.models import Type as MessageType
+from airbyte_cdk.sources.streams import Stream
+from airbyte_cdk.sources.streams.checkpoint import Cursor
+from airbyte_cdk.sources.utils.record_helper import stream_data_to_airbyte_message
+from airbyte_cdk.utils.traced_exception import AirbyteTracedException
+
+
+class IdentitiesStream(Stream, ABC):
+    """
+    The identities stream. A full refresh stream to sync identities from a certain domain.
+    The load_identity_groups method manage the logic to get such data.
+    """
+
+    IDENTITIES_STREAM_NAME = "identities"
+
+    is_resumable = False
+
+    def __init__(self) -> None:
+        super().__init__()
+        self._cursor: MutableMapping[str, Any] = {}
+
+    @property
+    def state(self) -> MutableMapping[str, Any]:
+        return self._cursor
+
+    @state.setter
+    def state(self, value: MutableMapping[str, Any]) -> None:
+        """State setter, accept state serialized by state getter."""
+        self._cursor = value
+
+    def read_records(
+        self,
+        sync_mode: SyncMode,
+        cursor_field: Optional[List[str]] = None,
+        stream_slice: Optional[Mapping[str, Any]] = None,
+        stream_state: Optional[Mapping[str, Any]] = None,
+    ) -> Iterable[Mapping[str, Any] | AirbyteMessage]:
+        try:
+            identity_groups = self.load_identity_groups()
+            for record in identity_groups:
+                yield stream_data_to_airbyte_message(self.name, record)
+        except AirbyteTracedException as exc:
+            # Re-raise the exception to stop the whole sync immediately as this is a fatal error
+            raise exc
+        except Exception as e:
+            yield AirbyteMessage(
+                type=MessageType.LOG,
+                log=AirbyteLogMessage(
+                    level=Level.ERROR,
+                    message=f"Error trying to read identities: {e} stream={self.name}",
+                    stack_trace=traceback.format_exc(),
+                ),
+            )
+
+    @abstractmethod
+    def load_identity_groups(self) -> Iterable[Dict[str, Any]]:
+        raise NotImplementedError("Implement this method to read identity records")
+
+    @property
+    def name(self) -> str:
+        return self.IDENTITIES_STREAM_NAME
+
+    def get_cursor(self) -> Optional[Cursor]:
+        return None

airbyte_cdk/test/mock_http/mocker.py

@@ -17,6 +17,7 @@ class SupportedHttpMethods(str, Enum):
     GET = "get"
     PATCH = "patch"
     POST = "post"
+    PUT = "put"
     DELETE = "delete"
 
 
@@ -77,7 +78,7 @@ class HttpMocker(contextlib.ContextDecorator):
             additional_matcher=self._matches_wrapper(matcher),
             response_list=[
                 {
-                    "text": response.body,
+                    self._get_body_field(response): response.body,
                     "status_code": response.status_code,
                     "headers": response.headers,
                 }
@@ -85,6 +86,10 @@ class HttpMocker(contextlib.ContextDecorator):
             ],
         )
 
+    @staticmethod
+    def _get_body_field(response: HttpResponse) -> str:
+        return "text" if isinstance(response.body, str) else "content"
+
     def get(self, request: HttpRequest, responses: Union[HttpResponse, List[HttpResponse]]) -> None:
         self._mock_request_method(SupportedHttpMethods.GET, request, responses)
 
@@ -98,6 +103,9 @@ class HttpMocker(contextlib.ContextDecorator):
     ) -> None:
         self._mock_request_method(SupportedHttpMethods.POST, request, responses)
 
+    def put(self, request: HttpRequest, responses: Union[HttpResponse, List[HttpResponse]]) -> None:
+        self._mock_request_method(SupportedHttpMethods.PUT, request, responses)
+
     def delete(
         self, request: HttpRequest, responses: Union[HttpResponse, List[HttpResponse]]
     ) -> None:

airbyte_cdk/test/mock_http/response.py

@@ -1,19 +1,22 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 
 from types import MappingProxyType
-from typing import Mapping
+from typing import Mapping, Union
 
 
 class HttpResponse:
     def __init__(
-        self, body: str, status_code: int = 200, headers: Mapping[str, str] = MappingProxyType({})
+        self,
+        body: Union[str, bytes],
+        status_code: int = 200,
+        headers: Mapping[str, str] = MappingProxyType({}),
     ):
         self._body = body
         self._status_code = status_code
         self._headers = headers
 
     @property
-    def body(self) -> str:
+    def body(self) -> Union[str, bytes]:
         return self._body
 
     @property

airbyte_cdk/utils/datetime_helpers.py

@@ -76,8 +76,8 @@ from airbyte_cdk.utils.datetime_helpers import ab_datetime_try_parse
 assert ab_datetime_try_parse("2023-03-14T15:09:26Z")       # Basic UTC format
 assert ab_datetime_try_parse("2023-03-14T15:09:26-04:00")  # With timezone offset
 assert ab_datetime_try_parse("2023-03-14T15:09:26+00:00")  # With explicit UTC offset
-assert not ab_datetime_try_parse("2023-03-14 15:09:26Z")   # Invalid: missing T delimiter
-assert not ab_datetime_try_parse("foo")                     # Invalid: not a datetime
+assert ab_datetime_try_parse("2023-03-14 15:09:26Z")       # Missing T delimiter but still parsable
+assert not ab_datetime_try_parse("foo")                    # Invalid: not parsable, returns `None`
 ```
 """
 
@@ -138,6 +138,14 @@ class AirbyteDateTime(datetime):
             dt.tzinfo or timezone.utc,
         )
 
+    def to_datetime(self) -> datetime:
+        """Converts this AirbyteDateTime to a standard datetime object.
+
+        Today, this just returns `self` because AirbyteDateTime is a subclass of `datetime`.
+        In the future, we may modify our internal representation to use a different base class.
+        """
+        return self
+
     def __str__(self) -> str:
         """Returns the datetime in ISO8601/RFC3339 format with 'T' delimiter.
 
@@ -148,12 +156,7 @@ class AirbyteDateTime(datetime):
             str: ISO8601/RFC3339 formatted string.
         """
         aware_self = self if self.tzinfo else self.replace(tzinfo=timezone.utc)
-        base = self.strftime("%Y-%m-%dT%H:%M:%S")
-        if self.microsecond:
-            base = f"{base}.{self.microsecond:06d}"
-        # Format timezone as ±HH:MM
-        offset = aware_self.strftime("%z")
-        return f"{base}{offset[:3]}:{offset[3:]}"
+        return aware_self.isoformat(sep="T", timespec="auto")
 
     def __repr__(self) -> str:
         """Returns the same string representation as __str__ for consistency.
@@ -358,15 +361,15 @@ def ab_datetime_now() -> AirbyteDateTime:
 def ab_datetime_parse(dt_str: str | int) -> AirbyteDateTime:
     """Parses a datetime string or timestamp into an AirbyteDateTime with timezone awareness.
 
-    Previously named: parse()
+    This implementation is as flexible as possible to handle various datetime formats.
+    Always returns a timezone-aware datetime (defaults to UTC if no timezone specified).
 
     Handles:
-        - ISO8601/RFC3339 format strings (with 'T' delimiter)
+        - ISO8601/RFC3339 format strings (with ' ' or 'T' delimiter)
         - Unix timestamps (as integers or strings)
         - Date-only strings (YYYY-MM-DD)
         - Timezone-aware formats (+00:00 for UTC, or ±HH:MM offset)
-
-    Always returns a timezone-aware datetime (defaults to UTC if no timezone specified).
+        - Anything that can be parsed by `dateutil.parser.parse()`
 
     Args:
         dt_str: A datetime string in ISO8601/RFC3339 format, Unix timestamp (int/str),
@@ -416,15 +419,16 @@ def ab_datetime_parse(dt_str: str | int) -> AirbyteDateTime:
             except (ValueError, TypeError):
                 raise ValueError(f"Invalid date format: {dt_str}")
 
-        # Validate datetime format
-        if "/" in dt_str or " " in dt_str or "GMT" in dt_str:
-            raise ValueError(f"Could not parse datetime string: {dt_str}")
+        # Reject time-only strings without date
+        if ":" in dt_str and dt_str.count("-") < 2 and dt_str.count("/") < 2:
+            raise ValueError(f"Missing date part in datetime string: {dt_str}")
 
         # Try parsing with dateutil for timezone handling
         try:
             parsed = parser.parse(dt_str)
             if parsed.tzinfo is None:
                 parsed = parsed.replace(tzinfo=timezone.utc)
+
             return AirbyteDateTime.from_datetime(parsed)
         except (ValueError, TypeError):
             raise ValueError(f"Could not parse datetime string: {dt_str}")
@@ -438,7 +442,29 @@ def ab_datetime_parse(dt_str: str | int) -> AirbyteDateTime:
         raise ValueError(f"Could not parse datetime string: {dt_str}")
 
 
-def ab_datetime_format(dt: Union[datetime, AirbyteDateTime]) -> str:
+def ab_datetime_try_parse(dt_str: str) -> AirbyteDateTime | None:
+    """Try to parse the input as a datetime, failing gracefully instead of raising an exception.
+
+    This is a thin wrapper around `ab_datetime_parse()` that catches parsing errors and
+    returns `None` instead of raising an exception.
+    The implementation is as flexible as possible to handle various datetime formats.
+    Always returns a timezone-aware datetime (defaults to `UTC` if no timezone specified).
+
+    Example:
+        >>> ab_datetime_try_parse("2023-03-14T15:09:26Z")  # Returns AirbyteDateTime
+        >>> ab_datetime_try_parse("2023-03-14 15:09:26Z")  # Missing 'T' delimiter still parsable
+        >>> ab_datetime_try_parse("2023-03-14")            # Returns midnight UTC time
+    """
+    try:
+        return ab_datetime_parse(dt_str)
+    except (ValueError, TypeError):
+        return None
+
+
+def ab_datetime_format(
+    dt: Union[datetime, AirbyteDateTime],
+    format: str | None = None,
+) -> str:
     """Formats a datetime object as an ISO8601/RFC3339 string with 'T' delimiter and timezone.
 
     Previously named: format()
@@ -449,6 +475,8 @@ def ab_datetime_format(dt: Union[datetime, AirbyteDateTime]) -> str:
 
     Args:
         dt: Any datetime object to format.
+        format: Optional format string. If provided, calls `strftime()` with this format.
+            Otherwise, uses the default ISO8601/RFC3339 format, adapted for available precision.
 
     Returns:
         str: ISO8601/RFC3339 formatted datetime string.
@@ -464,54 +492,8 @@ def ab_datetime_format(dt: Union[datetime, AirbyteDateTime]) -> str:
     if dt.tzinfo is None:
         dt = dt.replace(tzinfo=timezone.utc)
 
-    # Format with consistent timezone representation
-    base = dt.strftime("%Y-%m-%dT%H:%M:%S")
-    if dt.microsecond:
-        base = f"{base}.{dt.microsecond:06d}"
-    offset = dt.strftime("%z")
-    return f"{base}{offset[:3]}:{offset[3:]}"
-
-
-def ab_datetime_try_parse(dt_str: str) -> AirbyteDateTime | None:
-    """Try to parse the input string as an ISO8601/RFC3339 datetime, failing gracefully instead of raising an exception.
-
-    Requires strict ISO8601/RFC3339 format with:
-    - 'T' delimiter between date and time components
-    - Valid timezone (Z for UTC or ±HH:MM offset)
-    - Complete datetime representation (date and time)
+    if format:
+        return dt.strftime(format)
 
-    Returns None for any non-compliant formats including:
-    - Space-delimited datetimes
-    - Date-only strings
-    - Missing timezone
-    - Invalid timezone format
-    - Wrong date/time separators
-
-    Example:
-        >>> ab_datetime_try_parse("2023-03-14T15:09:26Z")  # Returns AirbyteDateTime
-        >>> ab_datetime_try_parse("2023-03-14 15:09:26Z")  # Returns None (invalid format)
-        >>> ab_datetime_try_parse("2023-03-14")  # Returns None (missing time and timezone)
-    """
-    if not isinstance(dt_str, str):
-        return None
-    try:
-        # Validate format before parsing
-        if "T" not in dt_str:
-            return None
-        if not any(x in dt_str for x in ["Z", "+", "-"]):
-            return None
-        if "/" in dt_str or " " in dt_str or "GMT" in dt_str:
-            return None
-
-        # Try parsing with dateutil
-        parsed = parser.parse(dt_str)
-        if parsed.tzinfo is None:
-            return None
-
-        # Validate time components
-        if not (0 <= parsed.hour <= 23 and 0 <= parsed.minute <= 59 and 0 <= parsed.second <= 59):
-            return None
-
-        return AirbyteDateTime.from_datetime(parsed)
-    except (ValueError, TypeError):
-        return None
+    # Format with consistent timezone representation and "T" delimiter
+    return dt.isoformat(sep="T", timespec="auto")

airbyte_cdk/utils/mapping_helpers.py

@@ -3,43 +3,143 @@
 #
 
 
-from typing import Any, List, Mapping, Optional, Set, Union
+import copy
+from typing import Any, Dict, List, Mapping, Optional, Union
+
+from airbyte_cdk.sources.declarative.requesters.request_option import (
+    RequestOption,
+    RequestOptionType,
+)
+from airbyte_cdk.sources.types import Config
+
+
+def _merge_mappings(
+    target: Dict[str, Any],
+    source: Mapping[str, Any],
+    path: Optional[List[str]] = None,
+    allow_same_value_merge: bool = False,
+) -> None:
+    """
+    Recursively merge two dictionaries, raising an error if there are any conflicts.
+    For body_json requests (allow_same_value_merge=True), a conflict occurs only when the same path has different values.
+    For other request types (allow_same_value_merge=False), any duplicate key is a conflict, regardless of value.
+
+    Args:
+        target: The dictionary to merge into
+        source: The dictionary to merge from
+        path: The current path in the nested structure (for error messages)
+        allow_same_value_merge: Whether to allow merging the same value into the same key. Set to false by default, should only be true for body_json injections
+    """
+    path = path or []
+    for key, source_value in source.items():
+        current_path = path + [str(key)]
+
+        if key in target:
+            target_value = target[key]
+            if isinstance(target_value, dict) and isinstance(source_value, dict):
+                # Only body_json supports nested_structures
+                if not allow_same_value_merge:
+                    raise ValueError(
+                        f"Request body collision, duplicate keys detected at key path: {'.'.join(current_path)}. Please ensure that all keys in the request are unique."
+                    )
+                # If both are dictionaries, recursively merge them
+                _merge_mappings(target_value, source_value, current_path, allow_same_value_merge)
+
+            elif not allow_same_value_merge or target_value != source_value:
+                # If same key has different values, that's a conflict
+                raise ValueError(
+                    f"Request body collision, duplicate keys detected at key path: {'.'.join(current_path)}. Please ensure that all keys in the request are unique."
+                )
+        else:
+            # No conflict, just copy the value (using deepcopy for nested structures)
+            target[key] = copy.deepcopy(source_value)
 
 
 def combine_mappings(
     mappings: List[Optional[Union[Mapping[str, Any], str]]],
+    allow_same_value_merge: bool = False,
 ) -> Union[Mapping[str, Any], str]:
     """
-    Combine multiple mappings into a single mapping. If any of the mappings are a string, return
-    that string. Raise errors in the following cases:
-    * If there are duplicate keys across mappings
-    * If there are multiple string mappings
-    * If there are multiple mappings containing keys and one of them is a string
+    Combine multiple mappings into a single mapping.
+
+    For body_json requests (allow_same_value_merge=True):
+        - Supports nested structures (e.g., {"data": {"user": {"id": 1}}})
+        - Allows duplicate keys if their values match
+        - Raises error if same path has different values
+
+    For other request types (allow_same_value_merge=False):
+        - Only supports flat structures
+        - Any duplicate key raises an error, regardless of value
+
+    Args:
+        mappings: List of mappings to combine
+        allow_same_value_merge: Whether to allow duplicate keys with matching values.
+                              Should only be True for body_json requests.
+
+    Returns:
+        A single mapping combining all inputs, or a string if there is exactly one
+        string mapping and no other non-empty mappings.
+
+    Raises:
+        ValueError: If there are:
+            - Multiple string mappings
+            - Both a string mapping and non-empty dictionary mappings
+            - Conflicting keys/paths based on allow_same_value_merge setting
     """
-    all_keys: List[Set[str]] = []
-    for part in mappings:
-        if part is None:
-            continue
-        keys = set(part.keys()) if not isinstance(part, str) else set()
-        all_keys.append(keys)
+    if not mappings:
+        return {}
 
-    string_options = sum(isinstance(mapping, str) for mapping in mappings)
-    # If more than one mapping is a string, raise a ValueError
+    # Count how many string options we have, ignoring None values
+    string_options = sum(isinstance(mapping, str) for mapping in mappings if mapping is not None)
     if string_options > 1:
         raise ValueError("Cannot combine multiple string options")
 
-    if string_options == 1 and sum(len(keys) for keys in all_keys) > 0:
-        raise ValueError("Cannot combine multiple options if one is a string")
+    # Filter out None values and empty mappings
+    non_empty_mappings = [
+        m for m in mappings if m is not None and not (isinstance(m, Mapping) and not m)
+    ]
+
+    # If there is only one string option and no other non-empty mappings, return it
+    if string_options == 1:
+        if len(non_empty_mappings) > 1:
+            raise ValueError("Cannot combine multiple options if one is a string")
+        return next(m for m in non_empty_mappings if isinstance(m, str))
+
+    # Start with an empty result and merge each mapping into it
+    result: Dict[str, Any] = {}
+    for mapping in non_empty_mappings:
+        if mapping and isinstance(mapping, Mapping):
+            _merge_mappings(result, mapping, allow_same_value_merge=allow_same_value_merge)
+
+    return result
 
-    # If any mapping is a string, return it
-    for mapping in mappings:
-        if isinstance(mapping, str):
-            return mapping
 
-    # If there are duplicate keys across mappings, raise a ValueError
-    intersection = set().union(*all_keys)
-    if len(intersection) < sum(len(keys) for keys in all_keys):
-        raise ValueError(f"Duplicate keys found: {intersection}")
+def _validate_component_request_option_paths(
+    config: Config, *request_options: Optional[RequestOption]
+) -> None:
+    """
+    Validates that a component with multiple request options does not have conflicting paths.
+    Uses dummy values for validation since actual values might not be available at init time.
+    """
+    grouped_options: Dict[RequestOptionType, List[RequestOption]] = {}
+    for option in request_options:
+        if option:
+            grouped_options.setdefault(option.inject_into, []).append(option)
+
+    for inject_type, options in grouped_options.items():
+        if len(options) <= 1:
+            continue
+
+        option_dicts: List[Optional[Union[Mapping[str, Any], str]]] = []
+        for i, option in enumerate(options):
+            option_dict: Dict[str, Any] = {}
+            # Use indexed dummy values to ensure we catch conflicts
+            option.inject_into_request(option_dict, f"dummy_value_{i}", config)
+            option_dicts.append(option_dict)
 
-    # Return the combined mappings
-    return {key: value for mapping in mappings if mapping for key, value in mapping.items()}  # type: ignore # mapping can't be string here
+        try:
+            combine_mappings(
+                option_dicts, allow_same_value_merge=(inject_type == RequestOptionType.body_json)
+            )
+        except ValueError as error:
+            raise ValueError(error)

{airbyte_cdk-6.34.1.dev0.dist-info → airbyte_cdk-6.34.1.dev1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 6.34.1.dev0
+Version: 6.34.1.dev1
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://airbyte.com
 License: MIT