airbyte-cdk 6.34.0.dev0__py3-none-any.whl → 6.34.0.dev2__py3-none-any.whl

airbyte_cdk/sources/file_based/file_based_stream_reader.py

@@ -13,6 +13,11 @@ from typing import Any, Dict, Iterable, List, Optional, Set
 from wcmatch.glob import GLOBSTAR, globmatch
 
 from airbyte_cdk.sources.file_based.config.abstract_file_based_spec import AbstractFileBasedSpec
+from airbyte_cdk.sources.file_based.config.validate_config_transfer_modes import (
+    include_identities_stream,
+    preserve_directory_structure,
+    use_file_transfer,
+)
 from airbyte_cdk.sources.file_based.remote_file import RemoteFile
 
 
@@ -128,24 +133,20 @@ class AbstractFileBasedStreamReader(ABC):
 
     def use_file_transfer(self) -> bool:
         if self.config:
-            use_file_transfer = (
-                hasattr(self.config.delivery_method, "delivery_type")
-                and self.config.delivery_method.delivery_type == "use_file_transfer"
-            )
-            return use_file_transfer
+            return use_file_transfer(self.config)
         return False
 
     def preserve_directory_structure(self) -> bool:
         # fall back to preserve subdirectories if config is not present or incomplete
-        if (
-            self.use_file_transfer()
-            and self.config
-            and hasattr(self.config.delivery_method, "preserve_directory_structure")
-            and self.config.delivery_method.preserve_directory_structure is not None
-        ):
-            return self.config.delivery_method.preserve_directory_structure
+        if self.config:
+            return preserve_directory_structure(self.config)
         return True
 
+    def include_identities_stream(self) -> bool:
+        if self.config:
+            return include_identities_stream(self.config)
+        return False
+
     @abstractmethod
     def get_file(
         self, file: RemoteFile, local_directory: str, logger: logging.Logger
@@ -183,3 +184,97 @@ class AbstractFileBasedStreamReader(ABC):
         makedirs(path.dirname(local_file_path), exist_ok=True)
         absolute_file_path = path.abspath(local_file_path)
         return [file_relative_path, local_file_path, absolute_file_path]
+
+    @abstractmethod
+    def get_file_acl_permissions(self, file: RemoteFile, logger: logging.Logger) -> Dict[str, Any]:
+        """
+        This function should return the allow list for a given file, i.e. the list of all identities and their permission levels associated with it
+
+        e.g.
+        def get_file_acl_permissions(self, file: RemoteFile, logger: logging.Logger):
+            api_conn = some_api.conn(credentials=SOME_CREDENTIALS)
+            result = api_conn.get_file_permissions_info(file.id)
+            return MyPermissionsModel(
+                id=result["id"],
+                access_control_list = result["access_control_list"],
+                is_public = result["is_public"],
+                ).dict()
+        """
+        raise NotImplementedError(
+            f"{self.__class__.__name__} does not implement get_file_acl_permissions(). To support ACL permissions, implement this method and update file_permissions_schema."
+        )
+
+    @abstractmethod
+    def load_identity_groups(self, logger: logging.Logger) -> Iterable[Dict[str, Any]]:
+        """
+        This function should return the Identities in a determined "space" or "domain" where the file metadata (ACLs) are fetched and ACLs items (Identities) exists.
+
+        e.g.
+        def load_identity_groups(self, logger: logging.Logger) -> Dict[str, Any]:
+            api_conn = some_api.conn(credentials=SOME_CREDENTIALS)
+            users_api = api_conn.users()
+            groups_api = api_conn.groups()
+            members_api = self.google_directory_service.members()
+            for user in users_api.list():
+                yield my_identity_model(id=user.id, name=user.name, email_address=user.email, type="user").dict()
+            for group in groups_api.list():
+                group_obj = my_identity_model(id=group.id, name=groups.name, email_address=user.email, type="group").dict()
+                for member in members_api.list(group=group):
+                    group_obj.member_email_addresses = group_obj.member_email_addresses or []
+                    group_obj.member_email_addresses.append(member.email)
+                yield group_obj.dict()
+        """
+        raise NotImplementedError(
+            f"{self.__class__.__name__} does not implement load_identity_groups(). To support identities, implement this method and update identities_schema."
+        )
+
+    @property
+    @abstractmethod
+    def file_permissions_schema(self) -> Dict[str, Any]:
+        """
+        This function should return the permissions schema for file permissions stream.
+
+        e.g.
+        def file_permissions_schema(self) -> Dict[str, Any]:
+            # you can also follow the patter we have for python connectors and have a json file and read from there e.g. schemas/identities.json
+            return {
+                  "type": "object",
+                  "properties": {
+                    "id": { "type": "string" },
+                    "file_path": { "type": "string" },
+                    "access_control_list": {
+                      "type": "array",
+                      "items": { "type": "string" }
+                    },
+                    "publicly_accessible": { "type": "boolean" }
+                  }
+                }
+        """
+        raise NotImplementedError(
+            f"{self.__class__.__name__} does not implement file_permissions_schema, please return json schema for your permissions streams."
+        )
+
+    @property
+    @abstractmethod
+    def identities_schema(self) -> Dict[str, Any]:
+        """
+        This function should return the identities schema for file identity stream.
+
+        e.g.
+        def identities_schema(self) -> Dict[str, Any]:
+            # you can also follow the patter we have for python connectors and have a json file and read from there e.g. schemas/identities.json
+            return {
+              "type": "object",
+              "properties": {
+                "id": { "type": "string" },
+                "remote_id": { "type": "string" },
+                "name": { "type": ["null", "string"] },
+                "email_address": { "type": ["null", "string"] },
+                "member_email_addresses": { "type": ["null", "array"] },
+                "type": { "type": "string" },
+              }
+            }
+        """
+        raise NotImplementedError(
+            f"{self.__class__.__name__} does not implement identities_schema, please return json schema for your identities stream."
+        )

airbyte_cdk/sources/file_based/stream/__init__.py

@@ -1,4 +1,13 @@
 from airbyte_cdk.sources.file_based.stream.abstract_file_based_stream import AbstractFileBasedStream
 from airbyte_cdk.sources.file_based.stream.default_file_based_stream import DefaultFileBasedStream
+from airbyte_cdk.sources.file_based.stream.identities_stream import FileIdentitiesStream
+from airbyte_cdk.sources.file_based.stream.permissions_file_based_stream import (
+    PermissionsFileBasedStream,
+)
 
-__all__ = ["AbstractFileBasedStream", "DefaultFileBasedStream"]
+__all__ = [
+    "AbstractFileBasedStream",
+    "DefaultFileBasedStream",
+    "FileIdentitiesStream",
+    "PermissionsFileBasedStream",
+]

airbyte_cdk/sources/file_based/stream/identities_stream.py

@@ -0,0 +1,47 @@
+#
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+#
+
+from functools import cache
+from typing import Any, Dict, Iterable, Mapping, MutableMapping, Optional
+
+from airbyte_cdk.sources.file_based.config.file_based_stream_config import PrimaryKeyType
+from airbyte_cdk.sources.file_based.discovery_policy import AbstractDiscoveryPolicy
+from airbyte_cdk.sources.file_based.exceptions import FileBasedErrorsCollector
+from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader
+from airbyte_cdk.sources.streams.core import JsonSchema
+from airbyte_cdk.sources.streams.permissions.identities_stream import IdentitiesStream
+
+
+class FileIdentitiesStream(IdentitiesStream):
+    """
+    The identities stream. A full refresh stream to sync identities from a certain domain.
+    The stream reader manage the logic to get such data, which is implemented on connector side.
+    """
+
+    is_resumable = False
+
+    def __init__(
+        self,
+        catalog_schema: Optional[Mapping[str, Any]],
+        stream_reader: AbstractFileBasedStreamReader,
+        discovery_policy: AbstractDiscoveryPolicy,
+        errors_collector: FileBasedErrorsCollector,
+    ) -> None:
+        super().__init__()
+        self.catalog_schema = catalog_schema
+        self.stream_reader = stream_reader
+        self._discovery_policy = discovery_policy
+        self.errors_collector = errors_collector
+        self._cursor: MutableMapping[str, Any] = {}
+
+    @property
+    def primary_key(self) -> PrimaryKeyType:
+        return None
+
+    def load_identity_groups(self) -> Iterable[Dict[str, Any]]:
+        return self.stream_reader.load_identity_groups(logger=self.logger)
+
+    @cache
+    def get_json_schema(self) -> JsonSchema:
+        return self.stream_reader.identities_schema

airbyte_cdk/sources/file_based/stream/permissions_file_based_stream.py

@@ -0,0 +1,85 @@
+#
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+#
+
+import traceback
+from typing import Any, Dict, Iterable
+
+from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, Level
+from airbyte_cdk.models import Type as MessageType
+from airbyte_cdk.sources.file_based.stream import DefaultFileBasedStream
+from airbyte_cdk.sources.file_based.types import StreamSlice
+from airbyte_cdk.sources.streams.core import JsonSchema
+from airbyte_cdk.sources.utils.record_helper import stream_data_to_airbyte_message
+
+
+class PermissionsFileBasedStream(DefaultFileBasedStream):
+    """
+    A specialized stream for handling file-based ACL permissions.
+
+    This stream works with the stream_reader to:
+    1. Fetch ACL permissions for each file in the source
+    2. Transform permissions into a standardized format
+    3. Generate records containing permission information
+
+    The stream_reader is responsible for the actual implementation of permission retrieval
+    and schema definition, while this class handles the streaming interface.
+    """
+
+    def _filter_schema_invalid_properties(
+        self, configured_catalog_json_schema: Dict[str, Any]
+    ) -> Dict[str, Any]:
+        return self.stream_reader.file_permissions_schema
+
+    def read_records_from_slice(self, stream_slice: StreamSlice) -> Iterable[AirbyteMessage]:
+        """
+        Yield permissions records from all remote files
+        """
+
+        for file in stream_slice["files"]:
+            no_permissions = False
+            file_datetime_string = file.last_modified.strftime(self.DATE_TIME_FORMAT)
+            try:
+                permissions_record = self.stream_reader.get_file_acl_permissions(
+                    file, logger=self.logger
+                )
+                if not permissions_record:
+                    no_permissions = True
+                    self.logger.warning(
+                        f"Unable to fetch permissions. stream={self.name} file={file.uri}"
+                    )
+                    continue
+                permissions_record = self.transform_record(
+                    permissions_record, file, file_datetime_string
+                )
+                yield stream_data_to_airbyte_message(
+                    self.name, permissions_record, is_file_transfer_message=False
+                )
+            except Exception as e:
+                self.logger.error(f"Failed to retrieve permissions for file {file.uri}: {str(e)}")
+                yield AirbyteMessage(
+                    type=MessageType.LOG,
+                    log=AirbyteLogMessage(
+                        level=Level.ERROR,
+                        message=f"Error retrieving files permissions: stream={self.name} file={file.uri}",
+                        stack_trace=traceback.format_exc(),
+                    ),
+                )
+            finally:
+                if no_permissions:
+                    yield AirbyteMessage(
+                        type=MessageType.LOG,
+                        log=AirbyteLogMessage(
+                            level=Level.WARN,
+                            message=f"Unable to fetch permissions. stream={self.name} file={file.uri}",
+                        ),
+                    )
+
+    def _get_raw_json_schema(self) -> JsonSchema:
+        """
+        Retrieve the raw JSON schema for file permissions from the stream reader.
+
+        Returns:
+           The file permissions schema that defines the structure of permission records
+        """
+        return self.stream_reader.file_permissions_schema

airbyte_cdk/sources/specs/transfer_modes.py

@@ -0,0 +1,26 @@
+#
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+#
+
+from typing import Literal
+
+from pydantic.v1 import AnyUrl, BaseModel, Field
+
+from airbyte_cdk import OneOfOptionConfig
+
+
+class DeliverPermissions(BaseModel):
+    class Config(OneOfOptionConfig):
+        title = "Replicate Permissions ACL"
+        description = "Sends one identity stream and one for more permissions (ACL) streams to the destination. This data can be used in downstream systems to recreate permission restrictions mirroring the original source."
+        discriminator = "delivery_type"
+
+    delivery_type: Literal["use_permissions_transfer"] = Field(
+        "use_permissions_transfer", const=True
+    )
+
+    include_identities_stream: bool = Field(
+        title="Include Identity Stream",
+        description="This data can be used in downstream systems to recreate permission restrictions mirroring the original source",
+        default=True,
+    )

airbyte_cdk/sources/streams/permissions/identities_stream.py

@@ -0,0 +1,75 @@
+#
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+#
+
+import traceback
+from abc import ABC, abstractmethod
+from typing import Any, Dict, Iterable, List, Mapping, MutableMapping, Optional
+
+from airbyte_protocol_dataclasses.models import SyncMode
+
+from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, Level
+from airbyte_cdk.models import Type as MessageType
+from airbyte_cdk.sources.streams import Stream
+from airbyte_cdk.sources.streams.checkpoint import Cursor
+from airbyte_cdk.sources.utils.record_helper import stream_data_to_airbyte_message
+from airbyte_cdk.utils.traced_exception import AirbyteTracedException
+
+
+class IdentitiesStream(Stream, ABC):
+    """
+    The identities stream. A full refresh stream to sync identities from a certain domain.
+    The load_identity_groups method manage the logic to get such data.
+    """
+
+    IDENTITIES_STREAM_NAME = "identities"
+
+    is_resumable = False
+
+    def __init__(self) -> None:
+        super().__init__()
+        self._cursor: MutableMapping[str, Any] = {}
+
+    @property
+    def state(self) -> MutableMapping[str, Any]:
+        return self._cursor
+
+    @state.setter
+    def state(self, value: MutableMapping[str, Any]) -> None:
+        """State setter, accept state serialized by state getter."""
+        self._cursor = value
+
+    def read_records(
+        self,
+        sync_mode: SyncMode,
+        cursor_field: Optional[List[str]] = None,
+        stream_slice: Optional[Mapping[str, Any]] = None,
+        stream_state: Optional[Mapping[str, Any]] = None,
+    ) -> Iterable[Mapping[str, Any] | AirbyteMessage]:
+        try:
+            identity_groups = self.load_identity_groups()
+            for record in identity_groups:
+                yield stream_data_to_airbyte_message(self.name, record)
+        except AirbyteTracedException as exc:
+            # Re-raise the exception to stop the whole sync immediately as this is a fatal error
+            raise exc
+        except Exception as e:
+            yield AirbyteMessage(
+                type=MessageType.LOG,
+                log=AirbyteLogMessage(
+                    level=Level.ERROR,
+                    message=f"Error trying to read identities: {e} stream={self.name}",
+                    stack_trace=traceback.format_exc(),
+                ),
+            )
+
+    @abstractmethod
+    def load_identity_groups(self) -> Iterable[Dict[str, Any]]:
+        raise NotImplementedError("Implement this method to read identity records")
+
+    @property
+    def name(self) -> str:
+        return self.IDENTITIES_STREAM_NAME
+
+    def get_cursor(self) -> Optional[Cursor]:
+        return None

airbyte_cdk/test/mock_http/mocker.py

@@ -17,6 +17,7 @@ class SupportedHttpMethods(str, Enum):
     GET = "get"
     PATCH = "patch"
     POST = "post"
+    PUT = "put"
     DELETE = "delete"
 
 
@@ -77,7 +78,7 @@ class HttpMocker(contextlib.ContextDecorator):
             additional_matcher=self._matches_wrapper(matcher),
             response_list=[
                 {
-                    "text": response.body,
+                    self._get_body_field(response): response.body,
                     "status_code": response.status_code,
                     "headers": response.headers,
                 }
@@ -85,6 +86,10 @@ class HttpMocker(contextlib.ContextDecorator):
             ],
         )
 
+    @staticmethod
+    def _get_body_field(response: HttpResponse) -> str:
+        return "text" if isinstance(response.body, str) else "content"
+
     def get(self, request: HttpRequest, responses: Union[HttpResponse, List[HttpResponse]]) -> None:
         self._mock_request_method(SupportedHttpMethods.GET, request, responses)
 
@@ -98,6 +103,9 @@ class HttpMocker(contextlib.ContextDecorator):
     ) -> None:
         self._mock_request_method(SupportedHttpMethods.POST, request, responses)
 
+    def put(self, request: HttpRequest, responses: Union[HttpResponse, List[HttpResponse]]) -> None:
+        self._mock_request_method(SupportedHttpMethods.PUT, request, responses)
+
     def delete(
         self, request: HttpRequest, responses: Union[HttpResponse, List[HttpResponse]]
     ) -> None:

airbyte_cdk/test/mock_http/response.py

@@ -1,19 +1,22 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 
 from types import MappingProxyType
-from typing import Mapping
+from typing import Mapping, Union
 
 
 class HttpResponse:
     def __init__(
-        self, body: str, status_code: int = 200, headers: Mapping[str, str] = MappingProxyType({})
+        self,
+        body: Union[str, bytes],
+        status_code: int = 200,
+        headers: Mapping[str, str] = MappingProxyType({}),
     ):
         self._body = body
         self._status_code = status_code
         self._headers = headers
 
     @property
-    def body(self) -> str:
+    def body(self) -> Union[str, bytes]:
         return self._body
 
     @property

airbyte_cdk/utils/mapping_helpers.py

@@ -6,6 +6,12 @@
 import copy
 from typing import Any, Dict, List, Mapping, Optional, Union
 
+from airbyte_cdk.sources.declarative.requesters.request_option import (
+    RequestOption,
+    RequestOptionType,
+)
+from airbyte_cdk.sources.types import Config
+
 
 def _merge_mappings(
     target: Dict[str, Any],
@@ -33,13 +39,17 @@ def _merge_mappings(
             if isinstance(target_value, dict) and isinstance(source_value, dict):
                 # Only body_json supports nested_structures
                 if not allow_same_value_merge:
-                    raise ValueError(f"Duplicate keys found: {'.'.join(current_path)}")
+                    raise ValueError(
+                        f"Request body collision, duplicate keys detected at key path: {'.'.join(current_path)}. Please ensure that all keys in the request are unique."
+                    )
                 # If both are dictionaries, recursively merge them
                 _merge_mappings(target_value, source_value, current_path, allow_same_value_merge)
 
             elif not allow_same_value_merge or target_value != source_value:
                 # If same key has different values, that's a conflict
-                raise ValueError(f"Duplicate keys found: {'.'.join(current_path)}")
+                raise ValueError(
+                    f"Request body collision, duplicate keys detected at key path: {'.'.join(current_path)}. Please ensure that all keys in the request are unique."
+                )
         else:
             # No conflict, just copy the value (using deepcopy for nested structures)
             target[key] = copy.deepcopy(source_value)
@@ -102,3 +112,34 @@ def combine_mappings(
             _merge_mappings(result, mapping, allow_same_value_merge=allow_same_value_merge)
 
     return result
+
+
+def _validate_component_request_option_paths(
+    config: Config, *request_options: Optional[RequestOption]
+) -> None:
+    """
+    Validates that a component with multiple request options does not have conflicting paths.
+    Uses dummy values for validation since actual values might not be available at init time.
+    """
+    grouped_options: Dict[RequestOptionType, List[RequestOption]] = {}
+    for option in request_options:
+        if option:
+            grouped_options.setdefault(option.inject_into, []).append(option)
+
+    for inject_type, options in grouped_options.items():
+        if len(options) <= 1:
+            continue
+
+        option_dicts: List[Optional[Union[Mapping[str, Any], str]]] = []
+        for i, option in enumerate(options):
+            option_dict: Dict[str, Any] = {}
+            # Use indexed dummy values to ensure we catch conflicts
+            option.inject_into_request(option_dict, f"dummy_value_{i}", config)
+            option_dicts.append(option_dict)
+
+        try:
+            combine_mappings(
+                option_dicts, allow_same_value_merge=(inject_type == RequestOptionType.body_json)
+            )
+        except ValueError as error:
+            raise ValueError(error)

airbyte_cdk/utils/print_buffer.py

@@ -73,7 +73,3 @@ class PrintBuffer:
     ) -> None:
         self.flush()
         sys.stdout, sys.stderr = self.old_stdout, self.old_stderr
-
-    def flush_logger(self) -> None:
-        """Explicit flush that can be triggered by logger to synchronize with PrintBuffer."""
-        self.flush()

{airbyte_cdk-6.34.0.dev0.dist-info → airbyte_cdk-6.34.0.dev2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 6.34.0.dev0
+Version: 6.34.0.dev2
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://airbyte.com
 License: MIT