airbyte-cdk 6.34.0.dev2__py3-none-any.whl → 6.34.1.dev0__py3-none-any.whl

airbyte_cdk/sources/file_based/config/validate_config_transfer_modes.py

@@ -1,81 +0,0 @@
-#
-# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
-#
-
-from airbyte_cdk.sources.file_based.config.abstract_file_based_spec import (
-    AbstractFileBasedSpec,
-    DeliverRawFiles,
-)
-from airbyte_cdk.sources.specs.transfer_modes import DeliverPermissions
-
-DELIVERY_TYPE_KEY = "delivery_type"
-DELIVERY_TYPE_PERMISSION_TRANSFER_MODE_VALUE = "use_permissions_transfer"
-DELIVERY_TYPE_FILES_TRANSFER_MODE_VALUE = "use_file_transfer"
-PRESERVE_DIRECTORY_STRUCTURE_KEY = "preserve_directory_structure"
-INCLUDE_IDENTITIES_STREAM_KEY = "include_identities_stream"
-
-
-def use_file_transfer(parsed_config: AbstractFileBasedSpec) -> bool:
-    """Returns `True` if the configuration uses file transfer mode."""
-    return (
-        hasattr(parsed_config.delivery_method, DELIVERY_TYPE_KEY)
-        and parsed_config.delivery_method.delivery_type == DELIVERY_TYPE_FILES_TRANSFER_MODE_VALUE
-    )
-
-
-def preserve_directory_structure(parsed_config: AbstractFileBasedSpec) -> bool:
-    """
-    Determines whether to preserve directory structure during file transfer.
-
-    When enabled, files maintain their subdirectory paths in the destination.
-    When disabled, files are flattened to the root of the destination.
-
-    Args:
-        parsed_config: The parsed configuration containing delivery method settings
-
-    Returns:
-        True if directory structure should be preserved (default), False otherwise
-    """
-    if (
-        use_file_transfer(parsed_config)
-        and hasattr(parsed_config.delivery_method, PRESERVE_DIRECTORY_STRUCTURE_KEY)
-        and isinstance(parsed_config.delivery_method, DeliverRawFiles)
-    ):
-        return parsed_config.delivery_method.preserve_directory_structure
-    return True
-
-
-def use_permissions_transfer(parsed_config: AbstractFileBasedSpec) -> bool:
-    """
-    Determines whether to use permissions transfer to sync ACLs and Identities
-
-    Args:
-        parsed_config: The parsed configuration containing delivery method settings
-
-    Returns:
-        True if permissions transfer should be enabled, False otherwise
-    """
-    return (
-        hasattr(parsed_config.delivery_method, DELIVERY_TYPE_KEY)
-        and parsed_config.delivery_method.delivery_type
-        == DELIVERY_TYPE_PERMISSION_TRANSFER_MODE_VALUE
-    )
-
-
-def include_identities_stream(parsed_config: AbstractFileBasedSpec) -> bool:
-    """
-    There are scenarios where user may not have access to identities but still is valuable to get ACLs
-
-    Args:
-        parsed_config: The parsed configuration containing delivery method settings
-
-    Returns:
-        True if we should include Identities stream.
-    """
-    if (
-        use_permissions_transfer(parsed_config)
-        and hasattr(parsed_config.delivery_method, INCLUDE_IDENTITIES_STREAM_KEY)
-        and isinstance(parsed_config.delivery_method, DeliverPermissions)
-    ):
-        return parsed_config.delivery_method.include_identities_stream
-    return False

airbyte_cdk/sources/file_based/stream/identities_stream.py

@@ -1,47 +0,0 @@
-#
-# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
-#
-
-from functools import cache
-from typing import Any, Dict, Iterable, Mapping, MutableMapping, Optional
-
-from airbyte_cdk.sources.file_based.config.file_based_stream_config import PrimaryKeyType
-from airbyte_cdk.sources.file_based.discovery_policy import AbstractDiscoveryPolicy
-from airbyte_cdk.sources.file_based.exceptions import FileBasedErrorsCollector
-from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader
-from airbyte_cdk.sources.streams.core import JsonSchema
-from airbyte_cdk.sources.streams.permissions.identities_stream import IdentitiesStream
-
-
-class FileIdentitiesStream(IdentitiesStream):
-    """
-    The identities stream. A full refresh stream to sync identities from a certain domain.
-    The stream reader manage the logic to get such data, which is implemented on connector side.
-    """
-
-    is_resumable = False
-
-    def __init__(
-        self,
-        catalog_schema: Optional[Mapping[str, Any]],
-        stream_reader: AbstractFileBasedStreamReader,
-        discovery_policy: AbstractDiscoveryPolicy,
-        errors_collector: FileBasedErrorsCollector,
-    ) -> None:
-        super().__init__()
-        self.catalog_schema = catalog_schema
-        self.stream_reader = stream_reader
-        self._discovery_policy = discovery_policy
-        self.errors_collector = errors_collector
-        self._cursor: MutableMapping[str, Any] = {}
-
-    @property
-    def primary_key(self) -> PrimaryKeyType:
-        return None
-
-    def load_identity_groups(self) -> Iterable[Dict[str, Any]]:
-        return self.stream_reader.load_identity_groups(logger=self.logger)
-
-    @cache
-    def get_json_schema(self) -> JsonSchema:
-        return self.stream_reader.identities_schema

airbyte_cdk/sources/file_based/stream/permissions_file_based_stream.py

@@ -1,85 +0,0 @@
-#
-# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
-#
-
-import traceback
-from typing import Any, Dict, Iterable
-
-from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, Level
-from airbyte_cdk.models import Type as MessageType
-from airbyte_cdk.sources.file_based.stream import DefaultFileBasedStream
-from airbyte_cdk.sources.file_based.types import StreamSlice
-from airbyte_cdk.sources.streams.core import JsonSchema
-from airbyte_cdk.sources.utils.record_helper import stream_data_to_airbyte_message
-
-
-class PermissionsFileBasedStream(DefaultFileBasedStream):
-    """
-    A specialized stream for handling file-based ACL permissions.
-
-    This stream works with the stream_reader to:
-    1. Fetch ACL permissions for each file in the source
-    2. Transform permissions into a standardized format
-    3. Generate records containing permission information
-
-    The stream_reader is responsible for the actual implementation of permission retrieval
-    and schema definition, while this class handles the streaming interface.
-    """
-
-    def _filter_schema_invalid_properties(
-        self, configured_catalog_json_schema: Dict[str, Any]
-    ) -> Dict[str, Any]:
-        return self.stream_reader.file_permissions_schema
-
-    def read_records_from_slice(self, stream_slice: StreamSlice) -> Iterable[AirbyteMessage]:
-        """
-        Yield permissions records from all remote files
-        """
-
-        for file in stream_slice["files"]:
-            no_permissions = False
-            file_datetime_string = file.last_modified.strftime(self.DATE_TIME_FORMAT)
-            try:
-                permissions_record = self.stream_reader.get_file_acl_permissions(
-                    file, logger=self.logger
-                )
-                if not permissions_record:
-                    no_permissions = True
-                    self.logger.warning(
-                        f"Unable to fetch permissions. stream={self.name} file={file.uri}"
-                    )
-                    continue
-                permissions_record = self.transform_record(
-                    permissions_record, file, file_datetime_string
-                )
-                yield stream_data_to_airbyte_message(
-                    self.name, permissions_record, is_file_transfer_message=False
-                )
-            except Exception as e:
-                self.logger.error(f"Failed to retrieve permissions for file {file.uri}: {str(e)}")
-                yield AirbyteMessage(
-                    type=MessageType.LOG,
-                    log=AirbyteLogMessage(
-                        level=Level.ERROR,
-                        message=f"Error retrieving files permissions: stream={self.name} file={file.uri}",
-                        stack_trace=traceback.format_exc(),
-                    ),
-                )
-            finally:
-                if no_permissions:
-                    yield AirbyteMessage(
-                        type=MessageType.LOG,
-                        log=AirbyteLogMessage(
-                            level=Level.WARN,
-                            message=f"Unable to fetch permissions. stream={self.name} file={file.uri}",
-                        ),
-                    )
-
-    def _get_raw_json_schema(self) -> JsonSchema:
-        """
-        Retrieve the raw JSON schema for file permissions from the stream reader.
-
-        Returns:
-           The file permissions schema that defines the structure of permission records
-        """
-        return self.stream_reader.file_permissions_schema

airbyte_cdk/sources/specs/transfer_modes.py

@@ -1,26 +0,0 @@
-#
-# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
-#
-
-from typing import Literal
-
-from pydantic.v1 import AnyUrl, BaseModel, Field
-
-from airbyte_cdk import OneOfOptionConfig
-
-
-class DeliverPermissions(BaseModel):
-    class Config(OneOfOptionConfig):
-        title = "Replicate Permissions ACL"
-        description = "Sends one identity stream and one for more permissions (ACL) streams to the destination. This data can be used in downstream systems to recreate permission restrictions mirroring the original source."
-        discriminator = "delivery_type"
-
-    delivery_type: Literal["use_permissions_transfer"] = Field(
-        "use_permissions_transfer", const=True
-    )
-
-    include_identities_stream: bool = Field(
-        title="Include Identity Stream",
-        description="This data can be used in downstream systems to recreate permission restrictions mirroring the original source",
-        default=True,
-    )

airbyte_cdk/sources/streams/permissions/identities_stream.py

@@ -1,75 +0,0 @@
-#
-# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
-#
-
-import traceback
-from abc import ABC, abstractmethod
-from typing import Any, Dict, Iterable, List, Mapping, MutableMapping, Optional
-
-from airbyte_protocol_dataclasses.models import SyncMode
-
-from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, Level
-from airbyte_cdk.models import Type as MessageType
-from airbyte_cdk.sources.streams import Stream
-from airbyte_cdk.sources.streams.checkpoint import Cursor
-from airbyte_cdk.sources.utils.record_helper import stream_data_to_airbyte_message
-from airbyte_cdk.utils.traced_exception import AirbyteTracedException
-
-
-class IdentitiesStream(Stream, ABC):
-    """
-    The identities stream. A full refresh stream to sync identities from a certain domain.
-    The load_identity_groups method manage the logic to get such data.
-    """
-
-    IDENTITIES_STREAM_NAME = "identities"
-
-    is_resumable = False
-
-    def __init__(self) -> None:
-        super().__init__()
-        self._cursor: MutableMapping[str, Any] = {}
-
-    @property
-    def state(self) -> MutableMapping[str, Any]:
-        return self._cursor
-
-    @state.setter
-    def state(self, value: MutableMapping[str, Any]) -> None:
-        """State setter, accept state serialized by state getter."""
-        self._cursor = value
-
-    def read_records(
-        self,
-        sync_mode: SyncMode,
-        cursor_field: Optional[List[str]] = None,
-        stream_slice: Optional[Mapping[str, Any]] = None,
-        stream_state: Optional[Mapping[str, Any]] = None,
-    ) -> Iterable[Mapping[str, Any] | AirbyteMessage]:
-        try:
-            identity_groups = self.load_identity_groups()
-            for record in identity_groups:
-                yield stream_data_to_airbyte_message(self.name, record)
-        except AirbyteTracedException as exc:
-            # Re-raise the exception to stop the whole sync immediately as this is a fatal error
-            raise exc
-        except Exception as e:
-            yield AirbyteMessage(
-                type=MessageType.LOG,
-                log=AirbyteLogMessage(
-                    level=Level.ERROR,
-                    message=f"Error trying to read identities: {e} stream={self.name}",
-                    stack_trace=traceback.format_exc(),
-                ),
-            )
-
-    @abstractmethod
-    def load_identity_groups(self) -> Iterable[Dict[str, Any]]:
-        raise NotImplementedError("Implement this method to read identity records")
-
-    @property
-    def name(self) -> str:
-        return self.IDENTITIES_STREAM_NAME
-
-    def get_cursor(self) -> Optional[Cursor]:
-        return None