airbyte-cdk 6.26.0.dev4108__tar.gz → 6.26.0.dev4109__tar.gz

{airbyte_cdk-6.26.0.dev4108 → airbyte_cdk-6.26.0.dev4109}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 6.26.0.dev4108
+Version: 6.26.0.dev4109
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://airbyte.com
 License: MIT

{airbyte_cdk-6.26.0.dev4108 → airbyte_cdk-6.26.0.dev4109}/airbyte_cdk/sources/file_based/config/abstract_file_based_spec.py

@@ -11,26 +11,10 @@ from pydantic.v1 import AnyUrl, BaseModel, Field
 
 from airbyte_cdk import OneOfOptionConfig
 from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileBasedStreamConfig
+from airbyte_cdk.sources.specs.transfer_modes import DeliverPermissions
 from airbyte_cdk.sources.utils import schema_helpers
 
 
-class DeliverPermissions(BaseModel):
-    class Config(OneOfOptionConfig):
-        title = "Replicate Permissions ACL"
-        description = "Sends one identity stream and one for more permissions (ACL) streams to the destination. This data can be used in downstream systems to recreate permission restrictions mirroring the original source."
-        discriminator = "delivery_type"
-
-    delivery_type: Literal["use_permissions_transfer"] = Field(
-        "use_permissions_transfer", const=True
-    )
-
-    include_identities_stream: bool = Field(
-        title="Include Identity Stream",
-        description="This data can be used in downstream systems to recreate permission restrictions mirroring the original source",
-        default=True,
-    )
-
-
 class DeliverRecords(BaseModel):
     class Config(OneOfOptionConfig):
         title = "Replicate Records"

{airbyte_cdk-6.26.0.dev4108 → airbyte_cdk-6.26.0.dev4109}/airbyte_cdk/sources/file_based/config/validate_config_transfer_modes.py

@@ -2,13 +2,23 @@
 # Copyright (c) 2025 Airbyte, Inc., all rights reserved.
 #
 
-from airbyte_cdk.sources.file_based.config.abstract_file_based_spec import AbstractFileBasedSpec
+from airbyte_cdk.sources.file_based.config.abstract_file_based_spec import (
+    AbstractFileBasedSpec,
+    DeliverRawFiles,
+)
+from airbyte_cdk.sources.specs.transfer_modes import DeliverPermissions
+
+DELIVERY_TYPE_KEY = "delivery_type"
+DELIVERY_TYPE_PERMISSION_TRANSFER_MODE_VALUE = "use_permissions_transfer"
+DELIVERY_TYPE_FILES_TRANSFER_MODE_VALUE = "use_file_transfer"
+PRESERVE_DIRECTORY_STRUCTURE_KEY = "preserve_directory_structure"
+INCLUDE_IDENTITIES_STREAM_KEY = "include_identities_stream"
 
 
 def use_file_transfer(parsed_config: AbstractFileBasedSpec) -> bool:
     return (
-        hasattr(parsed_config.delivery_method, "delivery_type")
-        and parsed_config.delivery_method.delivery_type == "use_file_transfer"
+        hasattr(parsed_config.delivery_method, DELIVERY_TYPE_KEY)
+        and parsed_config.delivery_method.delivery_type == DELIVERY_TYPE_FILES_TRANSFER_MODE_VALUE
     )
 
 
@@ -27,8 +37,8 @@ def preserve_directory_structure(parsed_config: AbstractFileBasedSpec) -> bool:
     """
     if (
         use_file_transfer(parsed_config)
-        and hasattr(parsed_config.delivery_method, "preserve_directory_structure")
-        and parsed_config.delivery_method.preserve_directory_structure is not None
+        and hasattr(parsed_config.delivery_method, PRESERVE_DIRECTORY_STRUCTURE_KEY)
+        and isinstance(parsed_config.delivery_method, DeliverRawFiles)
     ):
         return parsed_config.delivery_method.preserve_directory_structure
     return True
@@ -36,16 +46,17 @@ def preserve_directory_structure(parsed_config: AbstractFileBasedSpec) -> bool:
 
 def use_permissions_transfer(parsed_config: AbstractFileBasedSpec) -> bool:
     return (
-        hasattr(parsed_config.delivery_method, "delivery_type")
-        and parsed_config.delivery_method.delivery_type == "use_permissions_transfer"
+        hasattr(parsed_config.delivery_method, DELIVERY_TYPE_KEY)
+        and parsed_config.delivery_method.delivery_type
+        == DELIVERY_TYPE_PERMISSION_TRANSFER_MODE_VALUE
     )
 
 
 def include_identities_stream(parsed_config: AbstractFileBasedSpec) -> bool:
     if (
         use_permissions_transfer(parsed_config)
-        and hasattr(parsed_config.delivery_method, "include_identities_stream")
-        and parsed_config.delivery_method.include_identities_stream is not None
+        and hasattr(parsed_config.delivery_method, INCLUDE_IDENTITIES_STREAM_KEY)
+        and isinstance(parsed_config.delivery_method, DeliverPermissions)
     ):
         return parsed_config.delivery_method.include_identities_stream
     return False

{airbyte_cdk-6.26.0.dev4108 → airbyte_cdk-6.26.0.dev4109}/airbyte_cdk/sources/file_based/file_based_source.py

@@ -58,7 +58,7 @@ from airbyte_cdk.sources.file_based.schema_validation_policies import (
 from airbyte_cdk.sources.file_based.stream import (
     AbstractFileBasedStream,
     DefaultFileBasedStream,
-    IdentitiesStream,
+    FileIdentities,
 )
 from airbyte_cdk.sources.file_based.stream.concurrent.adapters import FileBasedStreamFacade
 from airbyte_cdk.sources.file_based.stream.concurrent.cursor import (
@@ -67,7 +67,6 @@ from airbyte_cdk.sources.file_based.stream.concurrent.cursor import (
     FileBasedFinalStateCursor,
 )
 from airbyte_cdk.sources.file_based.stream.cursor import AbstractFileBasedCursor
-from airbyte_cdk.sources.file_based.stream.identities_stream import IDENTITIES_STREAM_NAME
 from airbyte_cdk.sources.message.repository import InMemoryMessageRepository, MessageRepository
 from airbyte_cdk.sources.streams import Stream
 from airbyte_cdk.sources.streams.concurrent.cursor import CursorField
@@ -169,7 +168,7 @@ class FileBasedSource(ConcurrentSourceAdapter, ABC):
         errors = []
         tracebacks = []
         for stream in streams:
-            if isinstance(stream, IdentitiesStream):
+            if isinstance(stream, FileIdentities):
                 identity = next(iter(stream.load_identity_groups()))
                 if not identity:
                     errors.append(
@@ -341,8 +340,8 @@ class FileBasedSource(ConcurrentSourceAdapter, ABC):
     def _make_identities_stream(
         self,
     ) -> Stream:
-        return IdentitiesStream(
-            catalog_schema=self.stream_schemas.get(IDENTITIES_STREAM_NAME),
+        return FileIdentities(
+            catalog_schema=self.stream_schemas.get(FileIdentities.IDENTITIES_STREAM_NAME),
             stream_reader=self.stream_reader,
             discovery_policy=self.discovery_policy,
             errors_collector=self.errors_collector,

{airbyte_cdk-6.26.0.dev4108 → airbyte_cdk-6.26.0.dev4109}/airbyte_cdk/sources/file_based/file_based_stream_reader.py

@@ -192,7 +192,7 @@ class AbstractFileBasedStreamReader(ABC):
         ACL Permissions from files.
         """
         raise NotImplementedError(
-            f"{self.__class__.__name__} must implement get_file_acl_permissions()"
+            f"{self.__class__.__name__} does not implement get_file_acl_permissions(). To support ACL permissions, implement this method and update file_permissions_schema."
         )
 
     @abstractmethod
@@ -202,5 +202,27 @@ class AbstractFileBasedStreamReader(ABC):
         identities.
         """
         raise NotImplementedError(
-            f"{self.__class__.__name__} must implement load_identity_groups()"
+            f"{self.__class__.__name__} does not implement load_identity_groups(). To support identities, implement this method and update identities_schema."
+        )
+
+    @property
+    @abstractmethod
+    def file_permissions_schema(self) -> Dict[str, Any]:
+        """
+        This is required for connectors that will support syncing
+        ACL Permissions from files.
+        """
+        raise NotImplementedError(
+            f"{self.__class__.__name__} does not implement file_permissions_schema, please return json schema for your permissions streams."
+        )
+
+    @property
+    @abstractmethod
+    def identities_schema(self) -> Dict[str, Any]:
+        """
+        This is required for connectors that will support syncing
+        identities.
+        """
+        raise NotImplementedError(
+            f"{self.__class__.__name__} does not implement identities_schema, please return json schema for your identities stream."
         )

{airbyte_cdk-6.26.0.dev4108 → airbyte_cdk-6.26.0.dev4109}/airbyte_cdk/sources/file_based/schema_helpers.py

@@ -23,31 +23,6 @@ file_transfer_schema = {
     "properties": {"data": {"type": "object"}, "file": {"type": "object"}},
 }
 
-remote_file_permissions_schema = {
-    "type": "object",
-    "properties": {
-        "id": {"type": "string"},
-        "file_path": {"type": "string"},
-        "allowed_identity_remote_ids": {"type": "array", "items": {"type": "string"}},
-        "publicly_accessible": {"type": "boolean"},
-    },
-}
-
-remote_file_identity_schema = {
-    "type": "object",
-    "properties": {
-        "id": {"type": "string"},
-        "remote_id": {"type": "string"},
-        "parent_id": {"type": ["null", "string"]},
-        "name": {"type": ["null", "string"]},
-        "description": {"type": ["null", "string"]},
-        "email_address": {"type": ["null", "string"]},
-        "member_email_addresses": {"type": ["null", "array"]},
-        "type": {"type": "string"},
-        "modified_at": {"type": "string"},
-    },
-}
-
 
 @total_ordering
 class ComparableType(Enum):

{airbyte_cdk-6.26.0.dev4108 → airbyte_cdk-6.26.0.dev4109}/airbyte_cdk/sources/file_based/stream/__init__.py

@@ -1,5 +1,5 @@
 from airbyte_cdk.sources.file_based.stream.abstract_file_based_stream import AbstractFileBasedStream
 from airbyte_cdk.sources.file_based.stream.default_file_based_stream import DefaultFileBasedStream
-from airbyte_cdk.sources.file_based.stream.identities_stream import IdentitiesStream
+from airbyte_cdk.sources.file_based.stream.identities_stream import FileIdentities
 
-__all__ = ["AbstractFileBasedStream", "DefaultFileBasedStream", "IdentitiesStream"]
+__all__ = ["AbstractFileBasedStream", "DefaultFileBasedStream", "FileIdentities"]

{airbyte_cdk-6.26.0.dev4108 → airbyte_cdk-6.26.0.dev4109}/airbyte_cdk/sources/file_based/stream/default_file_based_stream.py

@@ -29,7 +29,6 @@ from airbyte_cdk.sources.file_based.schema_helpers import (
     SchemaType,
     file_transfer_schema,
     merge_schemas,
-    remote_file_permissions_schema,
     schemaless_schema,
 )
 from airbyte_cdk.sources.file_based.stream import AbstractFileBasedStream
@@ -111,7 +110,7 @@ class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin):
                 },
             }
         elif self.use_permissions_transfer:
-            return remote_file_permissions_schema
+            return self.stream_reader.file_permissions_schema
         else:
             return super()._filter_schema_invalid_properties(configured_catalog_json_schema)
 
@@ -315,7 +314,7 @@ class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin):
         if self.use_file_transfer:
             return file_transfer_schema
         elif self.use_permissions_transfer:
-            return remote_file_permissions_schema
+            return self.stream_reader.file_permissions_schema
         elif self.config.input_schema:
             return self.config.get_input_schema()  # type: ignore
         elif self.config.schemaless:

airbyte_cdk-6.26.0.dev4109/airbyte_cdk/sources/file_based/stream/identities_stream.py

@@ -0,0 +1,47 @@
+#
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+#
+
+from functools import cache
+from typing import Any, Dict, Iterable, Mapping, MutableMapping, Optional
+
+from airbyte_cdk.sources.file_based.config.file_based_stream_config import PrimaryKeyType
+from airbyte_cdk.sources.file_based.discovery_policy import AbstractDiscoveryPolicy
+from airbyte_cdk.sources.file_based.exceptions import FileBasedErrorsCollector
+from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader
+from airbyte_cdk.sources.streams.core import JsonSchema
+from airbyte_cdk.sources.streams.permissions.identities import Identities
+
+
+class FileIdentities(Identities):
+    """
+    The identities stream. A full refresh stream to sync identities from a certain domain.
+    The stream reader manage the logic to get such data, which is implemented on connector side.
+    """
+
+    is_resumable = False
+
+    def __init__(
+        self,
+        catalog_schema: Optional[Mapping[str, Any]],
+        stream_reader: AbstractFileBasedStreamReader,
+        discovery_policy: AbstractDiscoveryPolicy,
+        errors_collector: FileBasedErrorsCollector,
+    ) -> None:
+        super().__init__()
+        self.catalog_schema = catalog_schema
+        self.stream_reader = stream_reader
+        self._discovery_policy = discovery_policy
+        self.errors_collector = errors_collector
+        self._cursor: MutableMapping[str, Any] = {}
+
+    @property
+    def primary_key(self) -> PrimaryKeyType:
+        return None
+
+    def load_identity_groups(self) -> Iterable[Dict[str, Any]]:
+        return self.stream_reader.load_identity_groups(logger=self.logger)
+
+    @cache
+    def get_json_schema(self) -> JsonSchema:
+        return self.stream_reader.identities_schema

airbyte_cdk-6.26.0.dev4109/airbyte_cdk/sources/specs/transfer_modes.py

@@ -0,0 +1,26 @@
+#
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+#
+
+from typing import Literal
+
+from pydantic.v1 import AnyUrl, BaseModel, Field
+
+from airbyte_cdk import OneOfOptionConfig
+
+
+class DeliverPermissions(BaseModel):
+    class Config(OneOfOptionConfig):
+        title = "Replicate Permissions ACL"
+        description = "Sends one identity stream and one for more permissions (ACL) streams to the destination. This data can be used in downstream systems to recreate permission restrictions mirroring the original source."
+        discriminator = "delivery_type"
+
+    delivery_type: Literal["use_permissions_transfer"] = Field(
+        "use_permissions_transfer", const=True
+    )
+
+    include_identities_stream: bool = Field(
+        title="Include Identity Stream",
+        description="This data can be used in downstream systems to recreate permission restrictions mirroring the original source",
+        default=True,
+    )

airbyte_cdk-6.26.0.dev4108/airbyte_cdk/sources/file_based/stream/identities_stream.py → airbyte_cdk-6.26.0.dev4109/airbyte_cdk/sources/streams/permissions/identities.py

@@ -3,48 +3,33 @@
 #
 
 import traceback
-from functools import cache
+from abc import ABC, abstractmethod
 from typing import Any, Dict, Iterable, List, Mapping, MutableMapping, Optional
 
 from airbyte_protocol_dataclasses.models import SyncMode
 
 from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, Level
 from airbyte_cdk.models import Type as MessageType
-from airbyte_cdk.sources.file_based.config.file_based_stream_config import PrimaryKeyType
-from airbyte_cdk.sources.file_based.discovery_policy import AbstractDiscoveryPolicy
-from airbyte_cdk.sources.file_based.exceptions import FileBasedErrorsCollector, FileBasedSourceError
-from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader
-from airbyte_cdk.sources.file_based.schema_helpers import remote_file_identity_schema
-from airbyte_cdk.sources.file_based.types import StreamSlice
 from airbyte_cdk.sources.streams import Stream
 from airbyte_cdk.sources.streams.checkpoint import Cursor
-from airbyte_cdk.sources.streams.core import JsonSchema
 from airbyte_cdk.sources.utils.record_helper import stream_data_to_airbyte_message
 from airbyte_cdk.utils.traced_exception import AirbyteTracedException
 
-IDENTITIES_STREAM_NAME = "identities"
+DEFAULT_IDENTITIES_STREAM_NAME = "identities"
 
 
-class IdentitiesStream(Stream):
+class Identities(Stream, ABC):
     """
     The identities stream. A full refresh stream to sync identities from a certain domain.
-    The stream reader manage the logic to get such data, which is implemented on connector side.
+    The load_identity_groups method manage the logic to get such data.
     """
 
+    IDENTITIES_STREAM_NAME = DEFAULT_IDENTITIES_STREAM_NAME
+
     is_resumable = False
 
-    def __init__(
-        self,
-        catalog_schema: Optional[Mapping[str, Any]],
-        stream_reader: AbstractFileBasedStreamReader,
-        discovery_policy: AbstractDiscoveryPolicy,
-        errors_collector: FileBasedErrorsCollector,
-    ):
+    def __init__(self) -> None:
         super().__init__()
-        self.catalog_schema = catalog_schema
-        self.stream_reader = stream_reader
-        self._discovery_policy = discovery_policy
-        self.errors_collector = errors_collector
         self._cursor: MutableMapping[str, Any] = {}
 
     @property
@@ -56,15 +41,11 @@ class IdentitiesStream(Stream):
         """State setter, accept state serialized by state getter."""
         self._cursor = value
 
-    @property
-    def primary_key(self) -> PrimaryKeyType:
-        return None
-
     def read_records(
         self,
         sync_mode: SyncMode,
         cursor_field: Optional[List[str]] = None,
-        stream_slice: Optional[StreamSlice] = None,
+        stream_slice: Optional[Mapping[str, Any]] = None,
         stream_state: Optional[Mapping[str, Any]] = None,
     ) -> Iterable[Mapping[str, Any] | AirbyteMessage]:
         try:
@@ -74,26 +55,23 @@ class IdentitiesStream(Stream):
         except AirbyteTracedException as exc:
             # Re-raise the exception to stop the whole sync immediately as this is a fatal error
             raise exc
-        except Exception:
+        except Exception as e:
             yield AirbyteMessage(
                 type=MessageType.LOG,
                 log=AirbyteLogMessage(
                     level=Level.ERROR,
-                    message=f"{FileBasedSourceError.ERROR_PARSING_RECORD.value} stream={self.name}",
+                    message=f"Error trying to read identities: {e} stream={self.name}",
                     stack_trace=traceback.format_exc(),
                 ),
             )
 
+    @abstractmethod
     def load_identity_groups(self) -> Iterable[Dict[str, Any]]:
-        return self.stream_reader.load_identity_groups(logger=self.logger)
-
-    @cache
-    def get_json_schema(self) -> JsonSchema:
-        return remote_file_identity_schema
+        raise NotImplementedError("Implement this method to read identity records")
 
     @property
     def name(self) -> str:
-        return IDENTITIES_STREAM_NAME
+        return self.IDENTITIES_STREAM_NAME
 
     def get_cursor(self) -> Optional[Cursor]:
         return None

{airbyte_cdk-6.26.0.dev4108 → airbyte_cdk-6.26.0.dev4109}/pyproject.toml

@@ -23,7 +23,7 @@ classifiers = [
 keywords = ["airbyte", "connector-development-kit", "cdk"]
 
 # Python CDK uses dynamic versioning: https://github.com/mtkennerly/poetry-dynamic-versioning
-version = "6.26.0.dev04108"  # Version will be calculated dynamically.
+version = "6.26.0.dev04109"  # Version will be calculated dynamically.
 
 [tool.poetry-dynamic-versioning]
 enable = false