airbyte-cdk 6.26.0.dev4109__tar.gz → 6.26.0.dev4110__tar.gz

{airbyte_cdk-6.26.0.dev4109 → airbyte_cdk-6.26.0.dev4110}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 6.26.0.dev4109
+Version: 6.26.0.dev4110
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://airbyte.com
 License: MIT

{airbyte_cdk-6.26.0.dev4109 → airbyte_cdk-6.26.0.dev4110}/airbyte_cdk/sources/file_based/file_based_source.py

@@ -67,6 +67,9 @@ from airbyte_cdk.sources.file_based.stream.concurrent.cursor import (
     FileBasedFinalStateCursor,
 )
 from airbyte_cdk.sources.file_based.stream.cursor import AbstractFileBasedCursor
+from airbyte_cdk.sources.file_based.stream.permissions_file_based_stream import (
+    PermissionsFileBasedStream,
+)
 from airbyte_cdk.sources.message.repository import InMemoryMessageRepository, MessageRepository
 from airbyte_cdk.sources.streams import Stream
 from airbyte_cdk.sources.streams.concurrent.cursor import CursorField
@@ -257,7 +260,7 @@ class FileBasedSource(ConcurrentSourceAdapter, ABC):
                         message_repository=self.message_repository,
                     )
                     stream = FileBasedStreamFacade.create_from_stream(
-                        stream=self._make_default_stream(
+                        stream=self._make_file_based_stream(
                             stream_config=stream_config,
                             cursor=cursor,
                             parsed_config=parsed_config,
@@ -288,7 +291,7 @@ class FileBasedSource(ConcurrentSourceAdapter, ABC):
                         CursorField(DefaultFileBasedStream.ab_last_mod_col),
                     )
                     stream = FileBasedStreamFacade.create_from_stream(
-                        stream=self._make_default_stream(
+                        stream=self._make_file_based_stream(
                             stream_config=stream_config,
                             cursor=cursor,
                             parsed_config=parsed_config,
@@ -300,7 +303,7 @@ class FileBasedSource(ConcurrentSourceAdapter, ABC):
                     )
                 else:
                     cursor = self.cursor_cls(stream_config)
-                    stream = self._make_default_stream(
+                    stream = self._make_file_based_stream(
                         stream_config=stream_config,
                         cursor=cursor,
                         parsed_config=parsed_config,
@@ -334,9 +337,34 @@ class FileBasedSource(ConcurrentSourceAdapter, ABC):
             cursor=cursor,
             use_file_transfer=use_file_transfer(parsed_config),
             preserve_directory_structure=preserve_directory_structure(parsed_config),
-            use_permissions_transfer=use_permissions_transfer(parsed_config),
         )
 
+    def _make_permissions_stream(
+        self, stream_config: FileBasedStreamConfig, cursor: Optional[AbstractFileBasedCursor]
+    ) -> AbstractFileBasedStream:
+        return PermissionsFileBasedStream(
+            config=stream_config,
+            catalog_schema=self.stream_schemas.get(stream_config.name),
+            stream_reader=self.stream_reader,
+            availability_strategy=self.availability_strategy,
+            discovery_policy=self.discovery_policy,
+            parsers=self.parsers,
+            validation_policy=self._validate_and_get_validation_policy(stream_config),
+            errors_collector=self.errors_collector,
+            cursor=cursor,
+        )
+
+    def _make_file_based_stream(
+        self,
+        stream_config: FileBasedStreamConfig,
+        cursor: Optional[AbstractFileBasedCursor],
+        parsed_config: AbstractFileBasedSpec,
+    ) -> AbstractFileBasedStream:
+        if use_permissions_transfer(parsed_config):
+            return self._make_permissions_stream(stream_config, cursor)
+        else:
+            return self._make_default_stream(stream_config, cursor, parsed_config)
+
     def _make_identities_stream(
         self,
     ) -> Stream:

{airbyte_cdk-6.26.0.dev4109 → airbyte_cdk-6.26.0.dev4110}/airbyte_cdk/sources/file_based/stream/default_file_based_stream.py

@@ -47,7 +47,6 @@ class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin):
 
     FILE_TRANSFER_KW = "use_file_transfer"
     PRESERVE_DIRECTORY_STRUCTURE_KW = "preserve_directory_structure"
-    PERMISSIONS_TRANSFER_KW = "use_permissions_transfer"
     FILES_KEY = "files"
     DATE_TIME_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ"
     ab_last_mod_col = "_ab_source_file_last_modified"
@@ -57,7 +56,6 @@ class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin):
     airbyte_columns = [ab_last_mod_col, ab_file_name_col]
     use_file_transfer = False
     preserve_directory_structure = True
-    use_permissions_transfer = False
 
     def __init__(self, **kwargs: Any):
         if self.FILE_TRANSFER_KW in kwargs:
@@ -66,8 +64,6 @@ class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin):
             self.preserve_directory_structure = kwargs.pop(
                 self.PRESERVE_DIRECTORY_STRUCTURE_KW, True
             )
-        if self.PERMISSIONS_TRANSFER_KW in kwargs:
-            self.use_permissions_transfer = kwargs.pop(self.PERMISSIONS_TRANSFER_KW, False)
         super().__init__(**kwargs)
 
     @property
@@ -109,8 +105,6 @@ class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin):
                     self.ab_file_name_col: {"type": "string"},
                 },
             }
-        elif self.use_permissions_transfer:
-            return self.stream_reader.file_permissions_schema
         else:
             return super()._filter_schema_invalid_properties(configured_catalog_json_schema)
 
@@ -193,29 +187,6 @@ class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin):
                         yield stream_data_to_airbyte_message(
                             self.name, record, is_file_transfer_message=True
                         )
-                elif self.use_permissions_transfer:
-                    try:
-                        permissions_record = self.stream_reader.get_file_acl_permissions(
-                            file, logger=self.logger
-                        )
-                        permissions_record = self.transform_record(
-                            permissions_record, file, file_datetime_string
-                        )
-                        yield stream_data_to_airbyte_message(
-                            self.name, permissions_record, is_file_transfer_message=False
-                        )
-                    except Exception as e:
-                        self.logger.error(
-                            f"Failed to retrieve permissions for file {file.uri}: {str(e)}"
-                        )
-                        yield AirbyteMessage(
-                            type=MessageType.LOG,
-                            log=AirbyteLogMessage(
-                                level=Level.ERROR,
-                                message=f"Error retrieving files permissions: stream={self.name} file={file.uri}",
-                                stack_trace=traceback.format_exc(),
-                            ),
-                        )
                 else:
                     for record in parser.parse_records(
                         self.config, file, self.stream_reader, self.logger, schema
@@ -313,8 +284,6 @@ class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin):
     def _get_raw_json_schema(self) -> JsonSchema:
         if self.use_file_transfer:
             return file_transfer_schema
-        elif self.use_permissions_transfer:
-            return self.stream_reader.file_permissions_schema
         elif self.config.input_schema:
             return self.config.get_input_schema()  # type: ignore
         elif self.config.schemaless:

airbyte_cdk-6.26.0.dev4110/airbyte_cdk/sources/file_based/stream/permissions_file_based_stream.py

@@ -0,0 +1,54 @@
+#
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+#
+
+import traceback
+from typing import Any, Dict, Iterable
+
+from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, Level
+from airbyte_cdk.models import Type as MessageType
+from airbyte_cdk.sources.file_based.stream import DefaultFileBasedStream
+from airbyte_cdk.sources.file_based.types import StreamSlice
+from airbyte_cdk.sources.streams.core import JsonSchema
+from airbyte_cdk.sources.utils.record_helper import stream_data_to_airbyte_message
+
+
+class PermissionsFileBasedStream(DefaultFileBasedStream):
+    """
+    The permissions stream, stream_reader on source handles logic for schemas and ACLs permissions.
+    """
+
+    def _filter_schema_invalid_properties(
+        self, configured_catalog_json_schema: Dict[str, Any]
+    ) -> Dict[str, Any]:
+        return self.stream_reader.file_permissions_schema
+
+    def read_records_from_slice(self, stream_slice: StreamSlice) -> Iterable[AirbyteMessage]:
+        """
+        Yield permissions records from all remote files
+        """
+        for file in stream_slice["files"]:
+            file_datetime_string = file.last_modified.strftime(self.DATE_TIME_FORMAT)
+            try:
+                permissions_record = self.stream_reader.get_file_acl_permissions(
+                    file, logger=self.logger
+                )
+                permissions_record = self.transform_record(
+                    permissions_record, file, file_datetime_string
+                )
+                yield stream_data_to_airbyte_message(
+                    self.name, permissions_record, is_file_transfer_message=False
+                )
+            except Exception as e:
+                self.logger.error(f"Failed to retrieve permissions for file {file.uri}: {str(e)}")
+                yield AirbyteMessage(
+                    type=MessageType.LOG,
+                    log=AirbyteLogMessage(
+                        level=Level.ERROR,
+                        message=f"Error retrieving files permissions: stream={self.name} file={file.uri}",
+                        stack_trace=traceback.format_exc(),
+                    ),
+                )
+
+    def _get_raw_json_schema(self) -> JsonSchema:
+        return self.stream_reader.file_permissions_schema

{airbyte_cdk-6.26.0.dev4109 → airbyte_cdk-6.26.0.dev4110}/pyproject.toml

@@ -23,7 +23,7 @@ classifiers = [
 keywords = ["airbyte", "connector-development-kit", "cdk"]
 
 # Python CDK uses dynamic versioning: https://github.com/mtkennerly/poetry-dynamic-versioning
-version = "6.26.0.dev04109"  # Version will be calculated dynamically.
+version = "6.26.0.dev04110"  # Version will be calculated dynamically.
 
 [tool.poetry-dynamic-versioning]
 enable = false