airbyte-cdk 6.26.0.dev4109__py3-none-any.whl → 6.26.0.dev4110__py3-none-any.whl

airbyte_cdk/sources/file_based/file_based_source.py

@@ -67,6 +67,9 @@ from airbyte_cdk.sources.file_based.stream.concurrent.cursor import (
     FileBasedFinalStateCursor,
 )
 from airbyte_cdk.sources.file_based.stream.cursor import AbstractFileBasedCursor
+from airbyte_cdk.sources.file_based.stream.permissions_file_based_stream import (
+    PermissionsFileBasedStream,
+)
 from airbyte_cdk.sources.message.repository import InMemoryMessageRepository, MessageRepository
 from airbyte_cdk.sources.streams import Stream
 from airbyte_cdk.sources.streams.concurrent.cursor import CursorField
@@ -257,7 +260,7 @@ class FileBasedSource(ConcurrentSourceAdapter, ABC):
                         message_repository=self.message_repository,
                     )
                     stream = FileBasedStreamFacade.create_from_stream(
-                        stream=self._make_default_stream(
+                        stream=self._make_file_based_stream(
                             stream_config=stream_config,
                             cursor=cursor,
                             parsed_config=parsed_config,
@@ -288,7 +291,7 @@ class FileBasedSource(ConcurrentSourceAdapter, ABC):
                         CursorField(DefaultFileBasedStream.ab_last_mod_col),
                     )
                     stream = FileBasedStreamFacade.create_from_stream(
-                        stream=self._make_default_stream(
+                        stream=self._make_file_based_stream(
                             stream_config=stream_config,
                             cursor=cursor,
                             parsed_config=parsed_config,
@@ -300,7 +303,7 @@ class FileBasedSource(ConcurrentSourceAdapter, ABC):
                     )
                 else:
                     cursor = self.cursor_cls(stream_config)
-                    stream = self._make_default_stream(
+                    stream = self._make_file_based_stream(
                         stream_config=stream_config,
                         cursor=cursor,
                         parsed_config=parsed_config,
@@ -334,9 +337,34 @@ class FileBasedSource(ConcurrentSourceAdapter, ABC):
             cursor=cursor,
             use_file_transfer=use_file_transfer(parsed_config),
             preserve_directory_structure=preserve_directory_structure(parsed_config),
-            use_permissions_transfer=use_permissions_transfer(parsed_config),
         )
 
+    def _make_permissions_stream(
+        self, stream_config: FileBasedStreamConfig, cursor: Optional[AbstractFileBasedCursor]
+    ) -> AbstractFileBasedStream:
+        return PermissionsFileBasedStream(
+            config=stream_config,
+            catalog_schema=self.stream_schemas.get(stream_config.name),
+            stream_reader=self.stream_reader,
+            availability_strategy=self.availability_strategy,
+            discovery_policy=self.discovery_policy,
+            parsers=self.parsers,
+            validation_policy=self._validate_and_get_validation_policy(stream_config),
+            errors_collector=self.errors_collector,
+            cursor=cursor,
+        )
+
+    def _make_file_based_stream(
+        self,
+        stream_config: FileBasedStreamConfig,
+        cursor: Optional[AbstractFileBasedCursor],
+        parsed_config: AbstractFileBasedSpec,
+    ) -> AbstractFileBasedStream:
+        if use_permissions_transfer(parsed_config):
+            return self._make_permissions_stream(stream_config, cursor)
+        else:
+            return self._make_default_stream(stream_config, cursor, parsed_config)
+
     def _make_identities_stream(
         self,
     ) -> Stream:

airbyte_cdk/sources/file_based/stream/default_file_based_stream.py

@@ -47,7 +47,6 @@ class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin):
 
     FILE_TRANSFER_KW = "use_file_transfer"
     PRESERVE_DIRECTORY_STRUCTURE_KW = "preserve_directory_structure"
-    PERMISSIONS_TRANSFER_KW = "use_permissions_transfer"
     FILES_KEY = "files"
     DATE_TIME_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ"
     ab_last_mod_col = "_ab_source_file_last_modified"
@@ -57,7 +56,6 @@ class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin):
     airbyte_columns = [ab_last_mod_col, ab_file_name_col]
     use_file_transfer = False
     preserve_directory_structure = True
-    use_permissions_transfer = False
 
     def __init__(self, **kwargs: Any):
         if self.FILE_TRANSFER_KW in kwargs:
@@ -66,8 +64,6 @@ class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin):
             self.preserve_directory_structure = kwargs.pop(
                 self.PRESERVE_DIRECTORY_STRUCTURE_KW, True
             )
-        if self.PERMISSIONS_TRANSFER_KW in kwargs:
-            self.use_permissions_transfer = kwargs.pop(self.PERMISSIONS_TRANSFER_KW, False)
         super().__init__(**kwargs)
 
     @property
@@ -109,8 +105,6 @@ class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin):
                     self.ab_file_name_col: {"type": "string"},
                 },
             }
-        elif self.use_permissions_transfer:
-            return self.stream_reader.file_permissions_schema
         else:
             return super()._filter_schema_invalid_properties(configured_catalog_json_schema)
 
@@ -193,29 +187,6 @@ class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin):
                         yield stream_data_to_airbyte_message(
                             self.name, record, is_file_transfer_message=True
                         )
-                elif self.use_permissions_transfer:
-                    try:
-                        permissions_record = self.stream_reader.get_file_acl_permissions(
-                            file, logger=self.logger
-                        )
-                        permissions_record = self.transform_record(
-                            permissions_record, file, file_datetime_string
-                        )
-                        yield stream_data_to_airbyte_message(
-                            self.name, permissions_record, is_file_transfer_message=False
-                        )
-                    except Exception as e:
-                        self.logger.error(
-                            f"Failed to retrieve permissions for file {file.uri}: {str(e)}"
-                        )
-                        yield AirbyteMessage(
-                            type=MessageType.LOG,
-                            log=AirbyteLogMessage(
-                                level=Level.ERROR,
-                                message=f"Error retrieving files permissions: stream={self.name} file={file.uri}",
-                                stack_trace=traceback.format_exc(),
-                            ),
-                        )
                 else:
                     for record in parser.parse_records(
                         self.config, file, self.stream_reader, self.logger, schema
@@ -313,8 +284,6 @@ class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin):
     def _get_raw_json_schema(self) -> JsonSchema:
         if self.use_file_transfer:
             return file_transfer_schema
-        elif self.use_permissions_transfer:
-            return self.stream_reader.file_permissions_schema
         elif self.config.input_schema:
             return self.config.get_input_schema()  # type: ignore
         elif self.config.schemaless:

airbyte_cdk/sources/file_based/stream/permissions_file_based_stream.py

@@ -0,0 +1,54 @@
+#
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+#
+
+import traceback
+from typing import Any, Dict, Iterable
+
+from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, Level
+from airbyte_cdk.models import Type as MessageType
+from airbyte_cdk.sources.file_based.stream import DefaultFileBasedStream
+from airbyte_cdk.sources.file_based.types import StreamSlice
+from airbyte_cdk.sources.streams.core import JsonSchema
+from airbyte_cdk.sources.utils.record_helper import stream_data_to_airbyte_message
+
+
+class PermissionsFileBasedStream(DefaultFileBasedStream):
+    """
+    The permissions stream, stream_reader on source handles logic for schemas and ACLs permissions.
+    """
+
+    def _filter_schema_invalid_properties(
+        self, configured_catalog_json_schema: Dict[str, Any]
+    ) -> Dict[str, Any]:
+        return self.stream_reader.file_permissions_schema
+
+    def read_records_from_slice(self, stream_slice: StreamSlice) -> Iterable[AirbyteMessage]:
+        """
+        Yield permissions records from all remote files
+        """
+        for file in stream_slice["files"]:
+            file_datetime_string = file.last_modified.strftime(self.DATE_TIME_FORMAT)
+            try:
+                permissions_record = self.stream_reader.get_file_acl_permissions(
+                    file, logger=self.logger
+                )
+                permissions_record = self.transform_record(
+                    permissions_record, file, file_datetime_string
+                )
+                yield stream_data_to_airbyte_message(
+                    self.name, permissions_record, is_file_transfer_message=False
+                )
+            except Exception as e:
+                self.logger.error(f"Failed to retrieve permissions for file {file.uri}: {str(e)}")
+                yield AirbyteMessage(
+                    type=MessageType.LOG,
+                    log=AirbyteLogMessage(
+                        level=Level.ERROR,
+                        message=f"Error retrieving files permissions: stream={self.name} file={file.uri}",
+                        stack_trace=traceback.format_exc(),
+                    ),
+                )
+
+    def _get_raw_json_schema(self) -> JsonSchema:
+        return self.stream_reader.file_permissions_schema

{airbyte_cdk-6.26.0.dev4109.dist-info → airbyte_cdk-6.26.0.dev4110.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 6.26.0.dev4109
+Version: 6.26.0.dev4110
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://airbyte.com
 License: MIT

{airbyte_cdk-6.26.0.dev4109.dist-info → airbyte_cdk-6.26.0.dev4110.dist-info}/RECORD

@@ -214,7 +214,7 @@ airbyte_cdk/sources/file_based/discovery_policy/__init__.py,sha256=gl3ey6mZbyfra
 airbyte_cdk/sources/file_based/discovery_policy/abstract_discovery_policy.py,sha256=dCfXX529Rd5rtopg4VeEgTPJjFtqjtjzPq6LCw18Wt0,605
 airbyte_cdk/sources/file_based/discovery_policy/default_discovery_policy.py,sha256=-xujTidtrq6HC00WKbjQh1CZdT5LMuzkp5BLjqDmfTY,1007
 airbyte_cdk/sources/file_based/exceptions.py,sha256=WP0qkG6fpWoBpOyyicgp5YNE393VWyegq5qSy0v4QtM,7362
-airbyte_cdk/sources/file_based/file_based_source.py,sha256=zXZbLaQ-XW9D2l-k5N3466bOLYvwfRzCEDpih03NA2s,17603
+airbyte_cdk/sources/file_based/file_based_source.py,sha256=nWWTWfwp6SSrcRcJZOWMMfu2Joi-PFZhHlreni-pVR4,18778
 airbyte_cdk/sources/file_based/file_based_stream_reader.py,sha256=z_DNWGiKUQCgYO9vsOy40aNbTsU_QbOuSHMCGegNLuw,8409
 airbyte_cdk/sources/file_based/file_types/__init__.py,sha256=blCLn0-2LC-ZdgcNyDEhqM2RiUvEjEBh-G4-t32ZtuM,1268
 airbyte_cdk/sources/file_based/file_types/avro_parser.py,sha256=XNx-JC-sgzH9u3nOJ2M59FxBXvtig8LN6BIkeDOavZA,10858
@@ -241,8 +241,9 @@ airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_final_state_c
 airbyte_cdk/sources/file_based/stream/cursor/__init__.py,sha256=MhFB5hOo8sjwvCh8gangaymdg3EJWYt_72brFOZt068,191
 airbyte_cdk/sources/file_based/stream/cursor/abstract_file_based_cursor.py,sha256=om-x3gZFPgWDpi15S9RxZmR36VHnk8sytgN6LlBQhAw,1934
 airbyte_cdk/sources/file_based/stream/cursor/default_file_based_cursor.py,sha256=VGV7xLyBribuBMVrXtO1xqkWJD86bl7yhXtjnwLMohM,7051
-airbyte_cdk/sources/file_based/stream/default_file_based_stream.py,sha256=hqAbm61DZzXn39BwJReiT5l8Kcd6MkOTOTXzW-4j624,19728
+airbyte_cdk/sources/file_based/stream/default_file_based_stream.py,sha256=XLU5cNqQ-5mj243gNzMyXtm_oCtg1ORyoqbCsUo9Dn4,18044
 airbyte_cdk/sources/file_based/stream/identities_stream.py,sha256=KuhIVV0NFc6RhtzrAvswYvAO_dgPSYkWgOfqr4NAgQw,1765
+airbyte_cdk/sources/file_based/stream/permissions_file_based_stream.py,sha256=QTB3G-Bs6nK37W1yGAygHnUaNzVbF7KUuJyhkdMNKYo,2261
 airbyte_cdk/sources/file_based/types.py,sha256=INxG7OPnkdUP69oYNKMAbwhvV1AGvLRHs1J6pIia2FI,218
 airbyte_cdk/sources/http_config.py,sha256=OBZeuyFilm6NlDlBhFQvHhTWabEvZww6OHDIlZujIS0,730
 airbyte_cdk/sources/http_logger.py,sha256=l_1fk5YwdonZ1wvAsTwjj6d36fj2WrVraIAMj5jTQdM,1575
@@ -355,9 +356,9 @@ airbyte_cdk/utils/slice_hasher.py,sha256=EDxgROHDbfG-QKQb59m7h_7crN1tRiawdf5uU7G
 airbyte_cdk/utils/spec_schema_transformations.py,sha256=-5HTuNsnDBAhj-oLeQXwpTGA0HdcjFOf2zTEMUTTg_Y,816
 airbyte_cdk/utils/stream_status_utils.py,sha256=ZmBoiy5HVbUEHAMrUONxZvxnvfV9CesmQJLDTAIWnWw,1171
 airbyte_cdk/utils/traced_exception.py,sha256=C8uIBuCL_E4WnBAOPSxBicD06JAldoN9fGsQDp463OY,6292
-airbyte_cdk-6.26.0.dev4109.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
-airbyte_cdk-6.26.0.dev4109.dist-info/LICENSE_SHORT,sha256=aqF6D1NcESmpn-cqsxBtszTEnHKnlsp8L4x9wAh3Nxg,55
-airbyte_cdk-6.26.0.dev4109.dist-info/METADATA,sha256=OY2eDa-8oZYg5jjBx3kWiDnDXAdqkWkLEsUryerMYpA,6018
-airbyte_cdk-6.26.0.dev4109.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-airbyte_cdk-6.26.0.dev4109.dist-info/entry_points.txt,sha256=fj-e3PAQvsxsQzyyq8UkG1k8spunWnD4BAH2AwlR6NM,95
-airbyte_cdk-6.26.0.dev4109.dist-info/RECORD,,
+airbyte_cdk-6.26.0.dev4110.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
+airbyte_cdk-6.26.0.dev4110.dist-info/LICENSE_SHORT,sha256=aqF6D1NcESmpn-cqsxBtszTEnHKnlsp8L4x9wAh3Nxg,55
+airbyte_cdk-6.26.0.dev4110.dist-info/METADATA,sha256=M8Mus_j1GU55csZYUEiFBsJ9sUcXMQYZGjP65pyv3gE,6018
+airbyte_cdk-6.26.0.dev4110.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+airbyte_cdk-6.26.0.dev4110.dist-info/entry_points.txt,sha256=fj-e3PAQvsxsQzyyq8UkG1k8spunWnD4BAH2AwlR6NM,95
+airbyte_cdk-6.26.0.dev4110.dist-info/RECORD,,