airbyte-cdk 6.38.2__tar.gz → 6.38.3.dev4100__tar.gz

{airbyte_cdk-6.38.2 → airbyte_cdk-6.38.3.dev4100}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 6.38.2
+Version: 6.38.3.dev4100
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://airbyte.com
 License: MIT

{airbyte_cdk-6.38.2 → airbyte_cdk-6.38.3.dev4100}/airbyte_cdk/entrypoint.py

@@ -22,7 +22,7 @@ from requests import PreparedRequest, Response, Session
 
 from airbyte_cdk.connector import TConfig
 from airbyte_cdk.exception_handler import init_uncaught_exception_handler
-from airbyte_cdk.logger import init_logger
+from airbyte_cdk.logger import PRINT_BUFFER, init_logger
 from airbyte_cdk.models import (
     AirbyteConnectionStatus,
     AirbyteMessage,
@@ -337,11 +337,11 @@ def launch(source: Source, args: List[str]) -> None:
     parsed_args = source_entrypoint.parse_args(args)
     # temporarily removes the PrintBuffer because we're seeing weird print behavior for concurrent syncs
     # Refer to: https://github.com/airbytehq/oncall/issues/6235
-    # with PrintBuffer():
-    for message in source_entrypoint.run(parsed_args):
-        # simply printing is creating issues for concurrent CDK as Python uses different two instructions to print: one for the message and
-        # the other for the break line. Adding `\n` to the message ensure that both are printed at the same time
-        print(f"{message}\n", end="", flush=True)
+    with PRINT_BUFFER:
+        for message in source_entrypoint.run(parsed_args):
+            # simply printing is creating issues for concurrent CDK as Python uses different two instructions to print: one for the message and
+            # the other for the break line. Adding `\n` to the message ensure that both are printed at the same time
+            print(f"{message}\n", end="")
 
 
 def _init_internal_request_filter() -> None:

{airbyte_cdk-6.38.2 → airbyte_cdk-6.38.3.dev4100}/airbyte_cdk/logger.py

@@ -16,8 +16,11 @@ from airbyte_cdk.models import (
     Level,
     Type,
 )
+from airbyte_cdk.utils import PrintBuffer
 from airbyte_cdk.utils.airbyte_secrets_utils import filter_secrets
 
+PRINT_BUFFER = PrintBuffer(flush_interval=0.1)
+
 LOGGING_CONFIG = {
     "version": 1,
     "disable_existing_loggers": False,
@@ -27,7 +30,7 @@ LOGGING_CONFIG = {
     "handlers": {
         "console": {
             "class": "logging.StreamHandler",
-            "stream": "ext://sys.stdout",
+            "stream": PRINT_BUFFER,
             "formatter": "airbyte",
         },
     },

{airbyte_cdk-6.38.2 → airbyte_cdk-6.38.3.dev4100}/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py

@@ -1476,7 +1476,19 @@ class ModelToComponentFactory:
         try:
             module_ref = importlib.import_module(module_name_full)
         except ModuleNotFoundError as e:
-            raise ValueError(f"Could not load module `{module_name_full}`.") from e
+            if split[0] == "source_declarative_manifest":
+                # During testing, the modules containing the custom components are not moved to source_declarative_manifest. In order to run the test, add the source folder to your PYTHONPATH or add it runtime using sys.path.append
+                try:
+                    import os
+
+                    module_name_with_source_declarative_manifest = ".".join(split[1:-1])
+                    module_ref = importlib.import_module(
+                        module_name_with_source_declarative_manifest
+                    )
+                except ModuleNotFoundError:
+                    raise ValueError(f"Could not load module `{module_name_full}`.") from e
+            else:
+                raise ValueError(f"Could not load module `{module_name_full}`.") from e
 
         try:
             return getattr(module_ref, class_name)

{airbyte_cdk-6.38.2 → airbyte_cdk-6.38.3.dev4100}/airbyte_cdk/sources/declarative/yaml_declarative_source.py

@@ -39,13 +39,18 @@ class YamlDeclarativeSource(ConcurrentDeclarativeSource[List[AirbyteStateMessage
         )
 
     def _read_and_parse_yaml_file(self, path_to_yaml_file: str) -> ConnectionDefinition:
-        package = self.__class__.__module__.split(".")[0]
+        try:
+            # For testing purposes, we want to allow to just pass a file
+            with open(path_to_yaml_file, "r") as f:
+                return yaml.safe_load(f)  # type: ignore  # we assume the yaml represents a ConnectionDefinition
+        except FileNotFoundError:
+            # Running inside the container, the working directory during an operation is not structured the same as the static files
+            package = self.__class__.__module__.split(".")[0]
 
-        yaml_config = pkgutil.get_data(package, path_to_yaml_file)
-        if yaml_config:
-            decoded_yaml = yaml_config.decode()
-            return self._parse(decoded_yaml)
-        else:
+            yaml_config = pkgutil.get_data(package, path_to_yaml_file)
+            if yaml_config:
+                decoded_yaml = yaml_config.decode()
+                return self._parse(decoded_yaml)
             return {}
 
     def _emit_manifest_debug_message(self, extra_args: dict[str, Any]) -> None:

{airbyte_cdk-6.38.2 → airbyte_cdk-6.38.3.dev4100}/airbyte_cdk/sources/file_based/file_based_source.py

@@ -48,6 +48,9 @@ from airbyte_cdk.sources.file_based.exceptions import (
     FileBasedErrorsCollector,
     FileBasedSourceError,
 )
+from airbyte_cdk.sources.file_based.file_based_stream_permissions_reader import (
+    AbstractFileBasedStreamPermissionsReader,
+)
 from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader
 from airbyte_cdk.sources.file_based.file_types import default_parsers
 from airbyte_cdk.sources.file_based.file_types.file_type_parser import FileTypeParser
@@ -100,8 +103,10 @@ class FileBasedSource(ConcurrentSourceAdapter, ABC):
         cursor_cls: Type[
             Union[AbstractConcurrentFileBasedCursor, AbstractFileBasedCursor]
         ] = FileBasedConcurrentCursor,
+        stream_permissions_reader: Optional[AbstractFileBasedStreamPermissionsReader] = None,
     ):
         self.stream_reader = stream_reader
+        self.stream_permissions_reader = stream_permissions_reader
         self.spec_class = spec_class
         self.config = config
         self.catalog = catalog
@@ -337,9 +342,23 @@ class FileBasedSource(ConcurrentSourceAdapter, ABC):
             preserve_directory_structure=preserve_directory_structure(parsed_config),
         )
 
+    def _ensure_permissions_reader_available(self) -> None:
+        """
+        Validates that a stream permissions reader is available.
+        Raises a ValueError if the reader is not provided.
+        """
+        if not self.stream_permissions_reader:
+            raise ValueError(
+                "Stream permissions reader is required for streams that use permissions transfer mode."
+            )
+
     def _make_permissions_stream(
         self, stream_config: FileBasedStreamConfig, cursor: Optional[AbstractFileBasedCursor]
     ) -> AbstractFileBasedStream:
+        """
+        Creates a stream that reads permissions from files.
+        """
+        self._ensure_permissions_reader_available()
         return PermissionsFileBasedStream(
             config=stream_config,
             catalog_schema=self.stream_schemas.get(stream_config.name),
@@ -350,6 +369,7 @@ class FileBasedSource(ConcurrentSourceAdapter, ABC):
             validation_policy=self._validate_and_get_validation_policy(stream_config),
             errors_collector=self.errors_collector,
             cursor=cursor,
+            stream_permissions_reader=self.stream_permissions_reader,  # type: ignore
         )
 
     def _make_file_based_stream(
@@ -370,9 +390,10 @@ class FileBasedSource(ConcurrentSourceAdapter, ABC):
     def _make_identities_stream(
         self,
     ) -> Stream:
+        self._ensure_permissions_reader_available()
         return FileIdentitiesStream(
             catalog_schema=self.stream_schemas.get(FileIdentitiesStream.IDENTITIES_STREAM_NAME),
-            stream_reader=self.stream_reader,
+            stream_permissions_reader=self.stream_permissions_reader,  # type: ignore
             discovery_policy=self.discovery_policy,
             errors_collector=self.errors_collector,
         )

airbyte_cdk-6.38.3.dev4100/airbyte_cdk/sources/file_based/file_based_stream_permissions_reader.py

@@ -0,0 +1,109 @@
+#
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+#
+
+import logging
+from abc import ABC, abstractmethod
+from typing import Any, Dict, Iterable
+
+from airbyte_cdk.sources.file_based.remote_file import RemoteFile
+
+
+class AbstractFileBasedStreamPermissionsReader(ABC):
+    """
+    This class is responsible for reading file permissions and Identities from a source.
+    """
+
+    @abstractmethod
+    def get_file_acl_permissions(self, file: RemoteFile, logger: logging.Logger) -> Dict[str, Any]:
+        """
+        This function should return the allow list for a given file, i.e. the list of all identities and their permission levels associated with it
+
+        e.g.
+        def get_file_acl_permissions(self, file: RemoteFile, logger: logging.Logger):
+            api_conn = some_api.conn(credentials=SOME_CREDENTIALS)
+            result = api_conn.get_file_permissions_info(file.id)
+            return MyPermissionsModel(
+                id=result["id"],
+                access_control_list = result["access_control_list"],
+                is_public = result["is_public"],
+                ).dict()
+        """
+        raise NotImplementedError(
+            f"{self.__class__.__name__} does not implement get_file_acl_permissions(). To support ACL permissions, implement this method and update file_permissions_schema."
+        )
+
+    @abstractmethod
+    def load_identity_groups(self, logger: logging.Logger) -> Iterable[Dict[str, Any]]:
+        """
+        This function should return the Identities in a determined "space" or "domain" where the file metadata (ACLs) are fetched and ACLs items (Identities) exists.
+
+        e.g.
+        def load_identity_groups(self, logger: logging.Logger) -> Dict[str, Any]:
+            api_conn = some_api.conn(credentials=SOME_CREDENTIALS)
+            users_api = api_conn.users()
+            groups_api = api_conn.groups()
+            members_api = self.google_directory_service.members()
+            for user in users_api.list():
+                yield my_identity_model(id=user.id, name=user.name, email_address=user.email, type="user").dict()
+            for group in groups_api.list():
+                group_obj = my_identity_model(id=group.id, name=groups.name, email_address=user.email, type="group").dict()
+                for member in members_api.list(group=group):
+                    group_obj.member_email_addresses = group_obj.member_email_addresses or []
+                    group_obj.member_email_addresses.append(member.email)
+                yield group_obj.dict()
+        """
+        raise NotImplementedError(
+            f"{self.__class__.__name__} does not implement load_identity_groups(). To support identities, implement this method and update identities_schema."
+        )
+
+    @property
+    @abstractmethod
+    def file_permissions_schema(self) -> Dict[str, Any]:
+        """
+        This function should return the permissions schema for file permissions stream.
+
+        e.g.
+        def file_permissions_schema(self) -> Dict[str, Any]:
+            # you can also follow the patter we have for python connectors and have a json file and read from there e.g. schemas/identities.json
+            return {
+                  "type": "object",
+                  "properties": {
+                    "id": { "type": "string" },
+                    "file_path": { "type": "string" },
+                    "access_control_list": {
+                      "type": "array",
+                      "items": { "type": "string" }
+                    },
+                    "publicly_accessible": { "type": "boolean" }
+                  }
+                }
+        """
+        raise NotImplementedError(
+            f"{self.__class__.__name__} does not implement file_permissions_schema, please return json schema for your permissions streams."
+        )
+
+    @property
+    @abstractmethod
+    def identities_schema(self) -> Dict[str, Any]:
+        """
+        This function should return the identities schema for file identity stream.
+
+        e.g.
+        def identities_schema(self) -> Dict[str, Any]:
+            # you can also follow the patter we have for python connectors and have a json file and read from there e.g. schemas/identities.json
+            return {
+              "type": "object",
+              "properties": {
+                "id": { "type": "string" },
+                "remote_id": { "type": "string" },
+                "name": { "type": ["null", "string"] },
+                "email_address": { "type": ["null", "string"] },
+                "member_email_addresses": { "type": ["null", "array"] },
+                "type": { "type": "string" },
+              }
+            }
+        """
+        raise NotImplementedError(
+            f"{self.__class__.__name__} does not implement identities_schema, please return json schema for your identities stream."
+        )

{airbyte_cdk-6.38.2 → airbyte_cdk-6.38.3.dev4100}/airbyte_cdk/sources/file_based/file_based_stream_reader.py

@@ -184,97 +184,3 @@ class AbstractFileBasedStreamReader(ABC):
         makedirs(path.dirname(local_file_path), exist_ok=True)
         absolute_file_path = path.abspath(local_file_path)
         return [file_relative_path, local_file_path, absolute_file_path]
-
-    @abstractmethod
-    def get_file_acl_permissions(self, file: RemoteFile, logger: logging.Logger) -> Dict[str, Any]:
-        """
-        This function should return the allow list for a given file, i.e. the list of all identities and their permission levels associated with it
-
-        e.g.
-        def get_file_acl_permissions(self, file: RemoteFile, logger: logging.Logger):
-            api_conn = some_api.conn(credentials=SOME_CREDENTIALS)
-            result = api_conn.get_file_permissions_info(file.id)
-            return MyPermissionsModel(
-                id=result["id"],
-                access_control_list = result["access_control_list"],
-                is_public = result["is_public"],
-                ).dict()
-        """
-        raise NotImplementedError(
-            f"{self.__class__.__name__} does not implement get_file_acl_permissions(). To support ACL permissions, implement this method and update file_permissions_schema."
-        )
-
-    @abstractmethod
-    def load_identity_groups(self, logger: logging.Logger) -> Iterable[Dict[str, Any]]:
-        """
-        This function should return the Identities in a determined "space" or "domain" where the file metadata (ACLs) are fetched and ACLs items (Identities) exists.
-
-        e.g.
-        def load_identity_groups(self, logger: logging.Logger) -> Dict[str, Any]:
-            api_conn = some_api.conn(credentials=SOME_CREDENTIALS)
-            users_api = api_conn.users()
-            groups_api = api_conn.groups()
-            members_api = self.google_directory_service.members()
-            for user in users_api.list():
-                yield my_identity_model(id=user.id, name=user.name, email_address=user.email, type="user").dict()
-            for group in groups_api.list():
-                group_obj = my_identity_model(id=group.id, name=groups.name, email_address=user.email, type="group").dict()
-                for member in members_api.list(group=group):
-                    group_obj.member_email_addresses = group_obj.member_email_addresses or []
-                    group_obj.member_email_addresses.append(member.email)
-                yield group_obj.dict()
-        """
-        raise NotImplementedError(
-            f"{self.__class__.__name__} does not implement load_identity_groups(). To support identities, implement this method and update identities_schema."
-        )
-
-    @property
-    @abstractmethod
-    def file_permissions_schema(self) -> Dict[str, Any]:
-        """
-        This function should return the permissions schema for file permissions stream.
-
-        e.g.
-        def file_permissions_schema(self) -> Dict[str, Any]:
-            # you can also follow the patter we have for python connectors and have a json file and read from there e.g. schemas/identities.json
-            return {
-                  "type": "object",
-                  "properties": {
-                    "id": { "type": "string" },
-                    "file_path": { "type": "string" },
-                    "access_control_list": {
-                      "type": "array",
-                      "items": { "type": "string" }
-                    },
-                    "publicly_accessible": { "type": "boolean" }
-                  }
-                }
-        """
-        raise NotImplementedError(
-            f"{self.__class__.__name__} does not implement file_permissions_schema, please return json schema for your permissions streams."
-        )
-
-    @property
-    @abstractmethod
-    def identities_schema(self) -> Dict[str, Any]:
-        """
-        This function should return the identities schema for file identity stream.
-
-        e.g.
-        def identities_schema(self) -> Dict[str, Any]:
-            # you can also follow the patter we have for python connectors and have a json file and read from there e.g. schemas/identities.json
-            return {
-              "type": "object",
-              "properties": {
-                "id": { "type": "string" },
-                "remote_id": { "type": "string" },
-                "name": { "type": ["null", "string"] },
-                "email_address": { "type": ["null", "string"] },
-                "member_email_addresses": { "type": ["null", "array"] },
-                "type": { "type": "string" },
-              }
-            }
-        """
-        raise NotImplementedError(
-            f"{self.__class__.__name__} does not implement identities_schema, please return json schema for your identities stream."
-        )

{airbyte_cdk-6.38.2 → airbyte_cdk-6.38.3.dev4100}/airbyte_cdk/sources/file_based/stream/identities_stream.py

@@ -8,7 +8,9 @@ from typing import Any, Dict, Iterable, Mapping, MutableMapping, Optional
 from airbyte_cdk.sources.file_based.config.file_based_stream_config import PrimaryKeyType
 from airbyte_cdk.sources.file_based.discovery_policy import AbstractDiscoveryPolicy
 from airbyte_cdk.sources.file_based.exceptions import FileBasedErrorsCollector
-from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader
+from airbyte_cdk.sources.file_based.file_based_stream_permissions_reader import (
+    AbstractFileBasedStreamPermissionsReader,
+)
 from airbyte_cdk.sources.streams.core import JsonSchema
 from airbyte_cdk.sources.streams.permissions.identities_stream import IdentitiesStream
 
@@ -24,13 +26,13 @@ class FileIdentitiesStream(IdentitiesStream):
     def __init__(
         self,
         catalog_schema: Optional[Mapping[str, Any]],
-        stream_reader: AbstractFileBasedStreamReader,
+        stream_permissions_reader: AbstractFileBasedStreamPermissionsReader,
         discovery_policy: AbstractDiscoveryPolicy,
         errors_collector: FileBasedErrorsCollector,
     ) -> None:
         super().__init__()
         self.catalog_schema = catalog_schema
-        self.stream_reader = stream_reader
+        self.stream_permissions_reader = stream_permissions_reader
         self._discovery_policy = discovery_policy
         self.errors_collector = errors_collector
         self._cursor: MutableMapping[str, Any] = {}
@@ -40,8 +42,8 @@ class FileIdentitiesStream(IdentitiesStream):
         return None
 
     def load_identity_groups(self) -> Iterable[Dict[str, Any]]:
-        return self.stream_reader.load_identity_groups(logger=self.logger)
+        return self.stream_permissions_reader.load_identity_groups(logger=self.logger)
 
     @cache
     def get_json_schema(self) -> JsonSchema:
-        return self.stream_reader.identities_schema
+        return self.stream_permissions_reader.identities_schema

{airbyte_cdk-6.38.2 → airbyte_cdk-6.38.3.dev4100}/airbyte_cdk/sources/file_based/stream/permissions_file_based_stream.py

@@ -7,6 +7,9 @@ from typing import Any, Dict, Iterable
 
 from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, Level
 from airbyte_cdk.models import Type as MessageType
+from airbyte_cdk.sources.file_based.file_based_stream_permissions_reader import (
+    AbstractFileBasedStreamPermissionsReader,
+)
 from airbyte_cdk.sources.file_based.stream import DefaultFileBasedStream
 from airbyte_cdk.sources.file_based.types import StreamSlice
 from airbyte_cdk.sources.streams.core import JsonSchema
@@ -26,10 +29,16 @@ class PermissionsFileBasedStream(DefaultFileBasedStream):
     and schema definition, while this class handles the streaming interface.
     """
 
+    def __init__(
+        self, stream_permissions_reader: AbstractFileBasedStreamPermissionsReader, **kwargs: Any
+    ):
+        super().__init__(**kwargs)
+        self.stream_permissions_reader = stream_permissions_reader
+
     def _filter_schema_invalid_properties(
         self, configured_catalog_json_schema: Dict[str, Any]
     ) -> Dict[str, Any]:
-        return self.stream_reader.file_permissions_schema
+        return self.stream_permissions_reader.file_permissions_schema
 
     def read_records_from_slice(self, stream_slice: StreamSlice) -> Iterable[AirbyteMessage]:
         """
@@ -40,7 +49,7 @@ class PermissionsFileBasedStream(DefaultFileBasedStream):
             no_permissions = False
             file_datetime_string = file.last_modified.strftime(self.DATE_TIME_FORMAT)
             try:
-                permissions_record = self.stream_reader.get_file_acl_permissions(
+                permissions_record = self.stream_permissions_reader.get_file_acl_permissions(
                     file, logger=self.logger
                 )
                 if not permissions_record:
@@ -82,4 +91,4 @@ class PermissionsFileBasedStream(DefaultFileBasedStream):
         Returns:
            The file permissions schema that defines the structure of permission records
         """
-        return self.stream_reader.file_permissions_schema
+        return self.stream_permissions_reader.file_permissions_schema

{airbyte_cdk-6.38.2 → airbyte_cdk-6.38.3.dev4100}/airbyte_cdk/sources/streams/concurrent/adapters.py

@@ -276,7 +276,7 @@ class StreamPartition(Partition):
     def read(self) -> Iterable[Record]:
         """
         Read messages from the stream.
-        If the StreamData is a Mapping, it will be converted to a Record.
+        If the StreamData is a Mapping or an AirbyteMessage of type RECORD, it will be converted to a Record.
         Otherwise, the message will be emitted on the message repository.
         """
         try:
@@ -292,6 +292,8 @@ class StreamPartition(Partition):
                 stream_slice=copy.deepcopy(self._slice),
                 stream_state=self._state,
             ):
+                # Noting we'll also need to support FileTransferRecordMessage if we want to support file-based connectors in this facade
+                # For now, file-based connectors have their own stream facade
                 if isinstance(record_data, Mapping):
                     data_to_return = dict(record_data)
                     self._stream.transformer.transform(
@@ -302,6 +304,12 @@ class StreamPartition(Partition):
                         stream_name=self.stream_name(),
                         associated_slice=self._slice,  # type: ignore [arg-type]
                     )
+                elif isinstance(record_data, AirbyteMessage) and record_data.record is not None:
+                    yield Record(
+                        data=record_data.record.data or {},
+                        stream_name=self.stream_name(),
+                        associated_slice=self._slice,  # type: ignore [arg-type]
+                    )
                 else:
                     self._message_repository.emit_message(record_data)
         except Exception as e:

{airbyte_cdk-6.38.2 → airbyte_cdk-6.38.3.dev4100}/pyproject.toml

@@ -23,7 +23,7 @@ classifiers = [
 keywords = ["airbyte", "connector-development-kit", "cdk"]
 
 # Python CDK uses dynamic versioning: https://github.com/mtkennerly/poetry-dynamic-versioning
-version = "6.38.2"  # Version will be calculated dynamically.
+version = "6.38.3.dev04100"  # Version will be calculated dynamically.
 
 [tool.poetry-dynamic-versioning]
 enable = false