airbyte-cdk 6.26.0.dev4107__py3-none-any.whl → 6.26.0.dev4109__py3-none-any.whl

airbyte_cdk/sources/file_based/file_based_source.py

@@ -58,7 +58,7 @@ from airbyte_cdk.sources.file_based.schema_validation_policies import (
 from airbyte_cdk.sources.file_based.stream import (
     AbstractFileBasedStream,
     DefaultFileBasedStream,
-    IdentitiesStream,
+    FileIdentities,
 )
 from airbyte_cdk.sources.file_based.stream.concurrent.adapters import FileBasedStreamFacade
 from airbyte_cdk.sources.file_based.stream.concurrent.cursor import (
@@ -67,7 +67,6 @@ from airbyte_cdk.sources.file_based.stream.concurrent.cursor import (
     FileBasedFinalStateCursor,
 )
 from airbyte_cdk.sources.file_based.stream.cursor import AbstractFileBasedCursor
-from airbyte_cdk.sources.file_based.stream.identities_stream import IDENTITIES_STREAM_NAME
 from airbyte_cdk.sources.message.repository import InMemoryMessageRepository, MessageRepository
 from airbyte_cdk.sources.streams import Stream
 from airbyte_cdk.sources.streams.concurrent.cursor import CursorField
@@ -169,8 +168,12 @@ class FileBasedSource(ConcurrentSourceAdapter, ABC):
         errors = []
         tracebacks = []
         for stream in streams:
-            if isinstance(stream, IdentitiesStream):
-                # Probably need to check identities endpoint/api access but will skip for now.
+            if isinstance(stream, FileIdentities):
+                identity = next(iter(stream.load_identity_groups()))
+                if not identity:
+                    errors.append(
+                        "Unable to get identities for current configuration, please check your credentials"
+                    )
                 continue
             if not isinstance(stream, AbstractFileBasedStream):
                 raise ValueError(f"Stream {stream} is not a file-based stream.")
@@ -337,8 +340,8 @@ class FileBasedSource(ConcurrentSourceAdapter, ABC):
     def _make_identities_stream(
         self,
     ) -> Stream:
-        return IdentitiesStream(
-            catalog_schema=self.stream_schemas.get(IDENTITIES_STREAM_NAME),
+        return FileIdentities(
+            catalog_schema=self.stream_schemas.get(FileIdentities.IDENTITIES_STREAM_NAME),
             stream_reader=self.stream_reader,
             discovery_policy=self.discovery_policy,
             errors_collector=self.errors_collector,

airbyte_cdk/sources/file_based/file_based_stream_reader.py

@@ -191,7 +191,9 @@ class AbstractFileBasedStreamReader(ABC):
         This is required for connectors that will support syncing
         ACL Permissions from files.
         """
-        ...
+        raise NotImplementedError(
+            f"{self.__class__.__name__} does not implement get_file_acl_permissions(). To support ACL permissions, implement this method and update file_permissions_schema."
+        )
 
     @abstractmethod
     def load_identity_groups(self, logger: logging.Logger) -> Iterable[Dict[str, Any]]:
@@ -199,4 +201,28 @@ class AbstractFileBasedStreamReader(ABC):
         This is required for connectors that will support syncing
         identities.
         """
-        ...
+        raise NotImplementedError(
+            f"{self.__class__.__name__} does not implement load_identity_groups(). To support identities, implement this method and update identities_schema."
+        )
+
+    @property
+    @abstractmethod
+    def file_permissions_schema(self) -> Dict[str, Any]:
+        """
+        This is required for connectors that will support syncing
+        ACL Permissions from files.
+        """
+        raise NotImplementedError(
+            f"{self.__class__.__name__} does not implement file_permissions_schema, please return json schema for your permissions streams."
+        )
+
+    @property
+    @abstractmethod
+    def identities_schema(self) -> Dict[str, Any]:
+        """
+        This is required for connectors that will support syncing
+        identities.
+        """
+        raise NotImplementedError(
+            f"{self.__class__.__name__} does not implement identities_schema, please return json schema for your identities stream."
+        )

airbyte_cdk/sources/file_based/schema_helpers.py

@@ -23,31 +23,6 @@ file_transfer_schema = {
     "properties": {"data": {"type": "object"}, "file": {"type": "object"}},
 }
 
-remote_file_permissions_schema = {
-    "type": "object",
-    "properties": {
-        "id": {"type": "string"},
-        "file_path": {"type": "string"},
-        "allowed_identity_remote_ids": {"type": "array", "items": {"type": "string"}},
-        "publicly_accessible": {"type": "boolean"},
-    },
-}
-
-remote_file_identity_schema = {
-    "type": "object",
-    "properties": {
-        "id": {"type": "string"},
-        "remote_id": {"type": "string"},
-        "parent_id": {"type": ["null", "string"]},
-        "name": {"type": ["null", "string"]},
-        "description": {"type": ["null", "string"]},
-        "email_address": {"type": ["null", "string"]},
-        "member_email_addresses": {"type": ["null", "array"]},
-        "type": {"type": "string"},
-        "modified_at": {"type": "string"},
-    },
-}
-
 
 @total_ordering
 class ComparableType(Enum):

airbyte_cdk/sources/file_based/stream/__init__.py

@@ -1,5 +1,5 @@
 from airbyte_cdk.sources.file_based.stream.abstract_file_based_stream import AbstractFileBasedStream
 from airbyte_cdk.sources.file_based.stream.default_file_based_stream import DefaultFileBasedStream
-from airbyte_cdk.sources.file_based.stream.identities_stream import IdentitiesStream
+from airbyte_cdk.sources.file_based.stream.identities_stream import FileIdentities
 
-__all__ = ["AbstractFileBasedStream", "DefaultFileBasedStream", "IdentitiesStream"]
+__all__ = ["AbstractFileBasedStream", "DefaultFileBasedStream", "FileIdentities"]

airbyte_cdk/sources/file_based/stream/default_file_based_stream.py

@@ -29,7 +29,6 @@ from airbyte_cdk.sources.file_based.schema_helpers import (
     SchemaType,
     file_transfer_schema,
     merge_schemas,
-    remote_file_permissions_schema,
     schemaless_schema,
 )
 from airbyte_cdk.sources.file_based.stream import AbstractFileBasedStream
@@ -111,7 +110,7 @@ class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin):
                 },
             }
         elif self.use_permissions_transfer:
-            return remote_file_permissions_schema
+            return self.stream_reader.file_permissions_schema
         else:
             return super()._filter_schema_invalid_properties(configured_catalog_json_schema)
 
@@ -315,7 +314,7 @@ class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin):
         if self.use_file_transfer:
             return file_transfer_schema
         elif self.use_permissions_transfer:
-            return remote_file_permissions_schema
+            return self.stream_reader.file_permissions_schema
         elif self.config.input_schema:
             return self.config.get_input_schema()  # type: ignore
         elif self.config.schemaless:

airbyte_cdk/sources/file_based/stream/identities_stream.py

@@ -2,30 +2,18 @@
 # Copyright (c) 2024 Airbyte, Inc., all rights reserved.
 #
 
-import traceback
 from functools import cache
-from typing import Any, Iterable, List, Mapping, MutableMapping, Optional
+from typing import Any, Dict, Iterable, Mapping, MutableMapping, Optional
 
-from airbyte_protocol_dataclasses.models import SyncMode
-
-from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, Level
-from airbyte_cdk.models import Type as MessageType
 from airbyte_cdk.sources.file_based.config.file_based_stream_config import PrimaryKeyType
 from airbyte_cdk.sources.file_based.discovery_policy import AbstractDiscoveryPolicy
-from airbyte_cdk.sources.file_based.exceptions import FileBasedErrorsCollector, FileBasedSourceError
+from airbyte_cdk.sources.file_based.exceptions import FileBasedErrorsCollector
 from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader
-from airbyte_cdk.sources.file_based.schema_helpers import remote_file_identity_schema
-from airbyte_cdk.sources.file_based.types import StreamSlice
-from airbyte_cdk.sources.streams import Stream
-from airbyte_cdk.sources.streams.checkpoint import Cursor
 from airbyte_cdk.sources.streams.core import JsonSchema
-from airbyte_cdk.sources.utils.record_helper import stream_data_to_airbyte_message
-from airbyte_cdk.utils.traced_exception import AirbyteTracedException
-
-IDENTITIES_STREAM_NAME = "identities"
+from airbyte_cdk.sources.streams.permissions.identities import Identities
 
 
-class IdentitiesStream(Stream):
+class FileIdentities(Identities):
     """
     The identities stream. A full refresh stream to sync identities from a certain domain.
     The stream reader manage the logic to get such data, which is implemented on connector side.
@@ -39,7 +27,7 @@ class IdentitiesStream(Stream):
         stream_reader: AbstractFileBasedStreamReader,
         discovery_policy: AbstractDiscoveryPolicy,
         errors_collector: FileBasedErrorsCollector,
-    ):
+    ) -> None:
         super().__init__()
         self.catalog_schema = catalog_schema
         self.stream_reader = stream_reader
@@ -47,50 +35,13 @@ class IdentitiesStream(Stream):
         self.errors_collector = errors_collector
         self._cursor: MutableMapping[str, Any] = {}
 
-    @property
-    def state(self) -> MutableMapping[str, Any]:
-        return self._cursor
-
-    @state.setter
-    def state(self, value: MutableMapping[str, Any]) -> None:
-        """State setter, accept state serialized by state getter."""
-        self._cursor = value
-
     @property
     def primary_key(self) -> PrimaryKeyType:
         return None
 
-    def read_records(
-        self,
-        sync_mode: SyncMode,
-        cursor_field: Optional[List[str]] = None,
-        stream_slice: Optional[StreamSlice] = None,
-        stream_state: Optional[Mapping[str, Any]] = None,
-    ) -> Iterable[Mapping[str, Any] | AirbyteMessage]:
-        try:
-            identity_groups = self.stream_reader.load_identity_groups(logger=self.logger)
-            for record in identity_groups:
-                yield stream_data_to_airbyte_message(self.name, record)
-        except AirbyteTracedException as exc:
-            # Re-raise the exception to stop the whole sync immediately as this is a fatal error
-            raise exc
-        except Exception:
-            yield AirbyteMessage(
-                type=MessageType.LOG,
-                log=AirbyteLogMessage(
-                    level=Level.ERROR,
-                    message=f"{FileBasedSourceError.ERROR_PARSING_RECORD.value} stream={self.name}",
-                    stack_trace=traceback.format_exc(),
-                ),
-            )
+    def load_identity_groups(self) -> Iterable[Dict[str, Any]]:
+        return self.stream_reader.load_identity_groups(logger=self.logger)
 
     @cache
     def get_json_schema(self) -> JsonSchema:
-        return remote_file_identity_schema
-
-    @property
-    def name(self) -> str:
-        return IDENTITIES_STREAM_NAME
-
-    def get_cursor(self) -> Optional[Cursor]:
-        return None
+        return self.stream_reader.identities_schema

airbyte_cdk/sources/specs/transfer_modes.py

@@ -0,0 +1,26 @@
+#
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+#
+
+from typing import Literal
+
+from pydantic.v1 import AnyUrl, BaseModel, Field
+
+from airbyte_cdk import OneOfOptionConfig
+
+
+class DeliverPermissions(BaseModel):
+    class Config(OneOfOptionConfig):
+        title = "Replicate Permissions ACL"
+        description = "Sends one identity stream and one for more permissions (ACL) streams to the destination. This data can be used in downstream systems to recreate permission restrictions mirroring the original source."
+        discriminator = "delivery_type"
+
+    delivery_type: Literal["use_permissions_transfer"] = Field(
+        "use_permissions_transfer", const=True
+    )
+
+    include_identities_stream: bool = Field(
+        title="Include Identity Stream",
+        description="This data can be used in downstream systems to recreate permission restrictions mirroring the original source",
+        default=True,
+    )

airbyte_cdk/sources/streams/concurrent/state_converters/datetime_stream_state_converter.py

@@ -6,9 +6,6 @@ from abc import abstractmethod
 from datetime import datetime, timedelta, timezone
 from typing import Any, Callable, List, MutableMapping, Optional, Tuple
 
-import pendulum
-from pendulum.datetime import DateTime
-
 # FIXME We would eventually like the Concurrent package do be agnostic of the declarative package. However, this is a breaking change and
 #  the goal in the short term is only to fix the issue we are seeing for source-declarative-manifest.
 from airbyte_cdk.sources.declarative.datetime.datetime_parser import DatetimeParser
@@ -17,6 +14,7 @@ from airbyte_cdk.sources.streams.concurrent.state_converters.abstract_stream_sta
     AbstractStreamStateConverter,
     ConcurrencyCompatibleStateType,
 )
+from airbyte_cdk.utils.datetime_helpers import AirbyteDateTime, ab_datetime_now, ab_datetime_parse
 
 
 class DateTimeStreamStateConverter(AbstractStreamStateConverter):
@@ -36,7 +34,7 @@ class DateTimeStreamStateConverter(AbstractStreamStateConverter):
 
     @classmethod
     def get_end_provider(cls) -> Callable[[], datetime]:
-        return lambda: datetime.now(timezone.utc)
+        return ab_datetime_now
 
     @abstractmethod
     def increment(self, timestamp: datetime) -> datetime: ...
@@ -136,10 +134,10 @@ class EpochValueConcurrentStreamStateConverter(DateTimeStreamStateConverter):
         return int(timestamp.timestamp())
 
     def parse_timestamp(self, timestamp: int) -> datetime:
-        dt_object = pendulum.from_timestamp(timestamp)
-        if not isinstance(dt_object, DateTime):
+        dt_object = AirbyteDateTime.fromtimestamp(timestamp, timezone.utc)
+        if not isinstance(dt_object, AirbyteDateTime):
             raise ValueError(
-                f"DateTime object was expected but got {type(dt_object)} from pendulum.parse({timestamp})"
+                f"AirbyteDateTime object was expected but got {type(dt_object)} from AirbyteDateTime.fromtimestamp({timestamp})"
             )
         return dt_object
 
@@ -169,14 +167,25 @@ class IsoMillisConcurrentStreamStateConverter(DateTimeStreamStateConverter):
     def increment(self, timestamp: datetime) -> datetime:
         return timestamp + self._cursor_granularity
 
-    def output_format(self, timestamp: datetime) -> Any:
-        return timestamp.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z"
+    def output_format(self, timestamp: datetime) -> str:
+        """Format datetime with milliseconds always included.
+
+        Args:
+            timestamp: The datetime to format.
+
+        Returns:
+            str: ISO8601/RFC3339 formatted string with milliseconds.
+        """
+        dt = AirbyteDateTime.from_datetime(timestamp)
+        # Always include milliseconds, even if zero
+        millis = dt.microsecond // 1000 if dt.microsecond else 0
+        return f"{dt.year:04d}-{dt.month:02d}-{dt.day:02d}T{dt.hour:02d}:{dt.minute:02d}:{dt.second:02d}.{millis:03d}Z"
 
     def parse_timestamp(self, timestamp: str) -> datetime:
-        dt_object = pendulum.parse(timestamp)
-        if not isinstance(dt_object, DateTime):
+        dt_object = ab_datetime_parse(timestamp)
+        if not isinstance(dt_object, AirbyteDateTime):
             raise ValueError(
-                f"DateTime object was expected but got {type(dt_object)} from pendulum.parse({timestamp})"
+                f"AirbyteDateTime object was expected but got {type(dt_object)} from parse({timestamp})"
             )
         return dt_object
 
@@ -184,7 +193,7 @@ class IsoMillisConcurrentStreamStateConverter(DateTimeStreamStateConverter):
 class CustomFormatConcurrentStreamStateConverter(IsoMillisConcurrentStreamStateConverter):
     """
     Datetime State converter that emits state according to the supplied datetime format. The converter supports reading
-    incoming state in any valid datetime format via Pendulum.
+    incoming state in any valid datetime format using AirbyteDateTime parsing utilities.
     """
 
     def __init__(

airbyte_cdk/sources/streams/core.py

@@ -223,17 +223,17 @@ class Stream(ABC):
                     record_counter += 1
 
                     checkpoint_interval = self.state_checkpoint_interval
-                    checkpoint = checkpoint_reader.get_checkpoint()
                     if (
                         should_checkpoint
                         and checkpoint_interval
                         and record_counter % checkpoint_interval == 0
-                        and checkpoint is not None
                     ):
-                        airbyte_state_message = self._checkpoint_state(
-                            checkpoint, state_manager=state_manager
-                        )
-                        yield airbyte_state_message
+                        checkpoint = checkpoint_reader.get_checkpoint()
+                        if checkpoint:
+                            airbyte_state_message = self._checkpoint_state(
+                                checkpoint, state_manager=state_manager
+                            )
+                            yield airbyte_state_message
 
                     if internal_config.is_limit_reached(record_counter):
                         break

airbyte_cdk/sources/streams/http/http.py

@@ -423,8 +423,6 @@ class HttpStream(Stream, CheckpointMixin, ABC):
         stream_slice: Optional[Mapping[str, Any]] = None,
         stream_state: Optional[Mapping[str, Any]] = None,
     ) -> Iterable[StreamData]:
-        partition, _, _ = self._extract_slice_fields(stream_slice=stream_slice)
-
         stream_state = stream_state or {}
         pagination_complete = False
         next_page_token = None
@@ -438,6 +436,7 @@ class HttpStream(Stream, CheckpointMixin, ABC):
 
         cursor = self.get_cursor()
         if cursor and isinstance(cursor, SubstreamResumableFullRefreshCursor):
+            partition, _, _ = self._extract_slice_fields(stream_slice=stream_slice)
             # Substreams checkpoint state by marking an entire parent partition as completed so that on the subsequent attempt
             # after a failure, completed parents are skipped and the sync can make progress
             cursor.close_slice(StreamSlice(cursor_slice={}, partition=partition))