airbyte-cdk 6.26.0.dev4106__py3-none-any.whl → 6.26.0.dev4108__py3-none-any.whl

airbyte_cdk/sources/file_based/file_based_source.py

@@ -33,6 +33,12 @@ from airbyte_cdk.sources.file_based.config.file_based_stream_config import (
     FileBasedStreamConfig,
     ValidationPolicy,
 )
+from airbyte_cdk.sources.file_based.config.validate_config_transfer_modes import (
+    include_identities_stream,
+    preserve_directory_structure,
+    use_file_transfer,
+    use_permissions_transfer,
+)
 from airbyte_cdk.sources.file_based.discovery_policy import (
     AbstractDiscoveryPolicy,
     DefaultDiscoveryPolicy,
@@ -164,7 +170,11 @@ class FileBasedSource(ConcurrentSourceAdapter, ABC):
         tracebacks = []
         for stream in streams:
             if isinstance(stream, IdentitiesStream):
-                # Probably need to check identities endpoint/api access but will skip for now.
+                identity = next(iter(stream.load_identity_groups()))
+                if not identity:
+                    errors.append(
+                        "Unable to get identities for current configuration, please check your credentials"
+                    )
                 continue
             if not isinstance(stream, AbstractFileBasedStream):
                 raise ValueError(f"Stream {stream} is not a file-based stream.")
@@ -172,8 +182,7 @@ class FileBasedSource(ConcurrentSourceAdapter, ABC):
                 parsed_config = self._get_parsed_config(config)
                 availability_method = (
                     stream.availability_strategy.check_availability
-                    if self._use_file_transfer(parsed_config)
-                    or self._sync_acl_permissions(parsed_config)
+                    if use_file_transfer(parsed_config) or use_permissions_transfer(parsed_config)
                     else stream.availability_strategy.check_availability_and_parsability
                 )
                 (
@@ -300,7 +309,7 @@ class FileBasedSource(ConcurrentSourceAdapter, ABC):
 
                 streams.append(stream)
 
-            if self._sync_acl_permissions(parsed_config):
+            if include_identities_stream(parsed_config):
                 identities_stream = self._make_identities_stream()
                 streams.append(identities_stream)
             return streams
@@ -324,9 +333,9 @@ class FileBasedSource(ConcurrentSourceAdapter, ABC):
             validation_policy=self._validate_and_get_validation_policy(stream_config),
             errors_collector=self.errors_collector,
             cursor=cursor,
-            use_file_transfer=self._use_file_transfer(parsed_config),
-            preserve_directory_structure=self._preserve_directory_structure(parsed_config),
-            sync_acl_permissions=self._sync_acl_permissions(parsed_config),
+            use_file_transfer=use_file_transfer(parsed_config),
+            preserve_directory_structure=preserve_directory_structure(parsed_config),
+            use_permissions_transfer=use_permissions_transfer(parsed_config),
         )
 
     def _make_identities_stream(
@@ -403,51 +412,3 @@ class FileBasedSource(ConcurrentSourceAdapter, ABC):
                 "`input_schema` and `schemaless` options cannot both be set",
                 model=FileBasedStreamConfig,
             )
-
-    @staticmethod
-    def _use_file_transfer(parsed_config: AbstractFileBasedSpec) -> bool:
-        use_file_transfer = (
-            hasattr(parsed_config.delivery_method, "delivery_type")
-            and parsed_config.delivery_method.delivery_type == "use_file_transfer"
-        )
-        return use_file_transfer
-
-    @staticmethod
-    def _use_records_transfer(parsed_config: AbstractFileBasedSpec) -> bool:
-        use_records_transfer = (
-            hasattr(parsed_config.delivery_method, "delivery_type")
-            and parsed_config.delivery_method.delivery_type == "use_records_transfer"
-        )
-        return use_records_transfer
-
-    @staticmethod
-    def _preserve_directory_structure(parsed_config: AbstractFileBasedSpec) -> bool:
-        """
-        Determines whether to preserve directory structure during file transfer.
-
-        When enabled, files maintain their subdirectory paths in the destination.
-        When disabled, files are flattened to the root of the destination.
-
-        Args:
-            parsed_config: The parsed configuration containing delivery method settings
-
-        Returns:
-            True if directory structure should be preserved (default), False otherwise
-        """
-        if (
-            FileBasedSource._use_file_transfer(parsed_config)
-            and hasattr(parsed_config.delivery_method, "preserve_directory_structure")
-            and parsed_config.delivery_method.preserve_directory_structure is not None
-        ):
-            return parsed_config.delivery_method.preserve_directory_structure
-        return True
-
-    @staticmethod
-    def _sync_acl_permissions(parsed_config: AbstractFileBasedSpec) -> bool:
-        if (
-            FileBasedSource._use_records_transfer(parsed_config)
-            and hasattr(parsed_config.delivery_method, "sync_acl_permissions")
-            and parsed_config.delivery_method.sync_acl_permissions is not None
-        ):
-            return parsed_config.delivery_method.sync_acl_permissions
-        return False

airbyte_cdk/sources/file_based/file_based_stream_reader.py

@@ -13,6 +13,11 @@ from typing import Any, Dict, Iterable, List, Optional, Set
 from wcmatch.glob import GLOBSTAR, globmatch
 
 from airbyte_cdk.sources.file_based.config.abstract_file_based_spec import AbstractFileBasedSpec
+from airbyte_cdk.sources.file_based.config.validate_config_transfer_modes import (
+    include_identities_stream,
+    preserve_directory_structure,
+    use_file_transfer,
+)
 from airbyte_cdk.sources.file_based.remote_file import RemoteFile
 
 
@@ -128,41 +133,18 @@ class AbstractFileBasedStreamReader(ABC):
 
     def use_file_transfer(self) -> bool:
         if self.config:
-            use_file_transfer = (
-                hasattr(self.config.delivery_method, "delivery_type")
-                and self.config.delivery_method.delivery_type == "use_file_transfer"
-            )
-            return use_file_transfer
-        return False
-
-    def use_records_transfer(self) -> bool:
-        if self.config:
-            use_records_transfer = (
-                hasattr(self.config.delivery_method, "delivery_type")
-                and self.config.delivery_method.delivery_type == "use_records_transfer"
-            )
-            return use_records_transfer
+            return use_file_transfer(self.config)
         return False
 
     def preserve_directory_structure(self) -> bool:
         # fall back to preserve subdirectories if config is not present or incomplete
-        if (
-            self.use_file_transfer()
-            and self.config
-            and hasattr(self.config.delivery_method, "preserve_directory_structure")
-            and self.config.delivery_method.preserve_directory_structure is not None
-        ):
-            return self.config.delivery_method.preserve_directory_structure
+        if self.config:
+            return preserve_directory_structure(self.config)
         return True
 
-    def sync_acl_permissions(self) -> bool:
-        if (
-            self.config
-            and self.use_records_transfer()
-            and hasattr(self.config.delivery_method, "sync_acl_permissions")
-            and self.config.delivery_method.sync_acl_permissions is not None
-        ):
-            return self.config.delivery_method.sync_acl_permissions
+    def include_identities_stream(self) -> bool:
+        if self.config:
+            return include_identities_stream(self.config)
         return False
 
     @abstractmethod
@@ -203,16 +185,22 @@ class AbstractFileBasedStreamReader(ABC):
         absolute_file_path = path.abspath(local_file_path)
         return [file_relative_path, local_file_path, absolute_file_path]
 
+    @abstractmethod
     def get_file_acl_permissions(self, file: RemoteFile, logger: logging.Logger) -> Dict[str, Any]:
         """
         This is required for connectors that will support syncing
         ACL Permissions from files.
         """
-        return {}
+        raise NotImplementedError(
+            f"{self.__class__.__name__} must implement get_file_acl_permissions()"
+        )
 
+    @abstractmethod
     def load_identity_groups(self, logger: logging.Logger) -> Iterable[Dict[str, Any]]:
         """
         This is required for connectors that will support syncing
         identities.
         """
-        yield {}
+        raise NotImplementedError(
+            f"{self.__class__.__name__} must implement load_identity_groups()"
+        )

airbyte_cdk/sources/file_based/stream/default_file_based_stream.py

@@ -48,7 +48,7 @@ class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin):
 
     FILE_TRANSFER_KW = "use_file_transfer"
     PRESERVE_DIRECTORY_STRUCTURE_KW = "preserve_directory_structure"
-    SYNC_ACL_PERMISSIONS_KW = "sync_acl_permissions"
+    PERMISSIONS_TRANSFER_KW = "use_permissions_transfer"
     FILES_KEY = "files"
     DATE_TIME_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ"
     ab_last_mod_col = "_ab_source_file_last_modified"
@@ -58,7 +58,7 @@ class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin):
     airbyte_columns = [ab_last_mod_col, ab_file_name_col]
     use_file_transfer = False
     preserve_directory_structure = True
-    sync_acl_permissions = False
+    use_permissions_transfer = False
 
     def __init__(self, **kwargs: Any):
         if self.FILE_TRANSFER_KW in kwargs:
@@ -67,8 +67,8 @@ class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin):
             self.preserve_directory_structure = kwargs.pop(
                 self.PRESERVE_DIRECTORY_STRUCTURE_KW, True
             )
-        if self.SYNC_ACL_PERMISSIONS_KW in kwargs:
-            self.sync_acl_permissions = kwargs.pop(self.SYNC_ACL_PERMISSIONS_KW, False)
+        if self.PERMISSIONS_TRANSFER_KW in kwargs:
+            self.use_permissions_transfer = kwargs.pop(self.PERMISSIONS_TRANSFER_KW, False)
         super().__init__(**kwargs)
 
     @property
@@ -110,7 +110,7 @@ class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin):
                     self.ab_file_name_col: {"type": "string"},
                 },
             }
-        elif self.sync_acl_permissions:
+        elif self.use_permissions_transfer:
             return remote_file_permissions_schema
         else:
             return super()._filter_schema_invalid_properties(configured_catalog_json_schema)
@@ -194,7 +194,7 @@ class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin):
                         yield stream_data_to_airbyte_message(
                             self.name, record, is_file_transfer_message=True
                         )
-                elif self.sync_acl_permissions:
+                elif self.use_permissions_transfer:
                     try:
                         permissions_record = self.stream_reader.get_file_acl_permissions(
                             file, logger=self.logger
@@ -314,7 +314,7 @@ class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin):
     def _get_raw_json_schema(self) -> JsonSchema:
         if self.use_file_transfer:
             return file_transfer_schema
-        elif self.sync_acl_permissions:
+        elif self.use_permissions_transfer:
             return remote_file_permissions_schema
         elif self.config.input_schema:
             return self.config.get_input_schema()  # type: ignore

airbyte_cdk/sources/file_based/stream/identities_stream.py

@@ -4,7 +4,7 @@
 
 import traceback
 from functools import cache
-from typing import Any, Iterable, List, Mapping, MutableMapping, Optional
+from typing import Any, Dict, Iterable, List, Mapping, MutableMapping, Optional
 
 from airbyte_protocol_dataclasses.models import SyncMode
 
@@ -68,7 +68,7 @@ class IdentitiesStream(Stream):
         stream_state: Optional[Mapping[str, Any]] = None,
     ) -> Iterable[Mapping[str, Any] | AirbyteMessage]:
         try:
-            identity_groups = self.stream_reader.load_identity_groups(logger=self.logger)
+            identity_groups = self.load_identity_groups()
             for record in identity_groups:
                 yield stream_data_to_airbyte_message(self.name, record)
         except AirbyteTracedException as exc:
@@ -84,6 +84,9 @@ class IdentitiesStream(Stream):
                 ),
             )
 
+    def load_identity_groups(self) -> Iterable[Dict[str, Any]]:
+        return self.stream_reader.load_identity_groups(logger=self.logger)
+
     @cache
     def get_json_schema(self) -> JsonSchema:
         return remote_file_identity_schema

airbyte_cdk/sources/streams/concurrent/state_converters/datetime_stream_state_converter.py

@@ -6,9 +6,6 @@ from abc import abstractmethod
 from datetime import datetime, timedelta, timezone
 from typing import Any, Callable, List, MutableMapping, Optional, Tuple
 
-import pendulum
-from pendulum.datetime import DateTime
-
 # FIXME We would eventually like the Concurrent package do be agnostic of the declarative package. However, this is a breaking change and
 #  the goal in the short term is only to fix the issue we are seeing for source-declarative-manifest.
 from airbyte_cdk.sources.declarative.datetime.datetime_parser import DatetimeParser
@@ -17,6 +14,7 @@ from airbyte_cdk.sources.streams.concurrent.state_converters.abstract_stream_sta
     AbstractStreamStateConverter,
     ConcurrencyCompatibleStateType,
 )
+from airbyte_cdk.utils.datetime_helpers import AirbyteDateTime, ab_datetime_now, ab_datetime_parse
 
 
 class DateTimeStreamStateConverter(AbstractStreamStateConverter):
@@ -36,7 +34,7 @@ class DateTimeStreamStateConverter(AbstractStreamStateConverter):
 
     @classmethod
     def get_end_provider(cls) -> Callable[[], datetime]:
-        return lambda: datetime.now(timezone.utc)
+        return ab_datetime_now
 
     @abstractmethod
     def increment(self, timestamp: datetime) -> datetime: ...
@@ -136,10 +134,10 @@ class EpochValueConcurrentStreamStateConverter(DateTimeStreamStateConverter):
         return int(timestamp.timestamp())
 
     def parse_timestamp(self, timestamp: int) -> datetime:
-        dt_object = pendulum.from_timestamp(timestamp)
-        if not isinstance(dt_object, DateTime):
+        dt_object = AirbyteDateTime.fromtimestamp(timestamp, timezone.utc)
+        if not isinstance(dt_object, AirbyteDateTime):
             raise ValueError(
-                f"DateTime object was expected but got {type(dt_object)} from pendulum.parse({timestamp})"
+                f"AirbyteDateTime object was expected but got {type(dt_object)} from AirbyteDateTime.fromtimestamp({timestamp})"
             )
         return dt_object
 
@@ -169,14 +167,25 @@ class IsoMillisConcurrentStreamStateConverter(DateTimeStreamStateConverter):
     def increment(self, timestamp: datetime) -> datetime:
         return timestamp + self._cursor_granularity
 
-    def output_format(self, timestamp: datetime) -> Any:
-        return timestamp.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z"
+    def output_format(self, timestamp: datetime) -> str:
+        """Format datetime with milliseconds always included.
+
+        Args:
+            timestamp: The datetime to format.
+
+        Returns:
+            str: ISO8601/RFC3339 formatted string with milliseconds.
+        """
+        dt = AirbyteDateTime.from_datetime(timestamp)
+        # Always include milliseconds, even if zero
+        millis = dt.microsecond // 1000 if dt.microsecond else 0
+        return f"{dt.year:04d}-{dt.month:02d}-{dt.day:02d}T{dt.hour:02d}:{dt.minute:02d}:{dt.second:02d}.{millis:03d}Z"
 
     def parse_timestamp(self, timestamp: str) -> datetime:
-        dt_object = pendulum.parse(timestamp)
-        if not isinstance(dt_object, DateTime):
+        dt_object = ab_datetime_parse(timestamp)
+        if not isinstance(dt_object, AirbyteDateTime):
             raise ValueError(
-                f"DateTime object was expected but got {type(dt_object)} from pendulum.parse({timestamp})"
+                f"AirbyteDateTime object was expected but got {type(dt_object)} from parse({timestamp})"
             )
         return dt_object
 
@@ -184,7 +193,7 @@ class IsoMillisConcurrentStreamStateConverter(DateTimeStreamStateConverter):
 class CustomFormatConcurrentStreamStateConverter(IsoMillisConcurrentStreamStateConverter):
     """
     Datetime State converter that emits state according to the supplied datetime format. The converter supports reading
-    incoming state in any valid datetime format via Pendulum.
+    incoming state in any valid datetime format using AirbyteDateTime parsing utilities.
     """
 
     def __init__(

airbyte_cdk/sources/streams/core.py

@@ -223,17 +223,17 @@ class Stream(ABC):
                     record_counter += 1
 
                     checkpoint_interval = self.state_checkpoint_interval
-                    checkpoint = checkpoint_reader.get_checkpoint()
                     if (
                         should_checkpoint
                         and checkpoint_interval
                         and record_counter % checkpoint_interval == 0
-                        and checkpoint is not None
                     ):
-                        airbyte_state_message = self._checkpoint_state(
-                            checkpoint, state_manager=state_manager
-                        )
-                        yield airbyte_state_message
+                        checkpoint = checkpoint_reader.get_checkpoint()
+                        if checkpoint:
+                            airbyte_state_message = self._checkpoint_state(
+                                checkpoint, state_manager=state_manager
+                            )
+                            yield airbyte_state_message
 
                     if internal_config.is_limit_reached(record_counter):
                         break

airbyte_cdk/sources/streams/http/http.py

@@ -423,8 +423,6 @@ class HttpStream(Stream, CheckpointMixin, ABC):
         stream_slice: Optional[Mapping[str, Any]] = None,
         stream_state: Optional[Mapping[str, Any]] = None,
     ) -> Iterable[StreamData]:
-        partition, _, _ = self._extract_slice_fields(stream_slice=stream_slice)
-
         stream_state = stream_state or {}
         pagination_complete = False
         next_page_token = None
@@ -438,6 +436,7 @@ class HttpStream(Stream, CheckpointMixin, ABC):
 
         cursor = self.get_cursor()
         if cursor and isinstance(cursor, SubstreamResumableFullRefreshCursor):
+            partition, _, _ = self._extract_slice_fields(stream_slice=stream_slice)
             # Substreams checkpoint state by marking an entire parent partition as completed so that on the subsequent attempt
             # after a failure, completed parents are skipped and the sync can make progress
             cursor.close_slice(StreamSlice(cursor_slice={}, partition=partition))