airbyte-cdk 6.45.4.post72.dev14497997772__tar.gz → 6.45.6__tar.gz

{airbyte_cdk-6.45.4.post72.dev14497997772 → airbyte_cdk-6.45.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 6.45.4.post72.dev14497997772
+Version: 6.45.6
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://airbyte.com
 License: MIT

{airbyte_cdk-6.45.4.post72.dev14497997772 → airbyte_cdk-6.45.6}/airbyte_cdk/sources/declarative/auth/oauth.py

@@ -239,8 +239,8 @@ class DeclarativeOauth2Authenticator(AbstractOauth2Authenticator, DeclarativeAut
     def _has_access_token_been_initialized(self) -> bool:
         return self._access_token is not None
 
-    def set_token_expiry_date(self, value: Union[str, int]) -> None:
-        self._token_expiry_date = self._parse_token_expiration_date(value)
+    def set_token_expiry_date(self, value: AirbyteDateTime) -> None:
+        self._token_expiry_date = value
 
     def get_assertion_name(self) -> str:
         return self.assertion_name

{airbyte_cdk-6.45.4.post72.dev14497997772 → airbyte_cdk-6.45.6}/airbyte_cdk/sources/declarative/concurrent_declarative_source.py

@@ -19,7 +19,10 @@ from airbyte_cdk.sources.declarative.extractors import RecordSelector
 from airbyte_cdk.sources.declarative.extractors.record_filter import (
     ClientSideIncrementalRecordFilterDecorator,
 )
-from airbyte_cdk.sources.declarative.incremental import ConcurrentPerPartitionCursor
+from airbyte_cdk.sources.declarative.incremental import (
+    ConcurrentPerPartitionCursor,
+    GlobalSubstreamCursor,
+)
 from airbyte_cdk.sources.declarative.incremental.datetime_based_cursor import DatetimeBasedCursor
 from airbyte_cdk.sources.declarative.incremental.per_partition_with_global import (
     PerPartitionWithGlobalCursor,
@@ -361,7 +364,8 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
                     == DatetimeBasedCursorModel.__name__
                     and hasattr(declarative_stream.retriever, "stream_slicer")
                     and isinstance(
-                        declarative_stream.retriever.stream_slicer, PerPartitionWithGlobalCursor
+                        declarative_stream.retriever.stream_slicer,
+                        (GlobalSubstreamCursor, PerPartitionWithGlobalCursor),
                     )
                 ):
                     stream_state = self._connector_state_manager.get_stream_state(

{airbyte_cdk-6.45.4.post72.dev14497997772 → airbyte_cdk-6.45.6}/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py

@@ -1439,7 +1439,9 @@ class ModelToComponentFactory:
         stream_state = self.apply_stream_state_migrations(stream_state_migrations, stream_state)
 
         # Per-partition state doesn't make sense for GroupingPartitionRouter, so force the global state
-        use_global_cursor = isinstance(partition_router, GroupingPartitionRouter)
+        use_global_cursor = isinstance(
+            partition_router, GroupingPartitionRouter
+        ) or component_definition.get("global_substream_cursor", False)
 
         # Return the concurrent cursor and state converter
         return ConcurrentPerPartitionCursor(

{airbyte_cdk-6.45.4.post72.dev14497997772 → airbyte_cdk-6.45.6}/airbyte_cdk/sources/declarative/requesters/query_properties/property_chunking.py

@@ -52,8 +52,10 @@ class PropertyChunking:
         chunk_size = 0
         for property_field in property_fields:
             # If property_limit_type is not defined, we default to property_count which is just an incrementing count
+            # todo: Add ability to specify parameter delimiter representation and take into account in property_field_size
             property_field_size = (
                 len(property_field)
+                + 1  # The +1 represents the extra character for the delimiter in between properties
                 if self.property_limit_type == PropertyLimitType.characters
                 else 1
             )

{airbyte_cdk-6.45.4.post72.dev14497997772 → airbyte_cdk-6.45.6}/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py

@@ -10,6 +10,7 @@ from itertools import islice
 from typing import (
     Any,
     Callable,
+    Dict,
     Iterable,
     List,
     Mapping,
@@ -367,14 +368,65 @@ class SimpleRetriever(Retriever):
             {"next_page_token": initial_token} if initial_token is not None else None
         )
         while not pagination_complete:
-            response = self._fetch_next_page(stream_state, stream_slice, next_page_token)
+            property_chunks: List[List[str]] = (
+                list(
+                    self.additional_query_properties.get_request_property_chunks(
+                        stream_slice=stream_slice
+                    )
+                )
+                if self.additional_query_properties
+                else [
+                    []
+                ]  # A single empty property chunk represents the case where property chunking is not configured
+            )
 
+            merged_records: MutableMapping[str, Any] = defaultdict(dict)
             last_page_size = 0
             last_record: Optional[Record] = None
-            for record in records_generator_fn(response):
-                last_page_size += 1
-                last_record = record
-                yield record
+            response: Optional[requests.Response] = None
+            for properties in property_chunks:
+                if len(properties) > 0:
+                    stream_slice = StreamSlice(
+                        partition=stream_slice.partition or {},
+                        cursor_slice=stream_slice.cursor_slice or {},
+                        extra_fields={"query_properties": properties},
+                    )
+
+                response = self._fetch_next_page(stream_state, stream_slice, next_page_token)
+                for current_record in records_generator_fn(response):
+                    if (
+                        current_record
+                        and self.additional_query_properties
+                        and self.additional_query_properties.property_chunking
+                    ):
+                        merge_key = (
+                            self.additional_query_properties.property_chunking.get_merge_key(
+                                current_record
+                            )
+                        )
+                        if merge_key:
+                            _deep_merge(merged_records[merge_key], current_record)
+                        else:
+                            # We should still emit records even if the record did not have a merge key
+                            last_page_size += 1
+                            last_record = current_record
+                            yield current_record
+                    else:
+                        last_page_size += 1
+                        last_record = current_record
+                        yield current_record
+
+            if (
+                self.additional_query_properties
+                and self.additional_query_properties.property_chunking
+            ):
+                for merged_record in merged_records.values():
+                    record = Record(
+                        data=merged_record, stream_name=self.name, associated_slice=stream_slice
+                    )
+                    last_page_size += 1
+                    last_record = record
+                    yield record
 
             if not response:
                 pagination_complete = True
@@ -449,110 +501,43 @@ class SimpleRetriever(Retriever):
         :param stream_slice: The stream slice to read data for
         :return: The records read from the API source
         """
-
-        property_chunks = (
-            list(
-                self.additional_query_properties.get_request_property_chunks(
-                    stream_slice=stream_slice
-                )
-            )
-            if self.additional_query_properties
-            else []
-        )
-        records_without_merge_key = []
-        merged_records: MutableMapping[str, Any] = defaultdict(dict)
-
         _slice = stream_slice or StreamSlice(partition={}, cursor_slice={})  # None-check
+
         most_recent_record_from_slice = None
+        record_generator = partial(
+            self._parse_records,
+            stream_slice=stream_slice,
+            stream_state=self.state or {},
+            records_schema=records_schema,
+        )
 
-        if self.additional_query_properties:
-            for properties in property_chunks:
-                _slice = StreamSlice(
-                    partition=_slice.partition or {},
-                    cursor_slice=_slice.cursor_slice or {},
-                    extra_fields={"query_properties": properties},
-                )  # None-check
-
-                record_generator = partial(
-                    self._parse_records,
-                    stream_slice=_slice,
-                    stream_state=self.state or {},
-                    records_schema=records_schema,
-                )
+        if self.cursor and isinstance(self.cursor, ResumableFullRefreshCursor):
+            stream_state = self.state
 
-                for stream_data in self._read_pages(record_generator, self.state, _slice):
-                    current_record = self._extract_record(stream_data, _slice)
-                    if self.cursor and current_record:
-                        self.cursor.observe(_slice, current_record)
+            # Before syncing the RFR stream, we check if the job's prior attempt was successful and don't need to
+            # fetch more records. The platform deletes stream state for full refresh streams before starting a
+            # new job, so we don't need to worry about this value existing for the initial attempt
+            if stream_state.get(FULL_REFRESH_SYNC_COMPLETE_KEY):
+                return
 
-                    # Latest record read, not necessarily within slice boundaries.
-                    # TODO Remove once all custom components implement `observe` method.
-                    # https://github.com/airbytehq/airbyte-internal-issues/issues/6955
-                    most_recent_record_from_slice = self._get_most_recent_record(
-                        most_recent_record_from_slice, current_record, _slice
-                    )
+            yield from self._read_single_page(record_generator, stream_state, _slice)
+        else:
+            for stream_data in self._read_pages(record_generator, self.state, _slice):
+                current_record = self._extract_record(stream_data, _slice)
+                if self.cursor and current_record:
+                    self.cursor.observe(_slice, current_record)
+
+                # Latest record read, not necessarily within slice boundaries.
+                # TODO Remove once all custom components implement `observe` method.
+                # https://github.com/airbytehq/airbyte-internal-issues/issues/6955
+                most_recent_record_from_slice = self._get_most_recent_record(
+                    most_recent_record_from_slice, current_record, _slice
+                )
+                yield stream_data
 
-                    if current_record and self.additional_query_properties.property_chunking:
-                        merge_key = (
-                            self.additional_query_properties.property_chunking.get_merge_key(
-                                current_record
-                            )
-                        )
-                        if merge_key:
-                            merged_records[merge_key].update(current_record)
-                        else:
-                            # We should still emit records even if the record did not have a merge key
-                            records_without_merge_key.append(current_record)
-                    else:
-                        yield stream_data
             if self.cursor:
                 self.cursor.close_slice(_slice, most_recent_record_from_slice)
-
-            if len(merged_records) > 0:
-                yield from [
-                    Record(data=merged_record, stream_name=self.name, associated_slice=stream_slice)
-                    for merged_record in merged_records.values()
-                ]
-            if len(records_without_merge_key) > 0:
-                yield from records_without_merge_key
-        else:
-            _slice = stream_slice or StreamSlice(partition={}, cursor_slice={})  # None-check
-
-            most_recent_record_from_slice = None
-            record_generator = partial(
-                self._parse_records,
-                stream_slice=stream_slice,
-                stream_state=self.state or {},
-                records_schema=records_schema,
-            )
-
-            if self.cursor and isinstance(self.cursor, ResumableFullRefreshCursor):
-                stream_state = self.state
-
-                # Before syncing the RFR stream, we check if the job's prior attempt was successful and don't need to
-                # fetch more records. The platform deletes stream state for full refresh streams before starting a
-                # new job, so we don't need to worry about this value existing for the initial attempt
-                if stream_state.get(FULL_REFRESH_SYNC_COMPLETE_KEY):
-                    return
-
-                yield from self._read_single_page(record_generator, stream_state, _slice)
-            else:
-                for stream_data in self._read_pages(record_generator, self.state, _slice):
-                    current_record = self._extract_record(stream_data, _slice)
-                    if self.cursor and current_record:
-                        self.cursor.observe(_slice, current_record)
-
-                    # Latest record read, not necessarily within slice boundaries.
-                    # TODO Remove once all custom components implement `observe` method.
-                    # https://github.com/airbytehq/airbyte-internal-issues/issues/6955
-                    most_recent_record_from_slice = self._get_most_recent_record(
-                        most_recent_record_from_slice, current_record, _slice
-                    )
-                    yield stream_data
-
-                if self.cursor:
-                    self.cursor.close_slice(_slice, most_recent_record_from_slice)
-            return
+        return
 
     def _get_most_recent_record(
         self,
@@ -639,6 +624,26 @@ class SimpleRetriever(Retriever):
         return json.dumps(to_serialize, indent=None, separators=(",", ":"), sort_keys=True)
 
 
+def _deep_merge(
+    target: MutableMapping[str, Any], source: Union[Record, MutableMapping[str, Any]]
+) -> None:
+    """
+    Recursively merge two dictionaries, combining nested dictionaries instead of overwriting them.
+
+    :param target: The dictionary to merge into (modified in place)
+    :param source: The dictionary to merge from
+    """
+    for key, value in source.items():
+        if (
+            key in target
+            and isinstance(target[key], MutableMapping)
+            and isinstance(value, MutableMapping)
+        ):
+            _deep_merge(target[key], value)
+        else:
+            target[key] = value
+
+
 @dataclass
 class SimpleRetrieverTestReadDecorator(SimpleRetriever):
     """

{airbyte_cdk-6.45.4.post72.dev14497997772 → airbyte_cdk-6.45.6}/airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py

@@ -130,7 +130,7 @@ class AbstractOauth2Authenticator(AuthBase):
         headers = self.get_refresh_request_headers()
         return headers if headers else None
 
-    def refresh_access_token(self) -> Tuple[str, Union[str, int]]:
+    def refresh_access_token(self) -> Tuple[str, AirbyteDateTime]:
         """
         Returns the refresh token and its expiration datetime
 
@@ -148,6 +148,14 @@ class AbstractOauth2Authenticator(AuthBase):
     # PRIVATE METHODS
     # ----------------
 
+    def _default_token_expiry_date(self) -> AirbyteDateTime:
+        """
+        Returns the default token expiry date
+        """
+        # 1 hour was chosen as a middle ground to avoid unnecessary frequent refreshes and token expiration
+        default_token_expiry_duration_hours = 1  # 1 hour
+        return ab_datetime_now() + timedelta(hours=default_token_expiry_duration_hours)
+
     def _wrap_refresh_token_exception(
         self, exception: requests.exceptions.RequestException
     ) -> bool:
@@ -257,14 +265,10 @@ class AbstractOauth2Authenticator(AuthBase):
 
     def _parse_token_expiration_date(self, value: Union[str, int]) -> AirbyteDateTime:
         """
-        Return the expiration datetime of the refresh token
+        Parse a string or integer token expiration date into a datetime object
 
         :return: expiration datetime
         """
-        if not value and not self.token_has_expired():
-            # No expiry token was provided but the previous one is not expired so it's fine
-            return self.get_token_expiry_date()
-
         if self.token_expiry_is_time_of_expiration:
             if not self.token_expiry_date_format:
                 raise ValueError(
@@ -308,17 +312,30 @@ class AbstractOauth2Authenticator(AuthBase):
         """
         return self._find_and_get_value_from_response(response_data, self.get_refresh_token_name())
 
-    def _extract_token_expiry_date(self, response_data: Mapping[str, Any]) -> Any:
+    def _extract_token_expiry_date(self, response_data: Mapping[str, Any]) -> AirbyteDateTime:
         """
         Extracts the token_expiry_date, like `expires_in` or `expires_at`, etc from the given response data.
 
+        If the token_expiry_date is not found, it will return an existing token expiry date if set, or a default token expiry date.
+
         Args:
             response_data (Mapping[str, Any]): The response data from which to extract the token_expiry_date.
 
         Returns:
-            str: The extracted token_expiry_date.
+            The extracted token_expiry_date or None if not found.
         """
-        return self._find_and_get_value_from_response(response_data, self.get_expires_in_name())
+        expires_in = self._find_and_get_value_from_response(
+            response_data, self.get_expires_in_name()
+        )
+        if expires_in is not None:
+            return self._parse_token_expiration_date(expires_in)
+
+        # expires_in is None
+        existing_expiry_date = self.get_token_expiry_date()
+        if existing_expiry_date and not self.token_has_expired():
+            return existing_expiry_date
+
+        return self._default_token_expiry_date()
 
     def _find_and_get_value_from_response(
         self,
@@ -344,7 +361,7 @@ class AbstractOauth2Authenticator(AuthBase):
         """
         if current_depth > max_depth:
             # this is needed to avoid an inf loop, possible with a very deep nesting observed.
-            message = f"The maximum level of recursion is reached. Couldn't find the speficied `{key_name}` in the response."
+            message = f"The maximum level of recursion is reached. Couldn't find the specified `{key_name}` in the response."
             raise ResponseKeysMaxRecurtionReached(
                 internal_message=message, message=message, failure_type=FailureType.config_error
             )
@@ -441,7 +458,7 @@ class AbstractOauth2Authenticator(AuthBase):
         """Expiration date of the access token"""
 
     @abstractmethod
-    def set_token_expiry_date(self, value: Union[str, int]) -> None:
+    def set_token_expiry_date(self, value: AirbyteDateTime) -> None:
         """Setter for access token expiration date"""
 
     @abstractmethod

{airbyte_cdk-6.45.4.post72.dev14497997772 → airbyte_cdk-6.45.6}/airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py

@@ -120,8 +120,8 @@ class Oauth2Authenticator(AbstractOauth2Authenticator):
     def get_token_expiry_date(self) -> AirbyteDateTime:
         return self._token_expiry_date
 
-    def set_token_expiry_date(self, value: Union[str, int]) -> None:
-        self._token_expiry_date = self._parse_token_expiration_date(value)
+    def set_token_expiry_date(self, value: AirbyteDateTime) -> None:
+        self._token_expiry_date = value
 
     @property
     def token_expiry_is_time_of_expiration(self) -> bool:
@@ -316,26 +316,6 @@ class SingleUseRefreshTokenOauth2Authenticator(Oauth2Authenticator):
         """Returns True if the token is expired"""
         return ab_datetime_now() > self.get_token_expiry_date()
 
-    @staticmethod
-    def get_new_token_expiry_date(
-        access_token_expires_in: str,
-        token_expiry_date_format: str | None = None,
-    ) -> AirbyteDateTime:
-        """
-        Calculate the new token expiry date based on the provided expiration duration or format.
-
-        Args:
-            access_token_expires_in (str): The duration (in seconds) until the access token expires, or the expiry date in a specific format.
-            token_expiry_date_format (str | None, optional): The format of the expiry date if provided. Defaults to None.
-
-        Returns:
-            AirbyteDateTime: The calculated expiry date of the access token.
-        """
-        if token_expiry_date_format:
-            return ab_datetime_parse(access_token_expires_in)
-        else:
-            return ab_datetime_now() + timedelta(seconds=int(access_token_expires_in))
-
     def get_access_token(self) -> str:
         """Retrieve new access and refresh token if the access token has expired.
         The new refresh token is persisted with the set_refresh_token function
@@ -346,16 +326,13 @@ class SingleUseRefreshTokenOauth2Authenticator(Oauth2Authenticator):
             new_access_token, access_token_expires_in, new_refresh_token = (
                 self.refresh_access_token()
             )
-            new_token_expiry_date: AirbyteDateTime = self.get_new_token_expiry_date(
-                access_token_expires_in, self._token_expiry_date_format
-            )
             self.access_token = new_access_token
             self.set_refresh_token(new_refresh_token)
-            self.set_token_expiry_date(new_token_expiry_date)
+            self.set_token_expiry_date(access_token_expires_in)
             self._emit_control_message()
         return self.access_token
 
-    def refresh_access_token(self) -> Tuple[str, str, str]:  # type: ignore[override]
+    def refresh_access_token(self) -> Tuple[str, AirbyteDateTime, str]:  # type: ignore[override]
         """
         Refreshes the access token by making a handled request and extracting the necessary token information.
 

airbyte_cdk-6.45.4.post72.dev14497997772/airbyte_cdk/test/declarative/utils/job_runner.py → airbyte_cdk-6.45.6/airbyte_cdk/test/standard_tests/_job_runner.py

@@ -16,7 +16,7 @@ from airbyte_cdk.models import (
     Status,
 )
 from airbyte_cdk.test import entrypoint_wrapper
-from airbyte_cdk.test.declarative.models import (
+from airbyte_cdk.test.standard_tests.models import (
     ConnectorTestScenario,
 )
 

{airbyte_cdk-6.45.4.post72.dev14497997772 → airbyte_cdk-6.45.6}/airbyte_cdk/test/standard_tests/connector_base.py

@@ -18,10 +18,10 @@ from airbyte_cdk.models import (
     Type,
 )
 from airbyte_cdk.test import entrypoint_wrapper
-from airbyte_cdk.test.declarative.models import (
+from airbyte_cdk.test.standard_tests._job_runner import IConnector, run_test_job
+from airbyte_cdk.test.standard_tests.models import (
     ConnectorTestScenario,
 )
-from airbyte_cdk.test.declarative.utils.job_runner import IConnector, run_test_job
 
 ACCEPTANCE_TEST_CONFIG = "acceptance-test-config.yml"
 MANIFEST_YAML = "manifest.yaml"

{airbyte_cdk-6.45.4.post72.dev14497997772 → airbyte_cdk-6.45.6}/airbyte_cdk/test/standard_tests/declarative_sources.py

@@ -9,9 +9,9 @@ from boltons.typeutils import classproperty
 from airbyte_cdk.sources.declarative.concurrent_declarative_source import (
     ConcurrentDeclarativeSource,
 )
-from airbyte_cdk.test.declarative.models import ConnectorTestScenario
-from airbyte_cdk.test.declarative.utils.job_runner import IConnector
+from airbyte_cdk.test.standard_tests._job_runner import IConnector
 from airbyte_cdk.test.standard_tests.connector_base import MANIFEST_YAML
+from airbyte_cdk.test.standard_tests.models import ConnectorTestScenario
 from airbyte_cdk.test.standard_tests.source_base import SourceTestSuiteBase
 
 

{airbyte_cdk-6.45.4.post72.dev14497997772/airbyte_cdk/test/declarative → airbyte_cdk-6.45.6/airbyte_cdk/test/standard_tests}/models/__init__.py

@@ -1,4 +1,4 @@
-from airbyte_cdk.test.declarative.models.scenario import (
+from airbyte_cdk.test.standard_tests.models.scenario import (
     ConnectorTestScenario,
 )
 

{airbyte_cdk-6.45.4.post72.dev14497997772 → airbyte_cdk-6.45.6}/airbyte_cdk/test/standard_tests/source_base.py

@@ -13,13 +13,13 @@ from airbyte_cdk.models import (
     Type,
 )
 from airbyte_cdk.test import entrypoint_wrapper
-from airbyte_cdk.test.declarative.models import (
-    ConnectorTestScenario,
-)
-from airbyte_cdk.test.declarative.utils.job_runner import run_test_job
+from airbyte_cdk.test.standard_tests._job_runner import run_test_job
 from airbyte_cdk.test.standard_tests.connector_base import (
     ConnectorTestSuiteBase,
 )
+from airbyte_cdk.test.standard_tests.models import (
+    ConnectorTestScenario,
+)
 
 
 class SourceTestSuiteBase(ConnectorTestSuiteBase):

{airbyte_cdk-6.45.4.post72.dev14497997772 → airbyte_cdk-6.45.6}/pyproject.toml

@@ -23,7 +23,7 @@ classifiers = [
 keywords = ["airbyte", "connector-development-kit", "cdk"]
 
 # Python CDK uses dynamic versioning: https://github.com/mtkennerly/poetry-dynamic-versioning
-version = "6.45.4.post72.dev14497997772"  # Version will be calculated dynamically.
+version = "6.45.6"  # Version will be calculated dynamically.
 
 [tool.poetry-dynamic-versioning]
 enable = false

airbyte_cdk-6.45.4.post72.dev14497997772/airbyte_cdk/test/declarative/__init__.py

@@ -1,6 +0,0 @@
-# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
-"""Declarative tests framework.
-
-This module provides fixtures and utilities for testing Airbyte sources and destinations
-in a declarative way.
-"""