airbyte-cdk 7.0.5__tar.gz → 7.1.1__tar.gz

{airbyte_cdk-7.0.5 → airbyte_cdk-7.1.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 7.0.5
+Version: 7.1.1
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://airbyte.com
 License: MIT
@@ -84,7 +84,7 @@ Requires-Dist: unstructured.pytesseract (>=0.3.12) ; extra == "file-based"
 Requires-Dist: unstructured[docx,pptx] (==0.10.27) ; extra == "file-based"
 Requires-Dist: uvicorn (>=0.35.0) ; extra == "manifest-server"
 Requires-Dist: wcmatch (==10.0)
-Requires-Dist: whenever (>=0.6.16,<0.7.0)
+Requires-Dist: whenever (>=0.7.3,<0.9.0)
 Requires-Dist: xmltodict (>=0.13,<0.15)
 Project-URL: Documentation, https://docs.airbyte.io/
 Project-URL: Repository, https://github.com/airbytehq/airbyte-python-cdk

{airbyte_cdk-7.0.5 → airbyte_cdk-7.1.1}/airbyte_cdk/sources/declarative/checks/check_dynamic_stream.py

@@ -49,8 +49,9 @@ class CheckDynamicStream(ConnectionChecker):
             for stream in streams[: min(self.stream_count, len(streams))]:
                 stream_is_available, reason = evaluate_availability(stream, logger)
                 if not stream_is_available:
-                    logger.warning(f"Stream {stream.name} is not available: {reason}")
-                    return False, reason
+                    message = f"Stream {stream.name} is not available: {reason}"
+                    logger.warning(message)
+                    return False, message
         except Exception as error:
             error_message = (
                 f"Encountered an error trying to connect to stream {stream.name}. Error: {error}"

{airbyte_cdk-7.0.5 → airbyte_cdk-7.1.1}/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py

@@ -1264,22 +1264,12 @@ class ModelToComponentFactory:
         component_definition: ComponentDefinition,
         stream_name: str,
         stream_namespace: Optional[str],
+        stream_state: MutableMapping[str, Any],
         config: Config,
         message_repository: Optional[MessageRepository] = None,
         runtime_lookback_window: Optional[datetime.timedelta] = None,
-        stream_state_migrations: Optional[List[Any]] = None,
         **kwargs: Any,
     ) -> ConcurrentCursor:
-        # Per-partition incremental streams can dynamically create child cursors which will pass their current
-        # state via the stream_state keyword argument. Incremental syncs without parent streams use the
-        # incoming state and connector_state_manager that is initialized when the component factory is created
-        stream_state = (
-            self._connector_state_manager.get_stream_state(stream_name, stream_namespace)
-            if "stream_state" not in kwargs
-            else kwargs["stream_state"]
-        )
-        stream_state = self.apply_stream_state_migrations(stream_state_migrations, stream_state)
-
         component_type = component_definition.get("type")
         if component_definition.get("type") != model_type.__name__:
             raise ValueError(
@@ -1498,21 +1488,11 @@ class ModelToComponentFactory:
         component_definition: ComponentDefinition,
         stream_name: str,
         stream_namespace: Optional[str],
+        stream_state: MutableMapping[str, Any],
         config: Config,
         message_repository: Optional[MessageRepository] = None,
-        stream_state_migrations: Optional[List[Any]] = None,
         **kwargs: Any,
     ) -> ConcurrentCursor:
-        # Per-partition incremental streams can dynamically create child cursors which will pass their current
-        # state via the stream_state keyword argument. Incremental syncs without parent streams use the
-        # incoming state and connector_state_manager that is initialized when the component factory is created
-        stream_state = (
-            self._connector_state_manager.get_stream_state(stream_name, stream_namespace)
-            if "stream_state" not in kwargs
-            else kwargs["stream_state"]
-        )
-        stream_state = self.apply_stream_state_migrations(stream_state_migrations, stream_state)
-
         component_type = component_definition.get("type")
         if component_definition.get("type") != model_type.__name__:
             raise ValueError(
@@ -1587,7 +1567,6 @@ class ModelToComponentFactory:
         config: Config,
         stream_state: MutableMapping[str, Any],
         partition_router: PartitionRouter,
-        stream_state_migrations: Optional[List[Any]] = None,
         attempt_to_create_cursor_if_not_provided: bool = False,
         **kwargs: Any,
     ) -> ConcurrentPerPartitionCursor:
@@ -1647,11 +1626,9 @@ class ModelToComponentFactory:
                 stream_namespace=stream_namespace,
                 config=config,
                 message_repository=NoopMessageRepository(),
-                # stream_state_migrations=stream_state_migrations,  # FIXME is it expected to run migration on per partition state too?
             )
         )
 
-        stream_state = self.apply_stream_state_migrations(stream_state_migrations, stream_state)
         # Per-partition state doesn't make sense for GroupingPartitionRouter, so force the global state
         use_global_cursor = isinstance(
             partition_router, GroupingPartitionRouter
@@ -1974,6 +1951,7 @@ class ModelToComponentFactory:
         self, model: DeclarativeStreamModel, config: Config, is_parent: bool = False, **kwargs: Any
     ) -> AbstractStream:
         primary_key = model.primary_key.__root__ if model.primary_key else None
+        self._migrate_state(model, config)
 
         partition_router = self._build_stream_slicer_from_partition_router(
             model.retriever,
@@ -2135,6 +2113,23 @@ class ModelToComponentFactory:
             supports_file_transfer=hasattr(model, "file_uploader") and bool(model.file_uploader),
         )
 
+    def _migrate_state(self, model: DeclarativeStreamModel, config: Config) -> None:
+        stream_name = model.name or ""
+        stream_state = self._connector_state_manager.get_stream_state(
+            stream_name=stream_name, namespace=None
+        )
+        if model.state_migrations:
+            state_transformations = [
+                self._create_component_from_model(state_migration, config, declarative_stream=model)
+                for state_migration in model.state_migrations
+            ]
+        else:
+            state_transformations = []
+        stream_state = self.apply_stream_state_migrations(state_transformations, stream_state)
+        self._connector_state_manager.update_state_for_stream(
+            stream_name=stream_name, namespace=None, value=stream_state
+        )
+
     def _is_stop_condition_on_cursor(self, model: DeclarativeStreamModel) -> bool:
         return bool(
             model.incremental_sync
@@ -2206,17 +2201,7 @@ class ModelToComponentFactory:
         config: Config,
     ) -> Cursor:
         stream_name = model.name or ""
-        stream_state = self._connector_state_manager.get_stream_state(
-            stream_name=stream_name, namespace=None
-        )
-
-        if model.state_migrations:
-            state_transformations = [
-                self._create_component_from_model(state_migration, config, declarative_stream=model)
-                for state_migration in model.state_migrations
-            ]
-        else:
-            state_transformations = []
+        stream_state = self._connector_state_manager.get_stream_state(stream_name, None)
 
         if (
             model.incremental_sync
@@ -2228,10 +2213,9 @@ class ModelToComponentFactory:
                 model_type=DatetimeBasedCursorModel,
                 component_definition=model.incremental_sync.__dict__,
                 stream_name=stream_name,
+                stream_state=stream_state,
                 stream_namespace=None,
                 config=config or {},
-                stream_state=stream_state,
-                stream_state_migrations=state_transformations,
                 partition_router=stream_slicer,
                 attempt_to_create_cursor_if_not_provided=True,  # FIXME can we remove that now?
             )
@@ -2242,8 +2226,8 @@ class ModelToComponentFactory:
                     component_definition=model.incremental_sync.__dict__,
                     stream_name=stream_name,
                     stream_namespace=None,
+                    stream_state=stream_state,
                     config=config or {},
-                    stream_state_migrations=state_transformations,
                 )
             elif type(model.incremental_sync) == DatetimeBasedCursorModel:
                 return self.create_concurrent_cursor_from_datetime_based_cursor(  # type: ignore # This is a known issue that we are creating and returning a ConcurrentCursor which does not technically implement the (low-code) StreamSlicer. However, (low-code) StreamSlicer and ConcurrentCursor both implement StreamSlicer.stream_slices() which is the primary method needed for checkpointing
@@ -2251,8 +2235,8 @@ class ModelToComponentFactory:
                     component_definition=model.incremental_sync.__dict__,
                     stream_name=stream_name,
                     stream_namespace=None,
+                    stream_state=stream_state,
                     config=config or {},
-                    stream_state_migrations=state_transformations,
                     attempt_to_create_cursor_if_not_provided=True,
                 )
             else:

{airbyte_cdk-7.0.5 → airbyte_cdk-7.1.1}/airbyte_cdk/sources/streams/http/error_handlers/default_error_mapping.py

@@ -31,56 +31,56 @@ DEFAULT_ERROR_MAPPING: Mapping[Union[int, str, Type[Exception]], ErrorResolution
     400: ErrorResolution(
         response_action=ResponseAction.FAIL,
         failure_type=FailureType.system_error,
-        error_message="Bad request. Please check your request parameters.",
+        error_message="HTTP Status Code: 400. Error: Bad request. Please check your request parameters.",
     ),
     401: ErrorResolution(
         response_action=ResponseAction.FAIL,
         failure_type=FailureType.config_error,
-        error_message="Unauthorized. Please ensure you are authenticated correctly.",
+        error_message="HTTP Status Code: 401. Error: Unauthorized. Please ensure you are authenticated correctly.",
     ),
     403: ErrorResolution(
         response_action=ResponseAction.FAIL,
         failure_type=FailureType.config_error,
-        error_message="Forbidden. You don't have permission to access this resource.",
+        error_message="HTTP Status Code: 403. Error: Forbidden. You don't have permission to access this resource.",
     ),
     404: ErrorResolution(
         response_action=ResponseAction.FAIL,
         failure_type=FailureType.system_error,
-        error_message="Not found. The requested resource was not found on the server.",
+        error_message="HTTP Status Code: 404. Error: Not found. The requested resource was not found on the server.",
     ),
     405: ErrorResolution(
         response_action=ResponseAction.FAIL,
         failure_type=FailureType.system_error,
-        error_message="Method not allowed. Please check your request method.",
+        error_message="HTTP Status Code: 405. Error: Method not allowed. Please check your request method.",
     ),
     408: ErrorResolution(
         response_action=ResponseAction.RETRY,
         failure_type=FailureType.transient_error,
-        error_message="Request timeout.",
+        error_message="HTTP Status Code: 408. Error: Request timeout.",
     ),
     429: ErrorResolution(
         response_action=ResponseAction.RATE_LIMITED,
         failure_type=FailureType.transient_error,
-        error_message="Too many requests.",
+        error_message="HTTP Status Code: 429. Error: Too many requests.",
     ),
     500: ErrorResolution(
         response_action=ResponseAction.RETRY,
         failure_type=FailureType.transient_error,
-        error_message="Internal server error.",
+        error_message="HTTP Status Code: 500. Error: Internal server error.",
     ),
     502: ErrorResolution(
         response_action=ResponseAction.RETRY,
         failure_type=FailureType.transient_error,
-        error_message="Bad gateway.",
+        error_message="HTTP Status Code: 502. Error: Bad gateway.",
     ),
     503: ErrorResolution(
         response_action=ResponseAction.RETRY,
         failure_type=FailureType.transient_error,
-        error_message="Service unavailable.",
+        error_message="HTTP Status Code: 503. Error: Service unavailable.",
     ),
     504: ErrorResolution(
         response_action=ResponseAction.RETRY,
         failure_type=FailureType.transient_error,
-        error_message="Gateway timeout.",
+        error_message="HTTP Status Code: 504. Error: Gateway timeout.",
     ),
 }

{airbyte_cdk-7.0.5 → airbyte_cdk-7.1.1}/airbyte_cdk/sources/streams/http/exceptions.py

@@ -7,6 +7,8 @@ from typing import Optional, Union
 
 import requests
 
+from airbyte_cdk.models import FailureType
+
 
 class BaseBackoffException(requests.exceptions.HTTPError):
     def __init__(
@@ -14,7 +16,9 @@ class BaseBackoffException(requests.exceptions.HTTPError):
         request: requests.PreparedRequest,
         response: Optional[Union[requests.Response, Exception]],
         error_message: str = "",
+        failure_type: Optional[FailureType] = None,
     ):
+        self.failure_type = failure_type
         if isinstance(response, requests.Response):
             error_message = (
                 error_message
@@ -43,6 +47,7 @@ class UserDefinedBackoffException(BaseBackoffException):
         request: requests.PreparedRequest,
         response: Optional[Union[requests.Response, Exception]],
         error_message: str = "",
+        failure_type: Optional[FailureType] = None,
     ):
         """
         :param backoff: how long to backoff in seconds
@@ -50,7 +55,12 @@ class UserDefinedBackoffException(BaseBackoffException):
         :param response: the response that triggered the backoff exception
         """
         self.backoff = backoff
-        super().__init__(request=request, response=response, error_message=error_message)
+        super().__init__(
+            request=request,
+            response=response,
+            error_message=error_message,
+            failure_type=failure_type,
+        )
 
 
 class DefaultBackoffException(BaseBackoffException):

{airbyte_cdk-7.0.5 → airbyte_cdk-7.1.1}/airbyte_cdk/sources/streams/http/http_client.py

@@ -18,6 +18,7 @@ from airbyte_cdk.models import (
     AirbyteStreamStatus,
     AirbyteStreamStatusReason,
     AirbyteStreamStatusReasonType,
+    FailureType,
     Level,
     StreamDescriptor,
 )
@@ -35,6 +36,7 @@ from airbyte_cdk.sources.streams.http.error_handlers import (
     ResponseAction,
 )
 from airbyte_cdk.sources.streams.http.exceptions import (
+    BaseBackoffException,
     DefaultBackoffException,
     RateLimitBackoffException,
     RequestBodyException,
@@ -290,15 +292,25 @@ class HttpClient:
         backoff_handler = http_client_default_backoff_handler(
             max_tries=max_tries, max_time=max_time
         )
-        # backoff handlers wrap _send, so it will always return a response
-        response = backoff_handler(rate_limit_backoff_handler(user_backoff_handler))(
-            request,
-            request_kwargs,
-            log_formatter=log_formatter,
-            exit_on_rate_limit=exit_on_rate_limit,
-        )  # type: ignore # mypy can't infer that backoff_handler wraps _send
-
-        return response
+        # backoff handlers wrap _send, so it will always return a response -- except when all retries are exhausted
+        try:
+            response = backoff_handler(rate_limit_backoff_handler(user_backoff_handler))(
+                request,
+                request_kwargs,
+                log_formatter=log_formatter,
+                exit_on_rate_limit=exit_on_rate_limit,
+            )  # type: ignore # mypy can't infer that backoff_handler wraps _send
+
+            return response
+        except BaseBackoffException as e:
+            self._logger.error(f"Retries exhausted with backoff exception.", exc_info=True)
+            raise MessageRepresentationAirbyteTracedErrors(
+                internal_message=f"Exhausted available request attempts. Exception: {e}",
+                message=f"Exhausted available request attempts. Please see logs for more details. Exception: {e}",
+                failure_type=e.failure_type or FailureType.system_error,
+                exception=e,
+                stream_descriptor=StreamDescriptor(name=self._name),
+            )
 
     def _send(
         self,
@@ -492,6 +504,7 @@ class HttpClient:
                     request=request,
                     response=(response if response is not None else exc),
                     error_message=error_message,
+                    failure_type=error_resolution.failure_type,
                 )
 
             elif retry_endlessly:
@@ -499,12 +512,14 @@ class HttpClient:
                     request=request,
                     response=(response if response is not None else exc),
                     error_message=error_message,
+                    failure_type=error_resolution.failure_type,
                 )
 
             raise DefaultBackoffException(
                 request=request,
                 response=(response if response is not None else exc),
                 error_message=error_message,
+                failure_type=error_resolution.failure_type,
             )
 
         elif response:
@@ -545,6 +560,15 @@ class HttpClient:
             data=data,
         )
 
+        env_settings = self._session.merge_environment_settings(
+            url=request.url,
+            proxies=request_kwargs.get("proxies"),
+            stream=request_kwargs.get("stream"),
+            verify=request_kwargs.get("verify"),
+            cert=request_kwargs.get("cert"),
+        )
+        request_kwargs = {**request_kwargs, **env_settings}
+
         response: requests.Response = self._send_with_retry(
             request=request,
             request_kwargs=request_kwargs,

{airbyte_cdk-7.0.5 → airbyte_cdk-7.1.1}/airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py

@@ -240,7 +240,11 @@ class AbstractOauth2Authenticator(AuthBase):
         except requests.exceptions.RequestException as e:
             if e.response is not None:
                 if e.response.status_code == 429 or e.response.status_code >= 500:
-                    raise DefaultBackoffException(request=e.response.request, response=e.response)
+                    raise DefaultBackoffException(
+                        request=e.response.request,
+                        response=e.response,
+                        failure_type=FailureType.transient_error,
+                    )
             if self._wrap_refresh_token_exception(e):
                 message = "Refresh token is invalid or expired. Please re-authenticate from Sources/<your source>/Settings."
                 raise AirbyteTracedException(

{airbyte_cdk-7.0.5 → airbyte_cdk-7.1.1}/airbyte_cdk/sources/utils/transform.py

@@ -8,6 +8,7 @@ from enum import Flag, auto
 from typing import TYPE_CHECKING, Any, Callable, Dict, Generator, Mapping, Optional, cast
 
 from jsonschema import Draft7Validator, ValidationError, validators
+from jsonschema.protocols import Validator
 from referencing import Registry, Resource
 from referencing._core import Resolver
 from referencing.exceptions import Unresolvable
@@ -17,12 +18,6 @@ from airbyte_cdk.sources.utils.schema_helpers import expand_refs
 
 from .schema_helpers import get_ref_resolver_registry
 
-try:
-    from jsonschema.validators import Validator
-except:
-    from jsonschema import Validator
-
-
 MAX_NESTING_DEPTH = 3
 json_to_python_simple = {
     "string": str,

{airbyte_cdk-7.0.5 → airbyte_cdk-7.1.1}/airbyte_cdk/utils/datetime_helpers.py

@@ -82,11 +82,10 @@ assert not ab_datetime_try_parse("foo")                    # Invalid: not parsab
 """
 
 from datetime import datetime, timedelta, timezone
-from typing import Any, Optional, Union, overload
+from typing import Any, Union, overload
 
 from dateutil import parser
-from typing_extensions import Never
-from whenever import Instant, LocalDateTime, ZonedDateTime
+from whenever import Instant
 
 
 class AirbyteDateTime(datetime):

{airbyte_cdk-7.0.5 → airbyte_cdk-7.1.1}/pyproject.toml

@@ -23,7 +23,7 @@ classifiers = [
 keywords = ["airbyte", "connector-development-kit", "cdk"]
 
 # Python CDK uses dynamic versioning: https://github.com/mtkennerly/poetry-dynamic-versioning
-version = "7.0.5"  # Version will be calculated dynamically.
+version = "7.1.1"  # Version will be calculated dynamically.
 
 [tool.poetry-dynamic-versioning]
 enable = false
@@ -93,7 +93,7 @@ uvicorn = { version = ">=0.35.0", optional = true}
 ddtrace = { version = "^3.12.3", optional = true }
 xmltodict = ">=0.13,<0.15"
 anyascii = "^0.3.2"
-whenever = "^0.6.16"
+whenever = ">=0.7.3,<0.9.0"
 setuptools = "^80.9.0"  # Some connectors depend on this as an undeclared transitive dependency (e.g. source-shopify)
 unidecode = "^1.3.8"  # IMPORTANT: DO NOT USE. Added to support source-google-sheets migration. This dep is deprecated in favour of `anyascii` and is slated for removal.
 dateparser = "^1.2.2"