airbyte-cdk 7.0.4__tar.gz → 7.1.0__tar.gz

{airbyte_cdk-7.0.4 → airbyte_cdk-7.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 7.0.4
+Version: 7.1.0
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://airbyte.com
 License: MIT
@@ -84,7 +84,7 @@ Requires-Dist: unstructured.pytesseract (>=0.3.12) ; extra == "file-based"
 Requires-Dist: unstructured[docx,pptx] (==0.10.27) ; extra == "file-based"
 Requires-Dist: uvicorn (>=0.35.0) ; extra == "manifest-server"
 Requires-Dist: wcmatch (==10.0)
-Requires-Dist: whenever (>=0.6.16,<0.7.0)
+Requires-Dist: whenever (>=0.7.3,<0.9.0)
 Requires-Dist: xmltodict (>=0.13,<0.15)
 Project-URL: Documentation, https://docs.airbyte.io/
 Project-URL: Repository, https://github.com/airbytehq/airbyte-python-cdk

{airbyte_cdk-7.0.4 → airbyte_cdk-7.1.0}/airbyte_cdk/sources/declarative/checks/check_dynamic_stream.py

@@ -49,8 +49,9 @@ class CheckDynamicStream(ConnectionChecker):
             for stream in streams[: min(self.stream_count, len(streams))]:
                 stream_is_available, reason = evaluate_availability(stream, logger)
                 if not stream_is_available:
-                    logger.warning(f"Stream {stream.name} is not available: {reason}")
-                    return False, reason
+                    message = f"Stream {stream.name} is not available: {reason}"
+                    logger.warning(message)
+                    return False, message
         except Exception as error:
             error_message = (
                 f"Encountered an error trying to connect to stream {stream.name}. Error: {error}"

{airbyte_cdk-7.0.4 → airbyte_cdk-7.1.0}/airbyte_cdk/sources/declarative/concurrent_declarative_source.py

@@ -20,7 +20,7 @@ from typing import (
 
 import orjson
 import yaml
-from airbyte_protocol_dataclasses.models import Level
+from airbyte_protocol_dataclasses.models import AirbyteStreamStatus, Level, StreamDescriptor
 from jsonschema.exceptions import ValidationError
 from jsonschema.validators import validate
 
@@ -88,6 +88,7 @@ from airbyte_cdk.sources.utils.slice_logger import (
     DebugSliceLogger,
     SliceLogger,
 )
+from airbyte_cdk.utils.stream_status_utils import as_airbyte_message
 from airbyte_cdk.utils.traced_exception import AirbyteTracedException
 
 
@@ -630,9 +631,8 @@ class ConcurrentDeclarativeSource(Source):
 
         return dynamic_stream_configs
 
-    @staticmethod
     def _select_streams(
-        streams: List[AbstractStream], configured_catalog: ConfiguredAirbyteCatalog
+        self, streams: List[AbstractStream], configured_catalog: ConfiguredAirbyteCatalog
     ) -> List[AbstractStream]:
         stream_name_to_instance: Mapping[str, AbstractStream] = {s.name: s for s in streams}
         abstract_streams: List[AbstractStream] = []
@@ -640,5 +640,13 @@ class ConcurrentDeclarativeSource(Source):
             stream_instance = stream_name_to_instance.get(configured_stream.stream.name)
             if stream_instance:
                 abstract_streams.append(stream_instance)
-
+            else:
+                # Previous behavior in the legacy synchronous CDK was to also raise an error TRACE message if
+                # the source was configured with raise_exception_on_missing_stream=True. This was used on very
+                # few sources like facebook-marketing and google-ads. We decided not to port this feature over,
+                # but we can do so if we feel it necessary. With the current behavior,we should still result
+                # in a partial failure since missing streams will be marked as INCOMPLETE.
+                self._message_repository.emit_message(
+                    as_airbyte_message(configured_stream.stream, AirbyteStreamStatus.INCOMPLETE)
+                )
         return abstract_streams

{airbyte_cdk-7.0.4 → airbyte_cdk-7.1.0}/airbyte_cdk/sources/streams/http/error_handlers/default_error_mapping.py

@@ -31,56 +31,56 @@ DEFAULT_ERROR_MAPPING: Mapping[Union[int, str, Type[Exception]], ErrorResolution
     400: ErrorResolution(
         response_action=ResponseAction.FAIL,
         failure_type=FailureType.system_error,
-        error_message="Bad request. Please check your request parameters.",
+        error_message="HTTP Status Code: 400. Error: Bad request. Please check your request parameters.",
     ),
     401: ErrorResolution(
         response_action=ResponseAction.FAIL,
         failure_type=FailureType.config_error,
-        error_message="Unauthorized. Please ensure you are authenticated correctly.",
+        error_message="HTTP Status Code: 401. Error: Unauthorized. Please ensure you are authenticated correctly.",
     ),
     403: ErrorResolution(
         response_action=ResponseAction.FAIL,
         failure_type=FailureType.config_error,
-        error_message="Forbidden. You don't have permission to access this resource.",
+        error_message="HTTP Status Code: 403. Error: Forbidden. You don't have permission to access this resource.",
     ),
     404: ErrorResolution(
         response_action=ResponseAction.FAIL,
         failure_type=FailureType.system_error,
-        error_message="Not found. The requested resource was not found on the server.",
+        error_message="HTTP Status Code: 404. Error: Not found. The requested resource was not found on the server.",
     ),
     405: ErrorResolution(
         response_action=ResponseAction.FAIL,
         failure_type=FailureType.system_error,
-        error_message="Method not allowed. Please check your request method.",
+        error_message="HTTP Status Code: 405. Error: Method not allowed. Please check your request method.",
     ),
     408: ErrorResolution(
         response_action=ResponseAction.RETRY,
         failure_type=FailureType.transient_error,
-        error_message="Request timeout.",
+        error_message="HTTP Status Code: 408. Error: Request timeout.",
     ),
     429: ErrorResolution(
         response_action=ResponseAction.RATE_LIMITED,
         failure_type=FailureType.transient_error,
-        error_message="Too many requests.",
+        error_message="HTTP Status Code: 429. Error: Too many requests.",
     ),
     500: ErrorResolution(
         response_action=ResponseAction.RETRY,
         failure_type=FailureType.transient_error,
-        error_message="Internal server error.",
+        error_message="HTTP Status Code: 500. Error: Internal server error.",
     ),
     502: ErrorResolution(
         response_action=ResponseAction.RETRY,
         failure_type=FailureType.transient_error,
-        error_message="Bad gateway.",
+        error_message="HTTP Status Code: 502. Error: Bad gateway.",
     ),
     503: ErrorResolution(
         response_action=ResponseAction.RETRY,
         failure_type=FailureType.transient_error,
-        error_message="Service unavailable.",
+        error_message="HTTP Status Code: 503. Error: Service unavailable.",
     ),
     504: ErrorResolution(
         response_action=ResponseAction.RETRY,
         failure_type=FailureType.transient_error,
-        error_message="Gateway timeout.",
+        error_message="HTTP Status Code: 504. Error: Gateway timeout.",
     ),
 }

{airbyte_cdk-7.0.4 → airbyte_cdk-7.1.0}/airbyte_cdk/sources/streams/http/exceptions.py

@@ -7,6 +7,8 @@ from typing import Optional, Union
 
 import requests
 
+from airbyte_cdk.models import FailureType
+
 
 class BaseBackoffException(requests.exceptions.HTTPError):
     def __init__(
@@ -14,7 +16,9 @@ class BaseBackoffException(requests.exceptions.HTTPError):
         request: requests.PreparedRequest,
         response: Optional[Union[requests.Response, Exception]],
         error_message: str = "",
+        failure_type: Optional[FailureType] = None,
     ):
+        self.failure_type = failure_type
         if isinstance(response, requests.Response):
             error_message = (
                 error_message
@@ -43,6 +47,7 @@ class UserDefinedBackoffException(BaseBackoffException):
         request: requests.PreparedRequest,
         response: Optional[Union[requests.Response, Exception]],
         error_message: str = "",
+        failure_type: Optional[FailureType] = None,
     ):
         """
         :param backoff: how long to backoff in seconds
@@ -50,7 +55,12 @@ class UserDefinedBackoffException(BaseBackoffException):
         :param response: the response that triggered the backoff exception
         """
         self.backoff = backoff
-        super().__init__(request=request, response=response, error_message=error_message)
+        super().__init__(
+            request=request,
+            response=response,
+            error_message=error_message,
+            failure_type=failure_type,
+        )
 
 
 class DefaultBackoffException(BaseBackoffException):

{airbyte_cdk-7.0.4 → airbyte_cdk-7.1.0}/airbyte_cdk/sources/streams/http/http_client.py

@@ -18,6 +18,7 @@ from airbyte_cdk.models import (
     AirbyteStreamStatus,
     AirbyteStreamStatusReason,
     AirbyteStreamStatusReasonType,
+    FailureType,
     Level,
     StreamDescriptor,
 )
@@ -35,6 +36,7 @@ from airbyte_cdk.sources.streams.http.error_handlers import (
     ResponseAction,
 )
 from airbyte_cdk.sources.streams.http.exceptions import (
+    BaseBackoffException,
     DefaultBackoffException,
     RateLimitBackoffException,
     RequestBodyException,
@@ -290,15 +292,25 @@ class HttpClient:
         backoff_handler = http_client_default_backoff_handler(
             max_tries=max_tries, max_time=max_time
         )
-        # backoff handlers wrap _send, so it will always return a response
-        response = backoff_handler(rate_limit_backoff_handler(user_backoff_handler))(
-            request,
-            request_kwargs,
-            log_formatter=log_formatter,
-            exit_on_rate_limit=exit_on_rate_limit,
-        )  # type: ignore # mypy can't infer that backoff_handler wraps _send
-
-        return response
+        # backoff handlers wrap _send, so it will always return a response -- except when all retries are exhausted
+        try:
+            response = backoff_handler(rate_limit_backoff_handler(user_backoff_handler))(
+                request,
+                request_kwargs,
+                log_formatter=log_formatter,
+                exit_on_rate_limit=exit_on_rate_limit,
+            )  # type: ignore # mypy can't infer that backoff_handler wraps _send
+
+            return response
+        except BaseBackoffException as e:
+            self._logger.error(f"Retries exhausted with backoff exception.", exc_info=True)
+            raise MessageRepresentationAirbyteTracedErrors(
+                internal_message=f"Exhausted available request attempts. Exception: {e}",
+                message=f"Exhausted available request attempts. Please see logs for more details. Exception: {e}",
+                failure_type=e.failure_type or FailureType.system_error,
+                exception=e,
+                stream_descriptor=StreamDescriptor(name=self._name),
+            )
 
     def _send(
         self,
@@ -492,6 +504,7 @@ class HttpClient:
                     request=request,
                     response=(response if response is not None else exc),
                     error_message=error_message,
+                    failure_type=error_resolution.failure_type,
                 )
 
             elif retry_endlessly:
@@ -499,12 +512,14 @@ class HttpClient:
                     request=request,
                     response=(response if response is not None else exc),
                     error_message=error_message,
+                    failure_type=error_resolution.failure_type,
                 )
 
             raise DefaultBackoffException(
                 request=request,
                 response=(response if response is not None else exc),
                 error_message=error_message,
+                failure_type=error_resolution.failure_type,
             )
 
         elif response:

{airbyte_cdk-7.0.4 → airbyte_cdk-7.1.0}/airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py

@@ -240,7 +240,11 @@ class AbstractOauth2Authenticator(AuthBase):
         except requests.exceptions.RequestException as e:
             if e.response is not None:
                 if e.response.status_code == 429 or e.response.status_code >= 500:
-                    raise DefaultBackoffException(request=e.response.request, response=e.response)
+                    raise DefaultBackoffException(
+                        request=e.response.request,
+                        response=e.response,
+                        failure_type=FailureType.transient_error,
+                    )
             if self._wrap_refresh_token_exception(e):
                 message = "Refresh token is invalid or expired. Please re-authenticate from Sources/<your source>/Settings."
                 raise AirbyteTracedException(

{airbyte_cdk-7.0.4 → airbyte_cdk-7.1.0}/airbyte_cdk/sources/utils/transform.py

@@ -8,6 +8,7 @@ from enum import Flag, auto
 from typing import TYPE_CHECKING, Any, Callable, Dict, Generator, Mapping, Optional, cast
 
 from jsonschema import Draft7Validator, ValidationError, validators
+from jsonschema.protocols import Validator
 from referencing import Registry, Resource
 from referencing._core import Resolver
 from referencing.exceptions import Unresolvable
@@ -17,12 +18,6 @@ from airbyte_cdk.sources.utils.schema_helpers import expand_refs
 
 from .schema_helpers import get_ref_resolver_registry
 
-try:
-    from jsonschema.validators import Validator
-except:
-    from jsonschema import Validator
-
-
 MAX_NESTING_DEPTH = 3
 json_to_python_simple = {
     "string": str,

{airbyte_cdk-7.0.4 → airbyte_cdk-7.1.0}/airbyte_cdk/utils/datetime_helpers.py

@@ -82,11 +82,10 @@ assert not ab_datetime_try_parse("foo")                    # Invalid: not parsab
 """
 
 from datetime import datetime, timedelta, timezone
-from typing import Any, Optional, Union, overload
+from typing import Any, Union, overload
 
 from dateutil import parser
-from typing_extensions import Never
-from whenever import Instant, LocalDateTime, ZonedDateTime
+from whenever import Instant
 
 
 class AirbyteDateTime(datetime):

{airbyte_cdk-7.0.4 → airbyte_cdk-7.1.0}/pyproject.toml

@@ -23,7 +23,7 @@ classifiers = [
 keywords = ["airbyte", "connector-development-kit", "cdk"]
 
 # Python CDK uses dynamic versioning: https://github.com/mtkennerly/poetry-dynamic-versioning
-version = "7.0.4"  # Version will be calculated dynamically.
+version = "7.1.0"  # Version will be calculated dynamically.
 
 [tool.poetry-dynamic-versioning]
 enable = false
@@ -93,7 +93,7 @@ uvicorn = { version = ">=0.35.0", optional = true}
 ddtrace = { version = "^3.12.3", optional = true }
 xmltodict = ">=0.13,<0.15"
 anyascii = "^0.3.2"
-whenever = "^0.6.16"
+whenever = ">=0.7.3,<0.9.0"
 setuptools = "^80.9.0"  # Some connectors depend on this as an undeclared transitive dependency (e.g. source-shopify)
 unidecode = "^1.3.8"  # IMPORTANT: DO NOT USE. Added to support source-google-sheets migration. This dep is deprecated in favour of `anyascii` and is slated for removal.
 dateparser = "^1.2.2"