airbyte-cdk 6.7.0rc3__py3-none-any.whl → 6.7.1.dev0__py3-none-any.whl

airbyte_cdk/sources/streams/http/http_client.py

@@ -11,6 +11,8 @@ from typing import Any, Callable, Dict, List, Mapping, Optional, Tuple, Union
 import orjson
 import requests
 import requests_cache
+from requests.auth import AuthBase
+
 from airbyte_cdk.models import (
     AirbyteMessageSerializer,
     AirbyteStreamStatus,
@@ -43,12 +45,13 @@ from airbyte_cdk.sources.streams.http.rate_limiting import (
     rate_limit_default_backoff_handler,
     user_defined_backoff_handler,
 )
+from airbyte_cdk.sources.utils.types import JsonType
+from airbyte_cdk.utils.airbyte_secrets_utils import filter_secrets
 from airbyte_cdk.utils.constants import ENV_REQUEST_CACHE_PATH
 from airbyte_cdk.utils.stream_status_utils import (
     as_airbyte_message as stream_status_as_airbyte_message,
 )
 from airbyte_cdk.utils.traced_exception import AirbyteTracedException
-from requests.auth import AuthBase
 
 BODY_REQUEST_METHODS = ("GET", "POST", "PUT", "PATCH")
 
@@ -73,6 +76,7 @@ class MessageRepresentationAirbyteTracedErrors(AirbyteTracedException):
 class HttpClient:
     _DEFAULT_MAX_RETRY: int = 5
     _DEFAULT_MAX_TIME: int = 60 * 10
+    _ACTIONS_TO_RETRY_ON = {ResponseAction.RETRY, ResponseAction.RATE_LIMITED}
 
     def __init__(
         self,
@@ -333,6 +337,40 @@ class HttpClient:
 
         return response  # type: ignore # will either return a valid response of type requests.Response or raise an exception
 
+    def _get_response_body(self, response: requests.Response) -> Optional[JsonType]:
+        """
+        Extracts and returns the body of an HTTP response.
+
+        This method attempts to parse the response body as JSON. If the response
+        body is not valid JSON, it falls back to decoding the response content
+        as a UTF-8 string. If both attempts fail, it returns None.
+
+        Args:
+            response (requests.Response): The HTTP response object.
+
+        Returns:
+            Optional[JsonType]: The parsed JSON object as a string, the decoded
+            response content as a string, or None if both parsing attempts fail.
+        """
+        try:
+            return str(response.json())
+        except requests.exceptions.JSONDecodeError:
+            try:
+                return response.content.decode("utf-8")
+            except Exception:
+                return "The Content of the Response couldn't be decoded."
+
+    def _evict_key(self, prepared_request: requests.PreparedRequest) -> None:
+        """
+        Addresses high memory consumption when enabling concurrency in https://github.com/airbytehq/oncall/issues/6821.
+
+        The `_request_attempt_count` attribute keeps growing as multiple requests are made using the same `http_client`.
+        To mitigate this issue, we evict keys for completed requests once we confirm that no further retries are needed.
+        This helps manage memory usage more efficiently while maintaining the necessary logic for retry attempts.
+        """
+        if prepared_request in self._request_attempt_count:
+            del self._request_attempt_count[prepared_request]
+
     def _handle_error_resolution(
         self,
         response: Optional[requests.Response],
@@ -341,6 +379,9 @@ class HttpClient:
         error_resolution: ErrorResolution,
         exit_on_rate_limit: Optional[bool] = False,
     ) -> None:
+        if error_resolution.response_action not in self._ACTIONS_TO_RETRY_ON:
+            self._evict_key(request)
+
         # Emit stream status RUNNING with the reason RATE_LIMITED to log that the rate limit has been reached
         if error_resolution.response_action == ResponseAction.RATE_LIMITED:
             # TODO: Update to handle with message repository when concurrent message repository is ready
@@ -361,12 +402,18 @@ class HttpClient:
 
         if error_resolution.response_action == ResponseAction.FAIL:
             if response is not None:
-                error_message = f"'{request.method}' request to '{request.url}' failed with status code '{response.status_code}' and error message '{self._error_message_parser.parse_response_error_message(response)}'"
+                filtered_response_message = filter_secrets(
+                    f"Request (body): '{str(request.body)}'. Response (body): '{self._get_response_body(response)}'. Response (headers): '{response.headers}'."
+                )
+                error_message = f"'{request.method}' request to '{request.url}' failed with status code '{response.status_code}' and error message: '{self._error_message_parser.parse_response_error_message(response)}'. {filtered_response_message}"
             else:
                 error_message = (
                     f"'{request.method}' request to '{request.url}' failed with exception: '{exc}'"
                 )
 
+            # ensure the exception message is emitted before raised
+            self._logger.error(error_message)
+
             raise MessageRepresentationAirbyteTracedErrors(
                 internal_message=error_message,
                 message=error_resolution.error_message or error_message,

airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py

@@ -10,12 +10,13 @@ from typing import Any, List, Mapping, MutableMapping, Optional, Tuple, Union
 import backoff
 import pendulum
 import requests
+from requests.auth import AuthBase
+
 from airbyte_cdk.models import FailureType, Level
 from airbyte_cdk.sources.http_logger import format_http_message
 from airbyte_cdk.sources.message import MessageRepository, NoopMessageRepository
 from airbyte_cdk.utils import AirbyteTracedException
 from airbyte_cdk.utils.airbyte_secrets_utils import add_to_secrets
-from requests.auth import AuthBase
 
 from ..exceptions import DefaultBackoffException
 

airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py

@@ -6,6 +6,7 @@ from typing import Any, List, Mapping, Optional, Sequence, Tuple, Union
 
 import dpath
 import pendulum
+
 from airbyte_cdk.config_observation import (
     create_connector_config_control_message,
     emit_configuration_as_airbyte_control_message,

airbyte_cdk/sources/types.py

@@ -6,6 +6,8 @@ from __future__ import annotations
 
 from typing import Any, ItemsView, Iterator, KeysView, List, Mapping, Optional, ValuesView
 
+import orjson
+
 # A FieldPointer designates a path to a field inside a mapping. For example, retrieving ["k1", "k1.2"] in the object {"k1" :{"k1.2":
 # "hello"}] returns "hello"
 FieldPointer = List[str]
@@ -15,9 +17,17 @@ StreamState = Mapping[str, Any]
 
 
 class Record(Mapping[str, Any]):
-    def __init__(self, data: Mapping[str, Any], associated_slice: Optional[StreamSlice]):
+    def __init__(
+        self,
+        data: Mapping[str, Any],
+        stream_name: str,
+        associated_slice: Optional[StreamSlice] = None,
+        is_file_transfer_message: bool = False,
+    ):
         self._data = data
         self._associated_slice = associated_slice
+        self.stream_name = stream_name
+        self.is_file_transfer_message = is_file_transfer_message
 
     @property
     def data(self) -> Mapping[str, Any]:
@@ -139,3 +149,6 @@ class StreamSlice(Mapping[str, Any]):
 
     def __json_serializable__(self) -> Any:
         return self._stream_slice
+
+    def __hash__(self) -> int:
+        return hash(orjson.dumps(self._stream_slice, option=orjson.OPT_SORT_KEYS))

airbyte_cdk/sources/utils/schema_helpers.py

@@ -10,12 +10,13 @@ import pkgutil
 from typing import Any, ClassVar, Dict, List, Mapping, MutableMapping, Optional, Tuple
 
 import jsonref
-from airbyte_cdk.models import ConnectorSpecification, FailureType
-from airbyte_cdk.utils.traced_exception import AirbyteTracedException
 from jsonschema import RefResolver, validate
 from jsonschema.exceptions import ValidationError
 from pydantic.v1 import BaseModel, Field
 
+from airbyte_cdk.models import ConnectorSpecification, FailureType
+from airbyte_cdk.utils.traced_exception import AirbyteTracedException
+
 
 class JsonFileLoader:
     """

airbyte_cdk/sql/secrets.py

@@ -6,9 +6,10 @@ from __future__ import annotations
 import json
 from typing import TYPE_CHECKING, Any
 
-from airbyte_cdk.sql import exceptions as exc
 from pydantic_core import CoreSchema, core_schema
 
+from airbyte_cdk.sql import exceptions as exc
+
 if TYPE_CHECKING:
     from pydantic import GetCoreSchemaHandler, GetJsonSchemaHandler, ValidationInfo
     from pydantic.json_schema import JsonSchemaValue

airbyte_cdk/sql/shared/sql_processor.py

@@ -13,6 +13,12 @@ from typing import TYPE_CHECKING, Any, final
 import pandas as pd
 import sqlalchemy
 import ulid
+from airbyte_protocol_dataclasses.models import AirbyteStateMessage
+from pandas import Index
+from pydantic import BaseModel, Field
+from sqlalchemy import Column, Table, and_, create_engine, insert, null, select, text, update
+from sqlalchemy.exc import ProgrammingError, SQLAlchemyError
+
 from airbyte_cdk.sql import exceptions as exc
 from airbyte_cdk.sql._util.hashing import one_way_hash
 from airbyte_cdk.sql._util.name_normalizers import LowerCaseNormalizer
@@ -24,16 +30,10 @@ from airbyte_cdk.sql.constants import (
 )
 from airbyte_cdk.sql.secrets import SecretString
 from airbyte_cdk.sql.types import SQLTypeConverter
-from airbyte_protocol_dataclasses.models import AirbyteStateMessage
-from pandas import Index
-from pydantic import BaseModel, Field
-from sqlalchemy import Column, Table, and_, create_engine, insert, null, select, text, update
-from sqlalchemy.exc import ProgrammingError, SQLAlchemyError
 
 if TYPE_CHECKING:
     from collections.abc import Generator
 
-    from airbyte_cdk.sql.shared.catalog_providers import CatalogProvider
     from sqlalchemy.engine import Connection, Engine
     from sqlalchemy.engine.cursor import CursorResult
     from sqlalchemy.engine.reflection import Inspector
@@ -41,6 +41,8 @@ if TYPE_CHECKING:
     from sqlalchemy.sql.elements import TextClause
     from sqlalchemy.sql.type_api import TypeEngine
 
+    from airbyte_cdk.sql.shared.catalog_providers import CatalogProvider
+
 
 class SQLRuntimeError(Exception):
     """Raised when an SQL operation fails."""

airbyte_cdk/test/entrypoint_wrapper.py

@@ -23,6 +23,10 @@ from io import StringIO
 from pathlib import Path
 from typing import Any, List, Mapping, Optional, Union
 
+from orjson import orjson
+from pydantic import ValidationError as V2ValidationError
+from serpyco_rs import SchemaValidationError
+
 from airbyte_cdk.entrypoint import AirbyteEntrypoint
 from airbyte_cdk.exception_handler import assemble_uncaught_exception
 from airbyte_cdk.logger import AirbyteLogFormatter
@@ -40,9 +44,6 @@ from airbyte_cdk.models import (
     Type,
 )
 from airbyte_cdk.sources import Source
-from orjson import orjson
-from pydantic import ValidationError as V2ValidationError
-from serpyco_rs import SchemaValidationError
 
 
 class EntrypointOutput:

airbyte_cdk/test/mock_http/mocker.py

@@ -7,6 +7,7 @@ from types import TracebackType
 from typing import Callable, List, Optional, Union
 
 import requests_mock
+
 from airbyte_cdk.test.mock_http import HttpRequest, HttpRequestMatcher, HttpResponse
 
 

airbyte_cdk/utils/schema_inferrer.py

@@ -5,11 +5,12 @@
 from collections import defaultdict
 from typing import Any, Dict, List, Mapping, Optional
 
-from airbyte_cdk.models import AirbyteRecordMessage
 from genson import SchemaBuilder, SchemaNode
 from genson.schema.strategies.object import Object
 from genson.schema.strategies.scalar import Number
 
+from airbyte_cdk.models import AirbyteRecordMessage
+
 # schema keywords
 _TYPE = "type"
 _NULL_TYPE = "null"

airbyte_cdk/utils/slice_hasher.py

@@ -1,6 +1,6 @@
 import hashlib
 import json
-from typing import Any, Mapping, Optional, Final
+from typing import Any, Final, Mapping, Optional
 
 
 class SliceEncoder(json.JSONEncoder):

airbyte_cdk/utils/traced_exception.py

@@ -5,6 +5,8 @@ import time
 import traceback
 from typing import Optional
 
+from orjson import orjson
+
 from airbyte_cdk.models import (
     AirbyteConnectionStatus,
     AirbyteErrorTraceMessage,
@@ -18,7 +20,6 @@ from airbyte_cdk.models import (
 )
 from airbyte_cdk.models import Type as MessageType
 from airbyte_cdk.utils.airbyte_secrets_utils import filter_secrets
-from orjson import orjson
 
 
 class AirbyteTracedException(Exception):

{airbyte_cdk-6.7.0rc3.dist-info → airbyte_cdk-6.7.1.dev0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 6.7.0rc3
+Version: 6.7.1.dev0
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://airbyte.com
 License: MIT
@@ -25,7 +25,7 @@ Requires-Dist: Deprecated (>=1.2,<1.3)
 Requires-Dist: Jinja2 (>=3.1.2,<3.2.0)
 Requires-Dist: PyYAML (>=6.0.1,<7.0.0)
 Requires-Dist: Sphinx (>=4.2,<4.3) ; extra == "sphinx-docs"
-Requires-Dist: airbyte-protocol-models-dataclasses (>=0.13,<0.14)
+Requires-Dist: airbyte-protocol-models-dataclasses (>=0.14,<0.15)
 Requires-Dist: avro (>=1.11.2,<1.12.0) ; extra == "file-based"
 Requires-Dist: backoff
 Requires-Dist: cachetools
@@ -170,6 +170,13 @@ Installing all extras is required to run the full suite of unit tests.
 
 To see all available scripts, run `poetry run poe`.
 
+#### Formatting the code
+
+- Iterate on the CDK code locally
+- Run `poetry run ruff format` to format your changes.
+
+To see all available `ruff` options, run `poetry run ruff`.
+
 ##### Autogenerated files
 
 Low-code CDK models are generated from `sources/declarative/declarative_component_schema.yaml`. If