airbyte-cdk 6.37.0.dev1__py3-none-any.whl → 6.37.2__py3-none-any.whl

airbyte_cdk/sources/declarative/requesters/http_requester.py

@@ -25,8 +25,8 @@ from airbyte_cdk.sources.message import MessageRepository, NoopMessageRepository
 from airbyte_cdk.sources.streams.call_rate import APIBudget
 from airbyte_cdk.sources.streams.http import HttpClient
 from airbyte_cdk.sources.streams.http.error_handlers import ErrorHandler
-from airbyte_cdk.sources.types import Config, StreamSlice, StreamState
-from airbyte_cdk.utils.mapping_helpers import combine_mappings
+from airbyte_cdk.sources.types import Config, EmptyString, StreamSlice, StreamState
+from airbyte_cdk.utils.mapping_helpers import combine_mappings, get_interpolation_context
 
 
 @dataclass
@@ -49,9 +49,10 @@ class HttpRequester(Requester):
 
     name: str
     url_base: Union[InterpolatedString, str]
-    path: Union[InterpolatedString, str]
     config: Config
     parameters: InitVar[Mapping[str, Any]]
+
+    path: Optional[Union[InterpolatedString, str]] = None
     authenticator: Optional[DeclarativeAuthenticator] = None
     http_method: Union[str, HttpMethod] = HttpMethod.GET
     request_options_provider: Optional[InterpolatedRequestOptionsProvider] = None
@@ -66,7 +67,9 @@ class HttpRequester(Requester):
 
     def __post_init__(self, parameters: Mapping[str, Any]) -> None:
         self._url_base = InterpolatedString.create(self.url_base, parameters=parameters)
-        self._path = InterpolatedString.create(self.path, parameters=parameters)
+        self._path = InterpolatedString.create(
+            self.path if self.path else EmptyString, parameters=parameters
+        )
         if self.request_options_provider is None:
             self._request_options_provider = InterpolatedRequestOptionsProvider(
                 config=self.config, parameters=parameters
@@ -85,7 +88,7 @@ class HttpRequester(Requester):
         self._parameters = parameters
 
         if self.error_handler is not None and hasattr(self.error_handler, "backoff_strategies"):
-            backoff_strategies = self.error_handler.backoff_strategies
+            backoff_strategies = self.error_handler.backoff_strategies  # type: ignore
         else:
             backoff_strategies = None
 
@@ -112,21 +115,33 @@ class HttpRequester(Requester):
     def get_authenticator(self) -> DeclarativeAuthenticator:
         return self._authenticator
 
-    def get_url_base(self) -> str:
-        return os.path.join(self._url_base.eval(self.config), "")
+    def get_url_base(
+        self,
+        *,
+        stream_state: Optional[StreamState] = None,
+        stream_slice: Optional[StreamSlice] = None,
+        next_page_token: Optional[Mapping[str, Any]] = None,
+    ) -> str:
+        interpolation_context = get_interpolation_context(
+            stream_state=stream_state,
+            stream_slice=stream_slice,
+            next_page_token=next_page_token,
+        )
+        return os.path.join(self._url_base.eval(self.config, **interpolation_context), EmptyString)
 
     def get_path(
         self,
         *,
-        stream_state: Optional[StreamState],
-        stream_slice: Optional[StreamSlice],
-        next_page_token: Optional[Mapping[str, Any]],
+        stream_state: Optional[StreamState] = None,
+        stream_slice: Optional[StreamSlice] = None,
+        next_page_token: Optional[Mapping[str, Any]] = None,
     ) -> str:
-        kwargs = {
-            "stream_slice": stream_slice,
-            "next_page_token": next_page_token,
-        }
-        path = str(self._path.eval(self.config, **kwargs))
+        interpolation_context = get_interpolation_context(
+            stream_state=stream_state,
+            stream_slice=stream_slice,
+            next_page_token=next_page_token,
+        )
+        path = str(self._path.eval(self.config, **interpolation_context))
         return path.lstrip("/")
 
     def get_method(self) -> HttpMethod:
@@ -324,7 +339,20 @@ class HttpRequester(Requester):
 
     @classmethod
     def _join_url(cls, url_base: str, path: str) -> str:
-        return urljoin(url_base, path)
+        """
+        Joins a base URL with a given path and returns the resulting URL with any trailing slash removed.
+
+        This method ensures that there are no duplicate slashes when concatenating the base URL and the path,
+        which is useful when the full URL is provided from an interpolation context.
+
+        Args:
+            url_base (str): The base URL to which the path will be appended.
+            path (str): The path to join with the base URL.
+
+        Returns:
+            str: The concatenated URL with the trailing slash (if any) removed.
+        """
+        return urljoin(url_base, path).rstrip("/")
 
     def send_request(
         self,
@@ -341,7 +369,11 @@ class HttpRequester(Requester):
         request, response = self._http_client.send_request(
             http_method=self.get_method().value,
             url=self._join_url(
-                self.get_url_base(),
+                self.get_url_base(
+                    stream_state=stream_state,
+                    stream_slice=stream_slice,
+                    next_page_token=next_page_token,
+                ),
                 path
                 or self.get_path(
                     stream_state=stream_state,

airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py

@@ -25,6 +25,7 @@ from airbyte_cdk.sources.declarative.requesters.request_path import RequestPath
 from airbyte_cdk.sources.types import Config, Record, StreamSlice, StreamState
 from airbyte_cdk.utils.mapping_helpers import (
     _validate_component_request_option_paths,
+    get_interpolation_context,
 )
 
 
@@ -150,11 +151,22 @@ class DefaultPaginator(Paginator):
         else:
             return None
 
-    def path(self, next_page_token: Optional[Mapping[str, Any]]) -> Optional[str]:
+    def path(
+        self,
+        next_page_token: Optional[Mapping[str, Any]],
+        stream_state: Optional[Mapping[str, Any]] = None,
+        stream_slice: Optional[StreamSlice] = None,
+    ) -> Optional[str]:
         token = next_page_token.get("next_page_token") if next_page_token else None
         if token and self.page_token_option and isinstance(self.page_token_option, RequestPath):
+            # make additional interpolation context
+            interpolation_context = get_interpolation_context(
+                stream_state=stream_state,
+                stream_slice=stream_slice,
+                next_page_token=next_page_token,
+            )
             # Replace url base to only return the path
-            return str(token).replace(self.url_base.eval(self.config), "")  # type: ignore # url_base is casted to a InterpolatedString in __post_init__
+            return str(token).replace(self.url_base.eval(self.config, **interpolation_context), "")  # type: ignore # url_base is casted to a InterpolatedString in __post_init__
         else:
             return None
 
@@ -258,8 +270,17 @@ class PaginatorTestReadDecorator(Paginator):
             response, last_page_size, last_record, last_page_token_value
         )
 
-    def path(self, next_page_token: Optional[Mapping[str, Any]]) -> Optional[str]:
-        return self._decorated.path(next_page_token)
+    def path(
+        self,
+        next_page_token: Optional[Mapping[str, Any]],
+        stream_state: Optional[Mapping[str, Any]] = None,
+        stream_slice: Optional[StreamSlice] = None,
+    ) -> Optional[str]:
+        return self._decorated.path(
+            next_page_token=next_page_token,
+            stream_state=stream_state,
+            stream_slice=stream_slice,
+        )
 
     def get_request_params(
         self,

airbyte_cdk/sources/declarative/requesters/paginators/no_pagination.py

@@ -19,7 +19,12 @@ class NoPagination(Paginator):
 
     parameters: InitVar[Mapping[str, Any]]
 
-    def path(self, next_page_token: Optional[Mapping[str, Any]]) -> Optional[str]:
+    def path(
+        self,
+        next_page_token: Optional[Mapping[str, Any]],
+        stream_state: Optional[Mapping[str, Any]] = None,
+        stream_slice: Optional[StreamSlice] = None,
+    ) -> Optional[str]:
         return None
 
     def get_request_params(

airbyte_cdk/sources/declarative/requesters/paginators/paginator.py

@@ -11,7 +11,7 @@ import requests
 from airbyte_cdk.sources.declarative.requesters.request_options.request_options_provider import (
     RequestOptionsProvider,
 )
-from airbyte_cdk.sources.types import Record
+from airbyte_cdk.sources.types import Record, StreamSlice
 
 
 @dataclass
@@ -49,7 +49,12 @@ class Paginator(ABC, RequestOptionsProvider):
         pass
 
     @abstractmethod
-    def path(self, next_page_token: Optional[Mapping[str, Any]]) -> Optional[str]:
+    def path(
+        self,
+        next_page_token: Optional[Mapping[str, Any]],
+        stream_state: Optional[Mapping[str, Any]] = None,
+        stream_slice: Optional[StreamSlice] = None,
+    ) -> Optional[str]:
         """
         Returns the URL path to hit to fetch the next page of records
 

airbyte_cdk/sources/declarative/requesters/requester.py

@@ -35,7 +35,13 @@ class Requester(RequestOptionsProvider):
         pass
 
     @abstractmethod
-    def get_url_base(self) -> str:
+    def get_url_base(
+        self,
+        *,
+        stream_state: Optional[StreamState],
+        stream_slice: Optional[StreamSlice],
+        next_page_token: Optional[Mapping[str, Any]],
+    ) -> str:
         """
         :return: URL base for the  API endpoint e.g: if you wanted to hit https://myapi.com/v1/some_entity then this should return "https://myapi.com/v1/"
         """

airbyte_cdk/sources/declarative/retrievers/async_retriever.py

@@ -1,13 +1,12 @@
 # Copyright (c) 2024 Airbyte, Inc., all rights reserved.
 
 
-from dataclasses import InitVar, dataclass
+from dataclasses import InitVar, dataclass, field
 from typing import Any, Iterable, Mapping, Optional
 
 from typing_extensions import deprecated
 
 from airbyte_cdk.sources.declarative.async_job.job import AsyncJob
-from airbyte_cdk.sources.declarative.async_job.job_orchestrator import AsyncPartition
 from airbyte_cdk.sources.declarative.extractors.record_selector import RecordSelector
 from airbyte_cdk.sources.declarative.partition_routers.async_job_partition_router import (
     AsyncJobPartitionRouter,
@@ -16,6 +15,7 @@ from airbyte_cdk.sources.declarative.retrievers.retriever import Retriever
 from airbyte_cdk.sources.source import ExperimentalClassWarning
 from airbyte_cdk.sources.streams.core import StreamData
 from airbyte_cdk.sources.types import Config, StreamSlice, StreamState
+from airbyte_cdk.sources.utils.slice_logger import AlwaysLogSliceLogger
 
 
 @deprecated(
@@ -28,6 +28,10 @@ class AsyncRetriever(Retriever):
     parameters: InitVar[Mapping[str, Any]]
     record_selector: RecordSelector
     stream_slicer: AsyncJobPartitionRouter
+    slice_logger: AlwaysLogSliceLogger = field(
+        init=False,
+        default_factory=lambda: AlwaysLogSliceLogger(),
+    )
 
     def __post_init__(self, parameters: Mapping[str, Any]) -> None:
         self._parameters = parameters
@@ -75,13 +79,16 @@ class AsyncRetriever(Retriever):
         return stream_slice.extra_fields.get("jobs", []) if stream_slice else []
 
     def stream_slices(self) -> Iterable[Optional[StreamSlice]]:
-        return self.stream_slicer.stream_slices()
+        yield from self.stream_slicer.stream_slices()
 
     def read_records(
         self,
         records_schema: Mapping[str, Any],
         stream_slice: Optional[StreamSlice] = None,
     ) -> Iterable[StreamData]:
+        # emit the slice_descriptor log message, for connector builder TestRead
+        yield self.slice_logger.create_slice_log_message(stream_slice.cursor_slice)  # type: ignore
+
         stream_state: StreamState = self._get_stream_state()
         jobs: Iterable[AsyncJob] = self._validate_and_get_stream_slice_jobs(stream_slice)
         records: Iterable[Mapping[str, Any]] = self.stream_slicer.fetch_records(jobs)

airbyte_cdk/sources/declarative/retrievers/simple_retriever.py

@@ -234,13 +234,22 @@ class SimpleRetriever(Retriever):
             raise ValueError("Request body json cannot be a string")
         return body_json
 
-    def _paginator_path(self, next_page_token: Optional[Mapping[str, Any]] = None) -> Optional[str]:
+    def _paginator_path(
+        self,
+        next_page_token: Optional[Mapping[str, Any]] = None,
+        stream_state: Optional[Mapping[str, Any]] = None,
+        stream_slice: Optional[StreamSlice] = None,
+    ) -> Optional[str]:
         """
         If the paginator points to a path, follow it, else return nothing so the requester is used.
         :param next_page_token:
         :return:
         """
-        return self._paginator.path(next_page_token=next_page_token)
+        return self._paginator.path(
+            next_page_token=next_page_token,
+            stream_state=stream_state,
+            stream_slice=stream_slice,
+        )
 
     def _parse_response(
         self,
@@ -299,7 +308,11 @@ class SimpleRetriever(Retriever):
         next_page_token: Optional[Mapping[str, Any]] = None,
     ) -> Optional[requests.Response]:
         return self.requester.send_request(
-            path=self._paginator_path(next_page_token=next_page_token),
+            path=self._paginator_path(
+                next_page_token=next_page_token,
+                stream_state=stream_state,
+                stream_slice=stream_slice,
+            ),
             stream_state=stream_state,
             stream_slice=stream_slice,
             next_page_token=next_page_token,
@@ -570,7 +583,11 @@ class SimpleRetrieverTestReadDecorator(SimpleRetriever):
         next_page_token: Optional[Mapping[str, Any]] = None,
     ) -> Optional[requests.Response]:
         return self.requester.send_request(
-            path=self._paginator_path(next_page_token=next_page_token),
+            path=self._paginator_path(
+                next_page_token=next_page_token,
+                stream_state=stream_state,
+                stream_slice=stream_slice,
+            ),
             stream_state=stream_state,
             stream_slice=stream_slice,
             next_page_token=next_page_token,

airbyte_cdk/sources/declarative/transformations/keys_to_snake_transformation.py

@@ -6,7 +6,7 @@ import re
 from dataclasses import dataclass
 from typing import Any, Dict, List, Optional
 
-import unidecode
+import anyascii
 
 from airbyte_cdk.sources.declarative.transformations import RecordTransformation
 from airbyte_cdk.sources.types import Config, StreamSlice, StreamState
@@ -48,7 +48,7 @@ class KeysToSnakeCaseTransformation(RecordTransformation):
         return self.tokens_to_snake_case(tokens)
 
     def normalize_key(self, key: str) -> str:
-        return unidecode.unidecode(key)
+        return str(anyascii.anyascii(key))
 
     def tokenize_key(self, key: str) -> List[str]:
         tokens = []

airbyte_cdk/sources/http_logger.py

@@ -15,11 +15,14 @@ def format_http_message(
     description: str,
     stream_name: Optional[str],
     is_auxiliary: bool | None = None,
+    type: Optional[str] = None,
 ) -> LogMessage:
+    request_type: str = type if type else "HTTP"
     request = response.request
     log_message = {
         "http": {
             "title": title,
+            "type": request_type,
             "description": description,
             "request": {
                 "method": request.method,

airbyte_cdk/sources/streams/concurrent/state_converters/abstract_stream_state_converter.py

@@ -4,7 +4,7 @@
 
 from abc import ABC, abstractmethod
 from enum import Enum
-from typing import TYPE_CHECKING, Any, List, MutableMapping, Optional, Tuple
+from typing import TYPE_CHECKING, Any, Callable, List, MutableMapping, Optional, Tuple
 
 if TYPE_CHECKING:
     from airbyte_cdk.sources.streams.concurrent.cursor import CursorField
@@ -12,6 +12,7 @@ if TYPE_CHECKING:
 
 class ConcurrencyCompatibleStateType(Enum):
     date_range = "date-range"
+    integer = "integer"
 
 
 class AbstractStreamStateConverter(ABC):

airbyte_cdk/sources/streams/concurrent/state_converters/incrementing_count_stream_state_converter.py

@@ -0,0 +1,92 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+from typing import Any, Callable, MutableMapping, Optional, Tuple
+
+from airbyte_cdk.sources.streams.concurrent.cursor import CursorField
+from airbyte_cdk.sources.streams.concurrent.state_converters.abstract_stream_state_converter import (
+    AbstractStreamStateConverter,
+    ConcurrencyCompatibleStateType,
+)
+
+
+class IncrementingCountStreamStateConverter(AbstractStreamStateConverter):
+    def _from_state_message(self, value: Any) -> Any:
+        return value
+
+    def _to_state_message(self, value: Any) -> Any:
+        return value
+
+    @classmethod
+    def get_end_provider(cls) -> Callable[[], float]:
+        return lambda: float("inf")
+
+    def convert_from_sequential_state(
+        self,
+        cursor_field: "CursorField",  # to deprecate as it is only needed for sequential state
+        stream_state: MutableMapping[str, Any],
+        start: Optional[Any],
+    ) -> Tuple[Any, MutableMapping[str, Any]]:
+        """
+        Convert the state message to the format required by the ConcurrentCursor.
+
+        e.g.
+        {
+            "state_type": ConcurrencyCompatibleStateType.date_range.value,
+            "metadata": { … },
+            "slices": [
+                {"start": "10", "end": "2021-01-18T21:18:20.000+00:00"},
+            ]
+        }
+        """
+        sync_start = self._get_sync_start(cursor_field, stream_state, start)
+        if self.is_state_message_compatible(stream_state):
+            return sync_start, stream_state
+
+        # Create a slice to represent the records synced during prior syncs.
+        # The start and end are the same to avoid confusion as to whether the records for this slice
+        # were actually synced
+        slices = [
+            {
+                self.START_KEY: start if start is not None else sync_start,
+                self.END_KEY: sync_start,  # this may not be relevant anymore
+                self.MOST_RECENT_RECORD_KEY: sync_start,
+            }
+        ]
+
+        return sync_start, {
+            "state_type": ConcurrencyCompatibleStateType.integer.value,
+            "slices": slices,
+            "legacy": stream_state,
+        }
+
+    def parse_value(self, value: int) -> int:
+        return value
+
+    @property
+    def zero_value(self) -> int:
+        return 0
+
+    def increment(self, value: int) -> int:
+        return value + 1
+
+    def output_format(self, value: int) -> int:
+        return value
+
+    def _get_sync_start(
+        self,
+        cursor_field: CursorField,
+        stream_state: MutableMapping[str, Any],
+        start: Optional[int],
+    ) -> int:
+        sync_start = start if start is not None else self.zero_value
+        prev_sync_low_water_mark: Optional[int] = (
+            stream_state[cursor_field.cursor_field_key]
+            if cursor_field.cursor_field_key in stream_state
+            else None
+        )
+        if prev_sync_low_water_mark and prev_sync_low_water_mark >= sync_start:
+            return prev_sync_low_water_mark
+        else:
+            return sync_start

airbyte_cdk/sources/streams/http/error_handlers/default_error_mapping.py

@@ -19,9 +19,9 @@ DEFAULT_ERROR_MAPPING: Mapping[Union[int, str, Type[Exception]], ErrorResolution
         error_message="Invalid Protocol Schema: The endpoint that data is being requested from is using an invalid or insecure. Exception: requests.exceptions.InvalidSchema",
     ),
     InvalidURL: ErrorResolution(
-        response_action=ResponseAction.FAIL,
-        failure_type=FailureType.config_error,
-        error_message="Invalid URL specified: The endpoint that data is being requested from is not a valid URL. Exception: requests.exceptions.InvalidURL",
+        response_action=ResponseAction.RETRY,
+        failure_type=FailureType.transient_error,
+        error_message="Invalid URL specified or DNS error occurred: The endpoint that data is being requested from is not a valid URL. Exception: requests.exceptions.InvalidURL",
     ),
     RequestException: ErrorResolution(
         response_action=ResponseAction.RETRY,

airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py

@@ -396,6 +396,7 @@ class AbstractOauth2Authenticator(AuthBase):
                     "Obtains access token",
                     self._NO_STREAM_NAME,
                     is_auxiliary=True,
+                    type="AUTH",
                 ),
             )
 

airbyte_cdk/sources/types.py

@@ -14,6 +14,7 @@ FieldPointer = List[str]
 Config = Mapping[str, Any]
 ConnectionDefinition = Mapping[str, Any]
 StreamState = Mapping[str, Any]
+EmptyString = str()
 
 
 class Record(Mapping[str, Any]):

airbyte_cdk/utils/mapping_helpers.py

@@ -10,7 +10,7 @@ from airbyte_cdk.sources.declarative.requesters.request_option import (
     RequestOption,
     RequestOptionType,
 )
-from airbyte_cdk.sources.types import Config
+from airbyte_cdk.sources.types import Config, StreamSlice, StreamState
 
 
 def _merge_mappings(
@@ -143,3 +143,20 @@ def _validate_component_request_option_paths(
             )
         except ValueError as error:
             raise ValueError(error)
+
+
+def get_interpolation_context(
+    stream_state: Optional[StreamState] = None,
+    stream_slice: Optional[StreamSlice] = None,
+    next_page_token: Optional[Mapping[str, Any]] = None,
+) -> Mapping[str, Any]:
+    return {
+        "stream_slice": stream_slice,
+        "next_page_token": next_page_token,
+        # update the context with extra fields, if passed.
+        **(
+            stream_slice.extra_fields
+            if stream_slice is not None and hasattr(stream_slice, "extra_fields")
+            else {}
+        ),
+    }

{airbyte_cdk-6.37.0.dev1.dist-info → airbyte_cdk-6.37.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 6.37.0.dev1
+Version: 6.37.2
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://airbyte.com
 License: MIT
@@ -22,13 +22,13 @@ Provides-Extra: sql
 Provides-Extra: vector-db-based
 Requires-Dist: Jinja2 (>=3.1.2,<3.2.0)
 Requires-Dist: PyYAML (>=6.0.1,<7.0.0)
-Requires-Dist: Unidecode (>=1.3,<2.0)
 Requires-Dist: airbyte-protocol-models-dataclasses (>=0.14,<0.15)
+Requires-Dist: anyascii (>=0.3.2,<0.4.0)
 Requires-Dist: avro (>=1.11.2,<1.13.0) ; extra == "file-based"
 Requires-Dist: backoff
 Requires-Dist: cachetools
 Requires-Dist: cohere (==4.21) ; extra == "vector-db-based"
-Requires-Dist: cryptography (>=42.0.5,<44.0.0)
+Requires-Dist: cryptography (>=44.0.0,<45.0.0)
 Requires-Dist: dpath (>=2.1.6,<3.0.0)
 Requires-Dist: dunamai (>=1.22.0,<2.0.0)
 Requires-Dist: fastavro (>=1.8.0,<1.9.0) ; extra == "file-based"
@@ -47,7 +47,7 @@ Requires-Dist: pandas (==2.2.2)
 Requires-Dist: pdf2image (==1.16.3) ; extra == "file-based"
 Requires-Dist: pdfminer.six (==20221105) ; extra == "file-based"
 Requires-Dist: psutil (==6.1.0)
-Requires-Dist: pyarrow (>=15.0.0,<15.1.0) ; extra == "file-based"
+Requires-Dist: pyarrow (>=19.0.0,<20.0.0) ; extra == "file-based"
 Requires-Dist: pydantic (>=2.7,<3.0)
 Requires-Dist: pyjwt (>=2.8.0,<3.0.0)
 Requires-Dist: pyrate-limiter (>=3.1.0,<3.2.0)