airbyte-cdk 6.38.0.dev0__py3-none-any.whl → 6.38.1__py3-none-any.whl

airbyte_cdk/entrypoint.py

@@ -22,7 +22,7 @@ from requests import PreparedRequest, Response, Session
 
 from airbyte_cdk.connector import TConfig
 from airbyte_cdk.exception_handler import init_uncaught_exception_handler
-from airbyte_cdk.logger import PRINT_BUFFER, init_logger
+from airbyte_cdk.logger import init_logger
 from airbyte_cdk.models import (
     AirbyteConnectionStatus,
     AirbyteMessage,
@@ -337,11 +337,11 @@ def launch(source: Source, args: List[str]) -> None:
     parsed_args = source_entrypoint.parse_args(args)
     # temporarily removes the PrintBuffer because we're seeing weird print behavior for concurrent syncs
     # Refer to: https://github.com/airbytehq/oncall/issues/6235
-    with PRINT_BUFFER:
-        for message in source_entrypoint.run(parsed_args):
-            # simply printing is creating issues for concurrent CDK as Python uses different two instructions to print: one for the message and
-            # the other for the break line. Adding `\n` to the message ensure that both are printed at the same time
-            print(f"{message}\n", end="")
+    # with PrintBuffer():
+    for message in source_entrypoint.run(parsed_args):
+        # simply printing is creating issues for concurrent CDK as Python uses different two instructions to print: one for the message and
+        # the other for the break line. Adding `\n` to the message ensure that both are printed at the same time
+        print(f"{message}\n", end="", flush=True)
 
 
 def _init_internal_request_filter() -> None:

airbyte_cdk/logger.py

@@ -16,11 +16,8 @@ from airbyte_cdk.models import (
     Level,
     Type,
 )
-from airbyte_cdk.utils import PrintBuffer
 from airbyte_cdk.utils.airbyte_secrets_utils import filter_secrets
 
-PRINT_BUFFER = PrintBuffer(flush_interval=0.1)
-
 LOGGING_CONFIG = {
     "version": 1,
     "disable_existing_loggers": False,
@@ -30,7 +27,7 @@ LOGGING_CONFIG = {
     "handlers": {
         "console": {
             "class": "logging.StreamHandler",
-            "stream": PRINT_BUFFER,
+            "stream": "ext://sys.stdout",
             "formatter": "airbyte",
         },
     },

airbyte_cdk/sources/declarative/datetime/__init__.py

@@ -1,7 +1,3 @@
 #
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
-
-from airbyte_cdk.sources.declarative.datetime.min_max_datetime import MinMaxDatetime
-
-__all__ = ["MinMaxDatetime"]

airbyte_cdk/sources/declarative/datetime/datetime_parser.py

@@ -29,6 +29,8 @@ class DatetimeParser:
             return datetime.datetime.fromtimestamp(int(date), tz=datetime.timezone.utc)
         elif format == "%s_as_float":
             return datetime.datetime.fromtimestamp(float(date), tz=datetime.timezone.utc)
+        elif format == "%epoch_microseconds":
+            return self._UNIX_EPOCH + datetime.timedelta(microseconds=int(date))
         elif format == "%ms":
             return self._UNIX_EPOCH + datetime.timedelta(milliseconds=int(date))
         elif "%_ms" in format:
@@ -46,6 +48,8 @@ class DatetimeParser:
             return str(int(dt.timestamp()))
         if format == "%s_as_float":
             return str(float(dt.timestamp()))
+        if format == "%epoch_microseconds":
+            return str(int(dt.timestamp() * 1_000_000))
         if format == "%ms":
             # timstamp() returns a float representing the number of seconds since the unix epoch
             return str(int(dt.timestamp() * 1000))

airbyte_cdk/sources/declarative/declarative_component_schema.yaml

@@ -1794,7 +1794,6 @@ definitions:
     type: object
     required:
       - type
-      - path
       - url_base
     properties:
       type:
@@ -1806,9 +1805,18 @@ definitions:
         type: string
         interpolation_context:
           - config
+          - next_page_token
+          - stream_interval
+          - stream_partition
+          - stream_slice
+          - creation_response
+          - polling_response
+          - download_target
         examples:
           - "https://connect.squareup.com/v2"
-          - "{{ config['base_url'] or 'https://app.posthog.com'}}/api/"
+          - "{{ config['base_url'] or 'https://app.posthog.com'}}/api"
+          - "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups"
+          - "https://example.com/api/v1/resource/{{ next_page_token['id'] }}"
       path:
         title: URL Path
         description: Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.

airbyte_cdk/sources/declarative/decoders/composite_raw_decoder.py

@@ -151,6 +151,10 @@ class CompositeRawDecoder(Decoder):
         self, response: requests.Response
     ) -> Generator[MutableMapping[str, Any], None, None]:
         if self.is_stream_response():
+            # urllib mentions that some interfaces don't play nice with auto_close [here](https://urllib3.readthedocs.io/en/stable/user-guide.html#using-io-wrappers-with-response-content)
+            # We have indeed observed some issues with CSV parsing. Hence, we will manage the closing of the file ourselves until we find a better solution.
+            response.raw.auto_close = False
             yield from self.parser.parse(data=response.raw)  # type: ignore[arg-type]
+            response.raw.close()
         else:
             yield from self.parser.parse(data=io.BytesIO(response.content))

airbyte_cdk/sources/declarative/interpolation/macros.py

@@ -12,6 +12,8 @@ import pytz
 from dateutil import parser
 from isodate import parse_duration
 
+from airbyte_cdk.sources.declarative.datetime.datetime_parser import DatetimeParser
+
 """
 This file contains macros that can be evaluated by a `JinjaInterpolation` object
 """
@@ -171,11 +173,7 @@ def format_datetime(
     dt_datetime = (
         datetime.datetime.strptime(dt, input_format) if input_format else str_to_datetime(dt)
     )
-    if format == "%s":
-        return str(int(dt_datetime.timestamp()))
-    elif format == "%ms":
-        return str(int(dt_datetime.timestamp() * 1_000_000))
-    return dt_datetime.strftime(format)
+    return DatetimeParser().format(dt=dt_datetime, format=format)
 
 
 _macros_list = [

airbyte_cdk/sources/declarative/models/declarative_component_schema.py

@@ -939,7 +939,7 @@ class MinMaxDatetime(BaseModel):
     )
     datetime_format: Optional[str] = Field(
         "",
-        description='Format of the datetime value. Defaults to "%Y-%m-%dT%H:%M:%S.%f%z" if left empty. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available:\n  * **%s**: Epoch unix timestamp - `1686218963`\n  * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n  * **%ms**: Epoch unix timestamp - `1686218963123`\n  * **%a**: Weekday (abbreviated) - `Sun`\n  * **%A**: Weekday (full) - `Sunday`\n  * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n  * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n  * **%b**: Month (abbreviated) - `Jan`\n  * **%B**: Month (full) - `January`\n  * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n  * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n  * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n  * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n  * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n  * **%p**: AM/PM indicator\n  * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n  * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n  * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n  * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n  * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n  * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n  * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n  * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n  * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n  * **%x**: Date representation - `08/16/1988`\n  * **%X**: Time representation - `21:30:00`\n  * **%%**: Literal \'%\' character\n\n  Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n',
+        description='Format of the datetime value. Defaults to "%Y-%m-%dT%H:%M:%S.%f%z" if left empty. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available:\n  * **%s**: Epoch unix timestamp - `1686218963`\n  * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n  * **%ms**: Epoch unix timestamp - `1686218963123`\n  * **%a**: Weekday (abbreviated) - `Sun`\n  * **%A**: Weekday (full) - `Sunday`\n  * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n  * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n  * **%b**: Month (abbreviated) - `Jan`\n  * **%B**: Month (full) - `January`\n  * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n  * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n  * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n  * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n  * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n  * **%p**: AM/PM indicator\n  * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n  * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n  * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n  * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`, `001`, ..., `999`\n  * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n  * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n  * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n  * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n  * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n  * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n  * **%x**: Date representation - `08/16/1988`\n  * **%X**: Time representation - `21:30:00`\n  * **%%**: Literal \'%\' character\n\n  Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n',
         examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s"],
         title="Datetime Format",
     )
@@ -1545,7 +1545,7 @@ class DatetimeBasedCursor(BaseModel):
     )
     datetime_format: str = Field(
         ...,
-        description="The datetime format used to format the datetime values that are sent in outgoing requests to the API. Use placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n  * **%s**: Epoch unix timestamp - `1686218963`\n  * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n  * **%ms**: Epoch unix timestamp (milliseconds) - `1686218963123`\n  * **%a**: Weekday (abbreviated) - `Sun`\n  * **%A**: Weekday (full) - `Sunday`\n  * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n  * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n  * **%b**: Month (abbreviated) - `Jan`\n  * **%B**: Month (full) - `January`\n  * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n  * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n  * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n  * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n  * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n  * **%p**: AM/PM indicator\n  * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n  * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n  * **%f**: Microsecond (zero-padded to 6 digits) - `000000`\n  * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n  * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n  * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n  * **%U**: Week number of the year (starting Sunday) - `00`, ..., `53`\n  * **%W**: Week number of the year (starting Monday) - `00`, ..., `53`\n  * **%c**: Date and time - `Tue Aug 16 21:30:00 1988`\n  * **%x**: Date standard format - `08/16/1988`\n  * **%X**: Time standard format - `21:30:00`\n  * **%%**: Literal '%' character\n\n  Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n",
+        description="The datetime format used to format the datetime values that are sent in outgoing requests to the API. Use placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n  * **%s**: Epoch unix timestamp - `1686218963`\n  * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n  * **%ms**: Epoch unix timestamp (milliseconds) - `1686218963123`\n  * **%a**: Weekday (abbreviated) - `Sun`\n  * **%A**: Weekday (full) - `Sunday`\n  * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n  * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n  * **%b**: Month (abbreviated) - `Jan`\n  * **%B**: Month (full) - `January`\n  * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n  * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n  * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n  * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n  * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n  * **%p**: AM/PM indicator\n  * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n  * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n  * **%f**: Microsecond (zero-padded to 6 digits) - `000000`\n  * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`\n  * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n  * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n  * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n  * **%U**: Week number of the year (starting Sunday) - `00`, ..., `53`\n  * **%W**: Week number of the year (starting Monday) - `00`, ..., `53`\n  * **%c**: Date and time - `Tue Aug 16 21:30:00 1988`\n  * **%x**: Date standard format - `08/16/1988`\n  * **%X**: Time standard format - `21:30:00`\n  * **%%**: Literal '%' character\n\n  Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n",
         examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s", "%ms", "%s_as_float"],
         title="Outgoing Datetime Format",
     )
@@ -2072,12 +2072,14 @@ class HttpRequester(BaseModel):
         description="Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.",
         examples=[
             "https://connect.squareup.com/v2",
-            "{{ config['base_url'] or 'https://app.posthog.com'}}/api/",
+            "{{ config['base_url'] or 'https://app.posthog.com'}}/api",
+            "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups",
+            "https://example.com/api/v1/resource/{{ next_page_token['id'] }}",
         ],
         title="API Base URL",
     )
-    path: str = Field(
-        ...,
+    path: Optional[str] = Field(
+        None,
         description="Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.",
         examples=[
             "/products",

airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py

@@ -56,7 +56,7 @@ from airbyte_cdk.sources.declarative.auth.token_provider import (
 )
 from airbyte_cdk.sources.declarative.checks import CheckDynamicStream, CheckStream
 from airbyte_cdk.sources.declarative.concurrency_level import ConcurrencyLevel
-from airbyte_cdk.sources.declarative.datetime import MinMaxDatetime
+from airbyte_cdk.sources.declarative.datetime.min_max_datetime import MinMaxDatetime
 from airbyte_cdk.sources.declarative.declarative_stream import DeclarativeStream
 from airbyte_cdk.sources.declarative.decoders import (
     Decoder,

airbyte_cdk/sources/declarative/requesters/http_requester.py

@@ -25,8 +25,8 @@ from airbyte_cdk.sources.message import MessageRepository, NoopMessageRepository
 from airbyte_cdk.sources.streams.call_rate import APIBudget
 from airbyte_cdk.sources.streams.http import HttpClient
 from airbyte_cdk.sources.streams.http.error_handlers import ErrorHandler
-from airbyte_cdk.sources.types import Config, StreamSlice, StreamState
-from airbyte_cdk.utils.mapping_helpers import combine_mappings
+from airbyte_cdk.sources.types import Config, EmptyString, StreamSlice, StreamState
+from airbyte_cdk.utils.mapping_helpers import combine_mappings, get_interpolation_context
 
 
 @dataclass
@@ -49,9 +49,10 @@ class HttpRequester(Requester):
 
     name: str
     url_base: Union[InterpolatedString, str]
-    path: Union[InterpolatedString, str]
     config: Config
     parameters: InitVar[Mapping[str, Any]]
+
+    path: Optional[Union[InterpolatedString, str]] = None
     authenticator: Optional[DeclarativeAuthenticator] = None
     http_method: Union[str, HttpMethod] = HttpMethod.GET
     request_options_provider: Optional[InterpolatedRequestOptionsProvider] = None
@@ -66,7 +67,9 @@ class HttpRequester(Requester):
 
     def __post_init__(self, parameters: Mapping[str, Any]) -> None:
         self._url_base = InterpolatedString.create(self.url_base, parameters=parameters)
-        self._path = InterpolatedString.create(self.path, parameters=parameters)
+        self._path = InterpolatedString.create(
+            self.path if self.path else EmptyString, parameters=parameters
+        )
         if self.request_options_provider is None:
             self._request_options_provider = InterpolatedRequestOptionsProvider(
                 config=self.config, parameters=parameters
@@ -112,27 +115,33 @@ class HttpRequester(Requester):
     def get_authenticator(self) -> DeclarativeAuthenticator:
         return self._authenticator
 
-    def get_url_base(self) -> str:
-        return os.path.join(self._url_base.eval(self.config), "")
+    def get_url_base(
+        self,
+        *,
+        stream_state: Optional[StreamState] = None,
+        stream_slice: Optional[StreamSlice] = None,
+        next_page_token: Optional[Mapping[str, Any]] = None,
+    ) -> str:
+        interpolation_context = get_interpolation_context(
+            stream_state=stream_state,
+            stream_slice=stream_slice,
+            next_page_token=next_page_token,
+        )
+        return os.path.join(self._url_base.eval(self.config, **interpolation_context), EmptyString)
 
     def get_path(
         self,
         *,
-        stream_state: Optional[StreamState],
-        stream_slice: Optional[StreamSlice],
-        next_page_token: Optional[Mapping[str, Any]],
+        stream_state: Optional[StreamState] = None,
+        stream_slice: Optional[StreamSlice] = None,
+        next_page_token: Optional[Mapping[str, Any]] = None,
     ) -> str:
-        kwargs = {
-            "stream_slice": stream_slice,
-            "next_page_token": next_page_token,
-            # update the interpolation context with extra fields, if passed.
-            **(
-                stream_slice.extra_fields
-                if stream_slice is not None and hasattr(stream_slice, "extra_fields")
-                else {}
-            ),
-        }
-        path = str(self._path.eval(self.config, **kwargs))
+        interpolation_context = get_interpolation_context(
+            stream_state=stream_state,
+            stream_slice=stream_slice,
+            next_page_token=next_page_token,
+        )
+        path = str(self._path.eval(self.config, **interpolation_context))
         return path.lstrip("/")
 
     def get_method(self) -> HttpMethod:
@@ -330,7 +339,20 @@ class HttpRequester(Requester):
 
     @classmethod
     def _join_url(cls, url_base: str, path: str) -> str:
-        return urljoin(url_base, path)
+        """
+        Joins a base URL with a given path and returns the resulting URL with any trailing slash removed.
+
+        This method ensures that there are no duplicate slashes when concatenating the base URL and the path,
+        which is useful when the full URL is provided from an interpolation context.
+
+        Args:
+            url_base (str): The base URL to which the path will be appended.
+            path (str): The path to join with the base URL.
+
+        Returns:
+            str: The concatenated URL with the trailing slash (if any) removed.
+        """
+        return urljoin(url_base, path).rstrip("/")
 
     def send_request(
         self,
@@ -347,7 +369,11 @@ class HttpRequester(Requester):
         request, response = self._http_client.send_request(
             http_method=self.get_method().value,
             url=self._join_url(
-                self.get_url_base(),
+                self.get_url_base(
+                    stream_state=stream_state,
+                    stream_slice=stream_slice,
+                    next_page_token=next_page_token,
+                ),
                 path
                 or self.get_path(
                     stream_state=stream_state,

airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py

@@ -25,6 +25,7 @@ from airbyte_cdk.sources.declarative.requesters.request_path import RequestPath
 from airbyte_cdk.sources.types import Config, Record, StreamSlice, StreamState
 from airbyte_cdk.utils.mapping_helpers import (
     _validate_component_request_option_paths,
+    get_interpolation_context,
 )
 
 
@@ -150,11 +151,22 @@ class DefaultPaginator(Paginator):
         else:
             return None
 
-    def path(self, next_page_token: Optional[Mapping[str, Any]]) -> Optional[str]:
+    def path(
+        self,
+        next_page_token: Optional[Mapping[str, Any]],
+        stream_state: Optional[Mapping[str, Any]] = None,
+        stream_slice: Optional[StreamSlice] = None,
+    ) -> Optional[str]:
         token = next_page_token.get("next_page_token") if next_page_token else None
         if token and self.page_token_option and isinstance(self.page_token_option, RequestPath):
+            # make additional interpolation context
+            interpolation_context = get_interpolation_context(
+                stream_state=stream_state,
+                stream_slice=stream_slice,
+                next_page_token=next_page_token,
+            )
             # Replace url base to only return the path
-            return str(token).replace(self.url_base.eval(self.config), "")  # type: ignore # url_base is casted to a InterpolatedString in __post_init__
+            return str(token).replace(self.url_base.eval(self.config, **interpolation_context), "")  # type: ignore # url_base is casted to a InterpolatedString in __post_init__
         else:
             return None
 
@@ -258,8 +270,17 @@ class PaginatorTestReadDecorator(Paginator):
             response, last_page_size, last_record, last_page_token_value
         )
 
-    def path(self, next_page_token: Optional[Mapping[str, Any]]) -> Optional[str]:
-        return self._decorated.path(next_page_token)
+    def path(
+        self,
+        next_page_token: Optional[Mapping[str, Any]],
+        stream_state: Optional[Mapping[str, Any]] = None,
+        stream_slice: Optional[StreamSlice] = None,
+    ) -> Optional[str]:
+        return self._decorated.path(
+            next_page_token=next_page_token,
+            stream_state=stream_state,
+            stream_slice=stream_slice,
+        )
 
     def get_request_params(
         self,

airbyte_cdk/sources/declarative/requesters/paginators/no_pagination.py

@@ -19,7 +19,12 @@ class NoPagination(Paginator):
 
     parameters: InitVar[Mapping[str, Any]]
 
-    def path(self, next_page_token: Optional[Mapping[str, Any]]) -> Optional[str]:
+    def path(
+        self,
+        next_page_token: Optional[Mapping[str, Any]],
+        stream_state: Optional[Mapping[str, Any]] = None,
+        stream_slice: Optional[StreamSlice] = None,
+    ) -> Optional[str]:
         return None
 
     def get_request_params(

airbyte_cdk/sources/declarative/requesters/paginators/paginator.py

@@ -11,7 +11,7 @@ import requests
 from airbyte_cdk.sources.declarative.requesters.request_options.request_options_provider import (
     RequestOptionsProvider,
 )
-from airbyte_cdk.sources.types import Record
+from airbyte_cdk.sources.types import Record, StreamSlice
 
 
 @dataclass
@@ -49,7 +49,12 @@ class Paginator(ABC, RequestOptionsProvider):
         pass
 
     @abstractmethod
-    def path(self, next_page_token: Optional[Mapping[str, Any]]) -> Optional[str]:
+    def path(
+        self,
+        next_page_token: Optional[Mapping[str, Any]],
+        stream_state: Optional[Mapping[str, Any]] = None,
+        stream_slice: Optional[StreamSlice] = None,
+    ) -> Optional[str]:
         """
         Returns the URL path to hit to fetch the next page of records
 

airbyte_cdk/sources/declarative/requesters/requester.py

@@ -35,7 +35,13 @@ class Requester(RequestOptionsProvider):
         pass
 
     @abstractmethod
-    def get_url_base(self) -> str:
+    def get_url_base(
+        self,
+        *,
+        stream_state: Optional[StreamState],
+        stream_slice: Optional[StreamSlice],
+        next_page_token: Optional[Mapping[str, Any]],
+    ) -> str:
         """
         :return: URL base for the  API endpoint e.g: if you wanted to hit https://myapi.com/v1/some_entity then this should return "https://myapi.com/v1/"
         """

airbyte_cdk/sources/declarative/retrievers/simple_retriever.py

@@ -234,13 +234,22 @@ class SimpleRetriever(Retriever):
             raise ValueError("Request body json cannot be a string")
         return body_json
 
-    def _paginator_path(self, next_page_token: Optional[Mapping[str, Any]] = None) -> Optional[str]:
+    def _paginator_path(
+        self,
+        next_page_token: Optional[Mapping[str, Any]] = None,
+        stream_state: Optional[Mapping[str, Any]] = None,
+        stream_slice: Optional[StreamSlice] = None,
+    ) -> Optional[str]:
         """
         If the paginator points to a path, follow it, else return nothing so the requester is used.
         :param next_page_token:
         :return:
         """
-        return self._paginator.path(next_page_token=next_page_token)
+        return self._paginator.path(
+            next_page_token=next_page_token,
+            stream_state=stream_state,
+            stream_slice=stream_slice,
+        )
 
     def _parse_response(
         self,
@@ -299,7 +308,11 @@ class SimpleRetriever(Retriever):
         next_page_token: Optional[Mapping[str, Any]] = None,
     ) -> Optional[requests.Response]:
         return self.requester.send_request(
-            path=self._paginator_path(next_page_token=next_page_token),
+            path=self._paginator_path(
+                next_page_token=next_page_token,
+                stream_state=stream_state,
+                stream_slice=stream_slice,
+            ),
             stream_state=stream_state,
             stream_slice=stream_slice,
             next_page_token=next_page_token,
@@ -570,7 +583,11 @@ class SimpleRetrieverTestReadDecorator(SimpleRetriever):
         next_page_token: Optional[Mapping[str, Any]] = None,
     ) -> Optional[requests.Response]:
         return self.requester.send_request(
-            path=self._paginator_path(next_page_token=next_page_token),
+            path=self._paginator_path(
+                next_page_token=next_page_token,
+                stream_state=stream_state,
+                stream_slice=stream_slice,
+            ),
             stream_state=stream_state,
             stream_slice=stream_slice,
             next_page_token=next_page_token,

airbyte_cdk/sources/declarative/yaml_declarative_source.py

@@ -50,7 +50,6 @@ class YamlDeclarativeSource(ConcurrentDeclarativeSource[List[AirbyteStateMessage
 
     def _emit_manifest_debug_message(self, extra_args: dict[str, Any]) -> None:
         extra_args["path_to_yaml"] = self._path_to_yaml
-        self.logger.debug("declarative source created from parsed YAML manifest", extra=extra_args)
 
     @staticmethod
     def _parse(connection_definition_str: str) -> ConnectionDefinition:

airbyte_cdk/sources/streams/http/error_handlers/default_error_mapping.py

@@ -19,9 +19,9 @@ DEFAULT_ERROR_MAPPING: Mapping[Union[int, str, Type[Exception]], ErrorResolution
         error_message="Invalid Protocol Schema: The endpoint that data is being requested from is using an invalid or insecure. Exception: requests.exceptions.InvalidSchema",
     ),
     InvalidURL: ErrorResolution(
-        response_action=ResponseAction.FAIL,
-        failure_type=FailureType.config_error,
-        error_message="Invalid URL specified: The endpoint that data is being requested from is not a valid URL. Exception: requests.exceptions.InvalidURL",
+        response_action=ResponseAction.RETRY,
+        failure_type=FailureType.transient_error,
+        error_message="Invalid URL specified or DNS error occurred: The endpoint that data is being requested from is not a valid URL. Exception: requests.exceptions.InvalidURL",
     ),
     RequestException: ErrorResolution(
         response_action=ResponseAction.RETRY,

airbyte_cdk/sources/types.py

@@ -14,6 +14,7 @@ FieldPointer = List[str]
 Config = Mapping[str, Any]
 ConnectionDefinition = Mapping[str, Any]
 StreamState = Mapping[str, Any]
+EmptyString = str()
 
 
 class Record(Mapping[str, Any]):

airbyte_cdk/utils/mapping_helpers.py

@@ -10,7 +10,7 @@ from airbyte_cdk.sources.declarative.requesters.request_option import (
     RequestOption,
     RequestOptionType,
 )
-from airbyte_cdk.sources.types import Config
+from airbyte_cdk.sources.types import Config, StreamSlice, StreamState
 
 
 def _merge_mappings(
@@ -143,3 +143,20 @@ def _validate_component_request_option_paths(
             )
         except ValueError as error:
             raise ValueError(error)
+
+
+def get_interpolation_context(
+    stream_state: Optional[StreamState] = None,
+    stream_slice: Optional[StreamSlice] = None,
+    next_page_token: Optional[Mapping[str, Any]] = None,
+) -> Mapping[str, Any]:
+    return {
+        "stream_slice": stream_slice,
+        "next_page_token": next_page_token,
+        # update the context with extra fields, if passed.
+        **(
+            stream_slice.extra_fields
+            if stream_slice is not None and hasattr(stream_slice, "extra_fields")
+            else {}
+        ),
+    }

{airbyte_cdk-6.38.0.dev0.dist-info → airbyte_cdk-6.38.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 6.38.0.dev0
+Version: 6.38.1
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://airbyte.com
 License: MIT
@@ -28,7 +28,7 @@ Requires-Dist: avro (>=1.11.2,<1.13.0) ; extra == "file-based"
 Requires-Dist: backoff
 Requires-Dist: cachetools
 Requires-Dist: cohere (==4.21) ; extra == "vector-db-based"
-Requires-Dist: cryptography (>=42.0.5,<44.0.0)
+Requires-Dist: cryptography (>=44.0.0,<45.0.0)
 Requires-Dist: dpath (>=2.1.6,<3.0.0)
 Requires-Dist: dunamai (>=1.22.0,<2.0.0)
 Requires-Dist: fastavro (>=1.8.0,<1.9.0) ; extra == "file-based"
@@ -47,7 +47,7 @@ Requires-Dist: pandas (==2.2.2)
 Requires-Dist: pdf2image (==1.16.3) ; extra == "file-based"
 Requires-Dist: pdfminer.six (==20221105) ; extra == "file-based"
 Requires-Dist: psutil (==6.1.0)
-Requires-Dist: pyarrow (>=15.0.0,<15.1.0) ; extra == "file-based"
+Requires-Dist: pyarrow (>=19.0.0,<20.0.0) ; extra == "file-based"
 Requires-Dist: pydantic (>=2.7,<3.0)
 Requires-Dist: pyjwt (>=2.8.0,<3.0.0)
 Requires-Dist: pyrate-limiter (>=3.1.0,<3.2.0)

{airbyte_cdk-6.38.0.dev0.dist-info → airbyte_cdk-6.38.1.dist-info}/RECORD

@@ -26,9 +26,9 @@ airbyte_cdk/destinations/vector_db_based/indexer.py,sha256=beiSi2Uu67EoTr7yQSaCJ
 airbyte_cdk/destinations/vector_db_based/test_utils.py,sha256=MkqLiOJ5QyKbV4rNiJhe-BHM7FD-ADHQ4bQGf4c5lRY,1932
 airbyte_cdk/destinations/vector_db_based/utils.py,sha256=FOyEo8Lc-fY8UyhpCivhZtIqBRyxf3cUt6anmK03fUY,1127
 airbyte_cdk/destinations/vector_db_based/writer.py,sha256=nZ00xPiohElJmYktEZZIhr0m5EDETCHGhg0Lb2S7A20,5095
-airbyte_cdk/entrypoint.py,sha256=NRJv5BNZRSUEVTmNBa9N7ih6fW5sg4DwL0nkB9kI99Y,18570
+airbyte_cdk/entrypoint.py,sha256=xFLY2PV8mKXUaeBAknczbK6plrs4_B1WdWA6K3iaRJI,18555
 airbyte_cdk/exception_handler.py,sha256=D_doVl3Dt60ASXlJsfviOCswxGyKF2q0RL6rif3fNks,2013
-airbyte_cdk/logger.py,sha256=1cURbvawbunCAV178q-XhTHcbAQZTSf07WhU7U9AXWU,3744
+airbyte_cdk/logger.py,sha256=qi4UGuSYQQGaFaTVJlMD9lLppwqLXt1XBhwSXo-Q5IA,3660
 airbyte_cdk/models/__init__.py,sha256=MOTiuML2wShBaMSIwikdjyye2uUWBjo4J1QFSbnoiM4,2075
 airbyte_cdk/models/airbyte_protocol.py,sha256=MCmLir67-hF12YM5OKzeGbWrlxr7ChG_OQSE1xG8EIU,3748
 airbyte_cdk/models/airbyte_protocol_serializers.py,sha256=s6SaFB2CMrG_7jTQGn_fhFbQ1FUxhCxf5kq2RWGHMVI,1749
@@ -68,14 +68,14 @@ airbyte_cdk/sources/declarative/checks/connection_checker.py,sha256=MBRJo6WJlZQH
 airbyte_cdk/sources/declarative/concurrency_level/__init__.py,sha256=5XUqrmlstYlMM0j6crktlKQwALek0uiz2D3WdM46MyA,191
 airbyte_cdk/sources/declarative/concurrency_level/concurrency_level.py,sha256=YIwCTCpOr_QSNW4ltQK0yUGWInI8PKNY216HOOegYLk,2101
 airbyte_cdk/sources/declarative/concurrent_declarative_source.py,sha256=rAp-sgld4n8Tmybz-51m7VcYXqKwzKDpCJVr1elmkRc,26824
-airbyte_cdk/sources/declarative/datetime/__init__.py,sha256=l9LG7Qm6e5r_qgqfVKnx3mXYtg1I9MmMjomVIPfU4XA,177
-airbyte_cdk/sources/declarative/datetime/datetime_parser.py,sha256=0qs4hhmh_XOy2B4MHCn2qVMM79C6MizIBqnvpZj1aSE,2923
+airbyte_cdk/sources/declarative/datetime/__init__.py,sha256=4Hw-PX1-VgESLF16cDdvuYCzGJtHntThLF4qIiULWeo,61
+airbyte_cdk/sources/declarative/datetime/datetime_parser.py,sha256=_zGNGq31RNy_0QBLt_EcTvgPyhj7urPdx6oA3M5-r3o,3150
 airbyte_cdk/sources/declarative/datetime/min_max_datetime.py,sha256=0BHBtDNQZfvwM45-tY5pNlTcKAFSGGNxemoi0Jic-0E,5785
-airbyte_cdk/sources/declarative/declarative_component_schema.yaml,sha256=Vsem7b0YL_kaLeTwY_kX-EqHzuBDjik0lBN7e3srXT4,147126
+airbyte_cdk/sources/declarative/declarative_component_schema.yaml,sha256=n8hJVquDj00_VS_I0B2QgwYNcNcfsVZdkajAKArcOHU,147487
 airbyte_cdk/sources/declarative/declarative_source.py,sha256=nF7wBqFd3AQmEKAm4CnIo29CJoQL562cJGSCeL8U8bA,1531
 airbyte_cdk/sources/declarative/declarative_stream.py,sha256=venZjfpvtqr3oFSuvMBWtn4h9ayLhD4L65ACuXCDZ64,10445
 airbyte_cdk/sources/declarative/decoders/__init__.py,sha256=JHb_0d3SE6kNY10mxA5YBEKPeSbsWYjByq1gUQxepoE,953
-airbyte_cdk/sources/declarative/decoders/composite_raw_decoder.py,sha256=DJbWaaJ5LHCBpyWz-4bEw8rqtJYqabEYZtxnfRtWFE0,4946
+airbyte_cdk/sources/declarative/decoders/composite_raw_decoder.py,sha256=vo3l7BfsHaPqK14JfF4ZAodxqyW6PKHPZCmn87sRF3w,5368
 airbyte_cdk/sources/declarative/decoders/decoder.py,sha256=sl-Gt8lXi7yD2Q-sD8je5QS2PbgrgsYjxRLWsay7DMc,826
 airbyte_cdk/sources/declarative/decoders/json_decoder.py,sha256=BdWpXXPhEGf_zknggJmhojLosmxuw51RBVTS0jvdCPc,2080
 airbyte_cdk/sources/declarative/decoders/noop_decoder.py,sha256=iZh0yKY_JzgBnJWiubEusf5c0o6Khd-8EWFWT-8EgFo,542
@@ -107,19 +107,19 @@ airbyte_cdk/sources/declarative/interpolation/interpolated_nested_mapping.py,sha
 airbyte_cdk/sources/declarative/interpolation/interpolated_string.py,sha256=CQkHqGlfa87G6VYMtBAQWin7ECKpfMdrDcg0JO5_rhc,3212
 airbyte_cdk/sources/declarative/interpolation/interpolation.py,sha256=9IoeuWam3L6GyN10L6U8xNWXmkt9cnahSDNkez1OmFY,982
 airbyte_cdk/sources/declarative/interpolation/jinja.py,sha256=UQeuS4Vpyp4hlOn-R3tRyeBX0e9IoV6jQ6gH-Jz8lY0,7182
-airbyte_cdk/sources/declarative/interpolation/macros.py,sha256=uuXBZUWDWM-sPcUKjNSPRN657QhNQCx_hnhTuJj2zOA,5129
+airbyte_cdk/sources/declarative/interpolation/macros.py,sha256=HQKHKnjE17zKoPn27ZpTpugRZZQSaof4GVzUUZaV2eE,5081
 airbyte_cdk/sources/declarative/manifest_declarative_source.py,sha256=TN6GCgLXaWDONTaJwQ3A5ELqC-sxwKz-UYSraJYB-dI,17078
 airbyte_cdk/sources/declarative/migrations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 airbyte_cdk/sources/declarative/migrations/legacy_to_per_partition_state_migration.py,sha256=iemy3fKLczcU0-Aor7tx5jcT6DRedKMqyK7kCOp01hg,3924
 airbyte_cdk/sources/declarative/migrations/state_migration.py,sha256=KWPjealMLKSMtajXgkdGgKg7EmTLR-CqqD7UIh0-eDU,794
 airbyte_cdk/sources/declarative/models/__init__.py,sha256=nUFxNCiKeYRVXuZEKA7GD-lTHxsiKcQ8FitZjKhPIvE,100
-airbyte_cdk/sources/declarative/models/declarative_component_schema.py,sha256=Kd8HvvXqvGWZBey99eQzbK5u2k1ItnRAi2h7C7UNwBQ,103225
+airbyte_cdk/sources/declarative/models/declarative_component_schema.py,sha256=DfbPi512ovaBSWDICJfjIkC3pXDn2aNr1BP-eiLOLyA,103556
 airbyte_cdk/sources/declarative/parsers/__init__.py,sha256=ZnqYNxHsKCgO38IwB34RQyRMXTs4GTvlRi3ImKnIioo,61
 airbyte_cdk/sources/declarative/parsers/custom_code_compiler.py,sha256=jDw_TttD3_hpfevXOH-0Ws0eRuqt6wvED0BqosGPRjI,5938
 airbyte_cdk/sources/declarative/parsers/custom_exceptions.py,sha256=Rir9_z3Kcd5Es0-LChrzk-0qubAsiK_RSEnLmK2OXm8,553
 airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py,sha256=CXwTfD3wSQq3okcqwigpprbHhSURUokh4GK2OmOyKC8,9132
 airbyte_cdk/sources/declarative/parsers/manifest_reference_resolver.py,sha256=IWUOdF03o-aQn0Occo1BJCxU0Pz-QILk5L67nzw2thw,6803
-airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py,sha256=Mx0KJGbqIZeUWduKy-UvpVH-DRm0pzXDcz203r69oNY,140619
+airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py,sha256=Tq-T4iFS8Dh8cGEEN0KrowF_QqPxiXvcs-Djct1qlpA,140636
 airbyte_cdk/sources/declarative/partition_routers/__init__.py,sha256=HJ-Syp3p7RpyR_OK0X_a2kSyISfu3W-PKrRI16iY0a8,957
 airbyte_cdk/sources/declarative/partition_routers/async_job_partition_router.py,sha256=VelO7zKqKtzMJ35jyFeg0ypJLQC0plqqIBNXoBW1G2E,3001
 airbyte_cdk/sources/declarative/partition_routers/cartesian_product_stream_slicer.py,sha256=c5cuVFM6NFkuQqG8Z5IwkBuwDrvXZN1CunUOM_L0ezg,6892
@@ -143,11 +143,11 @@ airbyte_cdk/sources/declarative/requesters/error_handlers/default_http_response_
 airbyte_cdk/sources/declarative/requesters/error_handlers/error_handler.py,sha256=Tan66odx8VHzfdyyXMQkXz2pJYksllGqvxmpoajgcK4,669
 airbyte_cdk/sources/declarative/requesters/error_handlers/http_response_filter.py,sha256=E-fQbt4ShfxZVoqfnmOx69C6FUPWZz8BIqI3DN9Kcjs,7935
 airbyte_cdk/sources/declarative/requesters/http_job_repository.py,sha256=4wpP0ZNTMLugi-Rc1OFdFaxWfRZSl45nzhHqMFCE8SQ,11924
-airbyte_cdk/sources/declarative/requesters/http_requester.py,sha256=Sie8IyntFu66UoJASwpWV0WrRDBr9lpHWSOws7vZfM0,15228
+airbyte_cdk/sources/declarative/requesters/http_requester.py,sha256=KS2nEUKMnyow8HvvfJJbWdPjlglIWSOZIAxrwkmXCI4,16356
 airbyte_cdk/sources/declarative/requesters/paginators/__init__.py,sha256=uArbKs9JKNCt7t9tZoeWwjDpyI1HoPp29FNW0JzvaEM,644
-airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py,sha256=ZW4lwWNAzb4zL0jKc-HjowP5-y0Zg9xi0YlK6tkx_XY,12057
-airbyte_cdk/sources/declarative/requesters/paginators/no_pagination.py,sha256=j6j9QRPaTbKQ2N661RFVKthhkWiodEp6ut0tKeEd0Ng,2019
-airbyte_cdk/sources/declarative/requesters/paginators/paginator.py,sha256=OlN-y0PEOMzlUNUh3pzonoTpIJpGwkP4ibFengvpLVU,2230
+airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py,sha256=SB-Af3CRb4mJwhm4EKNxzl_PK2w5QS4tqrSNNMO2IV4,12760
+airbyte_cdk/sources/declarative/requesters/paginators/no_pagination.py,sha256=b1-zKxYOUMHn7ahdWpzKEzfG4A7s_WQWy-vzRqZWzME,2152
+airbyte_cdk/sources/declarative/requesters/paginators/paginator.py,sha256=TzJF1Q-CFlsHF9lMSfmnGCxRYm9_UQCmBcHYQpc7F30,2376
 airbyte_cdk/sources/declarative/requesters/paginators/strategies/__init__.py,sha256=2gly8fuZpDNwtu1Qg6oE2jBLGqQRdzSLJdnpk_iDV6I,767
 airbyte_cdk/sources/declarative/requesters/paginators/strategies/cursor_pagination_strategy.py,sha256=yLzzK5YIRTkXd2Z-BS__AZXuTd6HXjJIxq05K-lQoxI,3898
 airbyte_cdk/sources/declarative/requesters/paginators/strategies/offset_increment.py,sha256=WvGt_DTFcAgTR-NHrlrR7B71yG-L6jmfW-Gwm9iYzjY,3624
@@ -163,7 +163,7 @@ airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_
 airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_options_provider.py,sha256=vOsdHfWHiTFc89WENHPv1hcxLgdzycMXVT_IEtLuhfs,5012
 airbyte_cdk/sources/declarative/requesters/request_options/request_options_provider.py,sha256=8YRiDzjYvqJ-aMmKFcjqzv_-e8OZ5QG_TbpZ-nuCu6s,2590
 airbyte_cdk/sources/declarative/requesters/request_path.py,sha256=S3MeFvcaQrMbOkSY2W2VbXLNomqt_3eXqVd9ZhgNwUs,299
-airbyte_cdk/sources/declarative/requesters/requester.py,sha256=iVVpXQ4KEd9OyZNwmOofMvx7_06i8ZRxGo3aNTrEQLM,4946
+airbyte_cdk/sources/declarative/requesters/requester.py,sha256=OcDzuCBgD1owK_lBPG0KbRRHRn9kzbuRveU4HejDiv4,5116
 airbyte_cdk/sources/declarative/resolvers/__init__.py,sha256=NiDcz5qi8HPsfX94MUmnX0Rgs_kQXGvucOmJjNWlxKQ,1207
 airbyte_cdk/sources/declarative/resolvers/components_resolver.py,sha256=KPjKc0yb9artL4ZkeqN8RmEykHH6FJgqXD7fCEnh1X0,1936
 airbyte_cdk/sources/declarative/resolvers/config_components_resolver.py,sha256=dz4iJV9liD_LzY_Mn4XmAStoUll60R3MIGWV4aN3pgg,5223
@@ -171,7 +171,7 @@ airbyte_cdk/sources/declarative/resolvers/http_components_resolver.py,sha256=Aio
 airbyte_cdk/sources/declarative/retrievers/__init__.py,sha256=ix9m1dkR69DcXCXUKC5RK_ZZM7ojTLBQ4IkWQTfmfCk,456
 airbyte_cdk/sources/declarative/retrievers/async_retriever.py,sha256=dwYZ70eg9DKHEqZydHhMFPkEILbNcXu7E-djOCikNgI,3530
 airbyte_cdk/sources/declarative/retrievers/retriever.py,sha256=XPLs593Xv8c5cKMc37XzUAYmzlXd1a7eSsspM-CMuWA,1696
-airbyte_cdk/sources/declarative/retrievers/simple_retriever.py,sha256=bOAKQLgMv1Vca-ozMPRVAg1V5nkyUoPwqC02lKpnLiM,24575
+airbyte_cdk/sources/declarative/retrievers/simple_retriever.py,sha256=fDhc6dMx75UImh1_TfLm4Le59tsHpqIUZnau7uIJyYw,25043
 airbyte_cdk/sources/declarative/schema/__init__.py,sha256=xU45UvM5O4c1PSM13UHpCdh5hpW3HXy9vRRGEiAC1rg,795
 airbyte_cdk/sources/declarative/schema/default_schema_loader.py,sha256=KTACrIE23a83wsm3Rd9Eb4K6-20lrGqYxTHNp9yxsso,1820
 airbyte_cdk/sources/declarative/schema/dynamic_schema_loader.py,sha256=J8Q_iJYhcSQLWyt0bTZCbDAGpxt9G8FCc6Q9jtGsNzw,10703
@@ -193,12 +193,7 @@ airbyte_cdk/sources/declarative/transformations/keys_to_snake_transformation.py,
 airbyte_cdk/sources/declarative/transformations/remove_fields.py,sha256=EwUP0SZ2p4GRJ6Q8CUzlz9dcUeEidEFDlI2IBye2tlc,2745
 airbyte_cdk/sources/declarative/transformations/transformation.py,sha256=4sXtx9cNY2EHUPq-xHvDs8GQEBUy3Eo6TkRLKHPXx68,1161
 airbyte_cdk/sources/declarative/types.py,sha256=yqx0xlZv_76tkC7fqJKefmvl4GJJ8mXbeddwVV8XRJU,778
-airbyte_cdk/sources/declarative/yaml_declarative_source.py,sha256=MsKSAqtpwIqJfYOiUX01RbqMeTy7pvBoguvyTWrL7pI,2390
-airbyte_cdk/sources/embedded/__init__.py,sha256=ZnqYNxHsKCgO38IwB34RQyRMXTs4GTvlRi3ImKnIioo,61
-airbyte_cdk/sources/embedded/base_integration.py,sha256=0LbtEtWlnVdkYlweA5OJU4BIoyS6d4le5w9FsLn25Zc,2417
-airbyte_cdk/sources/embedded/catalog.py,sha256=EAnLw9u5fXLNBLfWr_I0itA7OEHMWdqEaM_rWc_tCpI,1653
-airbyte_cdk/sources/embedded/runner.py,sha256=S6cz7SWRTFfT_ZhP-zZYdiT9XRijulV_bMABal2h0OI,1493
-airbyte_cdk/sources/embedded/tools.py,sha256=kp26Au9xH4hxsnDQoavxE_a418ZrRUAMBr2MeOU52VI,746
+airbyte_cdk/sources/declarative/yaml_declarative_source.py,sha256=swZUN7g_AuLeE7n1yp_cquK7DBB082lZUSMIBNEHxgs,2290
 airbyte_cdk/sources/file_based/README.md,sha256=iMqww4VZ882jfNQIdljjDgqreKs-mkdtSrRKA94iX6A,11085
 airbyte_cdk/sources/file_based/__init__.py,sha256=EaxHv_9ot-eRlUCR47ZMZ0IOtB-n0HH24om7Bfn-uuQ,868
 airbyte_cdk/sources/file_based/availability_strategy/__init__.py,sha256=ddKQfUmk-Ls7LJaG8gtrqDybG3d8S7KXOAEjLeYLrTg,399
@@ -293,7 +288,7 @@ airbyte_cdk/sources/streams/http/availability_strategy.py,sha256=sovoGFThZr-doMN
 airbyte_cdk/sources/streams/http/error_handlers/__init__.py,sha256=nNO0bnD0ogDlR4AQrI-YmpcM911vWe83b7RJYgvjSYs,666
 airbyte_cdk/sources/streams/http/error_handlers/backoff_strategy.py,sha256=7fIkF00wD6bqIXtiG2QAgbje_xiQ5JTs99ibwR8LLXY,1030
 airbyte_cdk/sources/streams/http/error_handlers/default_backoff_strategy.py,sha256=1_1Gi2ImwTbh4SgIKCRh2P4kPaVeAeHc1ib6IrBfqKA,411
-airbyte_cdk/sources/streams/http/error_handlers/default_error_mapping.py,sha256=Z0gHhbBSFNWuatzCtBOQt9C-BkiN1G76wr6yl0qodEU,3376
+airbyte_cdk/sources/streams/http/error_handlers/default_error_mapping.py,sha256=drMniPygapMfOigh_ShJQvX6oqJaN4UA3cbRMkOgZJ4,3402
 airbyte_cdk/sources/streams/http/error_handlers/error_handler.py,sha256=GuqP7U1eC9RPaiD4y4Mkf17vKcOXo2ENnMB-CUBLsbo,1164
 airbyte_cdk/sources/streams/http/error_handlers/error_message_parser.py,sha256=xC93uB5BJd3iOnAXCrYLJTitWeGZlqzwe55VtsZqNnE,456
 airbyte_cdk/sources/streams/http/error_handlers/http_status_error_handler.py,sha256=2gqececTxxUqO6aIkVNNXADg48Px5EHUwnXHL9KiPT8,4188
@@ -310,7 +305,7 @@ airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py,sha256=C2j2uVfi9d
 airbyte_cdk/sources/streams/http/requests_native_auth/token.py,sha256=h5PTzcdH-RQLeCg7xZ45w_484OPUDSwNWl_iMJQmZoI,2526
 airbyte_cdk/sources/streams/permissions/identities_stream.py,sha256=9O9k6k18Xm3Zsiw_vnI_jsHXfMCQiek6V-jMkJJLxn8,2621
 airbyte_cdk/sources/streams/utils/__init__.py,sha256=4Hw-PX1-VgESLF16cDdvuYCzGJtHntThLF4qIiULWeo,61
-airbyte_cdk/sources/types.py,sha256=aFPGI4t2K1vHz2oFSUIYUyDN7kw-vcYq4D7aD2zgfAU,5128
+airbyte_cdk/sources/types.py,sha256=x5Rkfpottg46qgr1-g_O4kN6v0Fd0sWHttdYsftyo7w,5148
 airbyte_cdk/sources/utils/__init__.py,sha256=TTN6VUxVy6Is8BhYQZR5pxJGQh8yH4duXh4O1TiMiEY,118
 airbyte_cdk/sources/utils/casing.py,sha256=QC-gV1O4e8DR4-bhdXieUPKm_JamzslVyfABLYYRSXA,256
 airbyte_cdk/sources/utils/record_helper.py,sha256=jeB0mucudzna7Zvj-pCBbwFrbLJ36SlAWZTh5O4Fb9Y,2168
@@ -352,7 +347,7 @@ airbyte_cdk/utils/datetime_format_inferrer.py,sha256=Ne2cpk7Tx3eZDEW2Q3O7jnNOY9g
 airbyte_cdk/utils/datetime_helpers.py,sha256=8mqzZ67Or2PBp7tLtrhh6XFv4wFzYsjCL_DOQJRaftI,17751
 airbyte_cdk/utils/event_timing.py,sha256=aiuFmPU80buLlNdKq4fDTEqqhEIelHPF6AalFGwY8as,2557
 airbyte_cdk/utils/is_cloud_environment.py,sha256=DayV32Irh-SdnJ0MnjvstwCJ66_l5oEsd8l85rZtHoc,574
-airbyte_cdk/utils/mapping_helpers.py,sha256=imUTULHmZ1Ks-MRMRLIVqHCX1eJi_j6tFQrYsKIKtM4,5967
+airbyte_cdk/utils/mapping_helpers.py,sha256=nWjOpnz_5QE9tY9-GtSWMPvYQL95kDD6k8KwwjUmrh0,6526
 airbyte_cdk/utils/message_utils.py,sha256=OTzbkwN7AdMDA3iKYq1LKwfPFxpyEDfdgEF9BED3dkU,1366
 airbyte_cdk/utils/oneof_option_config.py,sha256=N8EmWdYdwt0FM7fuShh6H8nj_r4KEL9tb2DJJtwsPow,1180
 airbyte_cdk/utils/print_buffer.py,sha256=PhMOi0C4Z91kWKrSvCQXcp8qRh1uCimpIdvrg6voZIA,2810
@@ -361,9 +356,9 @@ airbyte_cdk/utils/slice_hasher.py,sha256=EDxgROHDbfG-QKQb59m7h_7crN1tRiawdf5uU7G
 airbyte_cdk/utils/spec_schema_transformations.py,sha256=-5HTuNsnDBAhj-oLeQXwpTGA0HdcjFOf2zTEMUTTg_Y,816
 airbyte_cdk/utils/stream_status_utils.py,sha256=ZmBoiy5HVbUEHAMrUONxZvxnvfV9CesmQJLDTAIWnWw,1171
 airbyte_cdk/utils/traced_exception.py,sha256=C8uIBuCL_E4WnBAOPSxBicD06JAldoN9fGsQDp463OY,6292
-airbyte_cdk-6.38.0.dev0.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
-airbyte_cdk-6.38.0.dev0.dist-info/LICENSE_SHORT,sha256=aqF6D1NcESmpn-cqsxBtszTEnHKnlsp8L4x9wAh3Nxg,55
-airbyte_cdk-6.38.0.dev0.dist-info/METADATA,sha256=1EhdJFpsvmQYoaGXbq_JPTxW9hAy5n81U-fuQsfp24A,6018
-airbyte_cdk-6.38.0.dev0.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-airbyte_cdk-6.38.0.dev0.dist-info/entry_points.txt,sha256=fj-e3PAQvsxsQzyyq8UkG1k8spunWnD4BAH2AwlR6NM,95
-airbyte_cdk-6.38.0.dev0.dist-info/RECORD,,
+airbyte_cdk-6.38.1.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
+airbyte_cdk-6.38.1.dist-info/LICENSE_SHORT,sha256=aqF6D1NcESmpn-cqsxBtszTEnHKnlsp8L4x9wAh3Nxg,55
+airbyte_cdk-6.38.1.dist-info/METADATA,sha256=Dd1RjRaSaSaEJ5_BvSaJ5cZnJ6hRq6btlHXT4P_kOzQ,6013
+airbyte_cdk-6.38.1.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+airbyte_cdk-6.38.1.dist-info/entry_points.txt,sha256=fj-e3PAQvsxsQzyyq8UkG1k8spunWnD4BAH2AwlR6NM,95
+airbyte_cdk-6.38.1.dist-info/RECORD,,

airbyte_cdk/sources/embedded/__init__.py

@@ -1,3 +0,0 @@
-#
-# Copyright (c) 2021 Airbyte, Inc., all rights reserved.
-#

airbyte_cdk/sources/embedded/base_integration.py

@@ -1,61 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-from abc import ABC, abstractmethod
-from typing import Generic, Iterable, Optional, TypeVar
-
-from airbyte_cdk.connector import TConfig
-from airbyte_cdk.models import AirbyteRecordMessage, AirbyteStateMessage, SyncMode, Type
-from airbyte_cdk.sources.embedded.catalog import (
-    create_configured_catalog,
-    get_stream,
-    get_stream_names,
-)
-from airbyte_cdk.sources.embedded.runner import SourceRunner
-from airbyte_cdk.sources.embedded.tools import get_defined_id
-from airbyte_cdk.sources.utils.schema_helpers import check_config_against_spec_or_exit
-
-TOutput = TypeVar("TOutput")
-
-
-class BaseEmbeddedIntegration(ABC, Generic[TConfig, TOutput]):
-    def __init__(self, runner: SourceRunner[TConfig], config: TConfig):
-        check_config_against_spec_or_exit(config, runner.spec())
-
-        self.source = runner
-        self.config = config
-
-        self.last_state: Optional[AirbyteStateMessage] = None
-
-    @abstractmethod
-    def _handle_record(self, record: AirbyteRecordMessage, id: Optional[str]) -> Optional[TOutput]:
-        """
-        Turn an Airbyte record into the appropriate output type for the integration.
-        """
-        pass
-
-    def _load_data(
-        self, stream_name: str, state: Optional[AirbyteStateMessage] = None
-    ) -> Iterable[TOutput]:
-        catalog = self.source.discover(self.config)
-        stream = get_stream(catalog, stream_name)
-        if not stream:
-            raise ValueError(
-                f"Stream {stream_name} not found, the following streams are available: {', '.join(get_stream_names(catalog))}"
-            )
-        if SyncMode.incremental not in stream.supported_sync_modes:
-            configured_catalog = create_configured_catalog(stream, sync_mode=SyncMode.full_refresh)
-        else:
-            configured_catalog = create_configured_catalog(stream, sync_mode=SyncMode.incremental)
-
-        for message in self.source.read(self.config, configured_catalog, state):
-            if message.type == Type.RECORD:
-                output = self._handle_record(
-                    message.record,
-                    get_defined_id(stream, message.record.data),  # type: ignore[union-attr, arg-type]
-                )
-                if output:
-                    yield output
-            elif message.type is Type.STATE and message.state:
-                self.last_state = message.state

airbyte_cdk/sources/embedded/catalog.py

@@ -1,57 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-from typing import List, Optional
-
-from airbyte_cdk.models import (
-    AirbyteCatalog,
-    AirbyteStream,
-    ConfiguredAirbyteCatalog,
-    ConfiguredAirbyteStream,
-    DestinationSyncMode,
-    SyncMode,
-)
-from airbyte_cdk.sources.embedded.tools import get_first
-
-
-def get_stream(catalog: AirbyteCatalog, stream_name: str) -> Optional[AirbyteStream]:
-    return get_first(catalog.streams, lambda s: s.name == stream_name)
-
-
-def get_stream_names(catalog: AirbyteCatalog) -> List[str]:
-    return [stream.name for stream in catalog.streams]
-
-
-def to_configured_stream(
-    stream: AirbyteStream,
-    sync_mode: SyncMode = SyncMode.full_refresh,
-    destination_sync_mode: DestinationSyncMode = DestinationSyncMode.append,
-    cursor_field: Optional[List[str]] = None,
-    primary_key: Optional[List[List[str]]] = None,
-) -> ConfiguredAirbyteStream:
-    return ConfiguredAirbyteStream(
-        stream=stream,
-        sync_mode=sync_mode,
-        destination_sync_mode=destination_sync_mode,
-        cursor_field=cursor_field,
-        primary_key=primary_key,
-    )
-
-
-def to_configured_catalog(
-    configured_streams: List[ConfiguredAirbyteStream],
-) -> ConfiguredAirbyteCatalog:
-    return ConfiguredAirbyteCatalog(streams=configured_streams)
-
-
-def create_configured_catalog(
-    stream: AirbyteStream, sync_mode: SyncMode = SyncMode.full_refresh
-) -> ConfiguredAirbyteCatalog:
-    configured_streams = [
-        to_configured_stream(
-            stream, sync_mode=sync_mode, primary_key=stream.source_defined_primary_key
-        )
-    ]
-
-    return to_configured_catalog(configured_streams)

airbyte_cdk/sources/embedded/runner.py

@@ -1,57 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-
-import logging
-from abc import ABC, abstractmethod
-from typing import Generic, Iterable, Optional
-
-from airbyte_cdk.connector import TConfig
-from airbyte_cdk.models import (
-    AirbyteCatalog,
-    AirbyteMessage,
-    AirbyteStateMessage,
-    ConfiguredAirbyteCatalog,
-    ConnectorSpecification,
-)
-from airbyte_cdk.sources.source import Source
-
-
-class SourceRunner(ABC, Generic[TConfig]):
-    @abstractmethod
-    def spec(self) -> ConnectorSpecification:
-        pass
-
-    @abstractmethod
-    def discover(self, config: TConfig) -> AirbyteCatalog:
-        pass
-
-    @abstractmethod
-    def read(
-        self,
-        config: TConfig,
-        catalog: ConfiguredAirbyteCatalog,
-        state: Optional[AirbyteStateMessage],
-    ) -> Iterable[AirbyteMessage]:
-        pass
-
-
-class CDKRunner(SourceRunner[TConfig]):
-    def __init__(self, source: Source, name: str):
-        self._source = source
-        self._logger = logging.getLogger(name)
-
-    def spec(self) -> ConnectorSpecification:
-        return self._source.spec(self._logger)
-
-    def discover(self, config: TConfig) -> AirbyteCatalog:
-        return self._source.discover(self._logger, config)
-
-    def read(
-        self,
-        config: TConfig,
-        catalog: ConfiguredAirbyteCatalog,
-        state: Optional[AirbyteStateMessage],
-    ) -> Iterable[AirbyteMessage]:
-        return self._source.read(self._logger, config, catalog, state=[state] if state else [])

airbyte_cdk/sources/embedded/tools.py

@@ -1,27 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-from typing import Any, Callable, Dict, Iterable, Optional
-
-import dpath
-
-from airbyte_cdk.models import AirbyteStream
-
-
-def get_first(
-    iterable: Iterable[Any], predicate: Callable[[Any], bool] = lambda m: True
-) -> Optional[Any]:
-    return next(filter(predicate, iterable), None)
-
-
-def get_defined_id(stream: AirbyteStream, data: Dict[str, Any]) -> Optional[str]:
-    if not stream.source_defined_primary_key:
-        return None
-    primary_key = []
-    for key in stream.source_defined_primary_key:
-        try:
-            primary_key.append(str(dpath.get(data, key)))
-        except KeyError:
-            primary_key.append("__not_found__")
-    return "_".join(primary_key)