airbyte-cdk 6.38.0.dev0__tar.gz → 6.38.1__tar.gz

{airbyte_cdk-6.38.0.dev0 → airbyte_cdk-6.38.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 6.38.0.dev0
+Version: 6.38.1
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://airbyte.com
 License: MIT
@@ -28,7 +28,7 @@ Requires-Dist: avro (>=1.11.2,<1.13.0) ; extra == "file-based"
 Requires-Dist: backoff
 Requires-Dist: cachetools
 Requires-Dist: cohere (==4.21) ; extra == "vector-db-based"
-Requires-Dist: cryptography (>=42.0.5,<44.0.0)
+Requires-Dist: cryptography (>=44.0.0,<45.0.0)
 Requires-Dist: dpath (>=2.1.6,<3.0.0)
 Requires-Dist: dunamai (>=1.22.0,<2.0.0)
 Requires-Dist: fastavro (>=1.8.0,<1.9.0) ; extra == "file-based"
@@ -47,7 +47,7 @@ Requires-Dist: pandas (==2.2.2)
 Requires-Dist: pdf2image (==1.16.3) ; extra == "file-based"
 Requires-Dist: pdfminer.six (==20221105) ; extra == "file-based"
 Requires-Dist: psutil (==6.1.0)
-Requires-Dist: pyarrow (>=15.0.0,<15.1.0) ; extra == "file-based"
+Requires-Dist: pyarrow (>=19.0.0,<20.0.0) ; extra == "file-based"
 Requires-Dist: pydantic (>=2.7,<3.0)
 Requires-Dist: pyjwt (>=2.8.0,<3.0.0)
 Requires-Dist: pyrate-limiter (>=3.1.0,<3.2.0)

{airbyte_cdk-6.38.0.dev0 → airbyte_cdk-6.38.1}/airbyte_cdk/entrypoint.py

@@ -22,7 +22,7 @@ from requests import PreparedRequest, Response, Session
 
 from airbyte_cdk.connector import TConfig
 from airbyte_cdk.exception_handler import init_uncaught_exception_handler
-from airbyte_cdk.logger import PRINT_BUFFER, init_logger
+from airbyte_cdk.logger import init_logger
 from airbyte_cdk.models import (
     AirbyteConnectionStatus,
     AirbyteMessage,
@@ -337,11 +337,11 @@ def launch(source: Source, args: List[str]) -> None:
     parsed_args = source_entrypoint.parse_args(args)
     # temporarily removes the PrintBuffer because we're seeing weird print behavior for concurrent syncs
     # Refer to: https://github.com/airbytehq/oncall/issues/6235
-    with PRINT_BUFFER:
-        for message in source_entrypoint.run(parsed_args):
-            # simply printing is creating issues for concurrent CDK as Python uses different two instructions to print: one for the message and
-            # the other for the break line. Adding `\n` to the message ensure that both are printed at the same time
-            print(f"{message}\n", end="")
+    # with PrintBuffer():
+    for message in source_entrypoint.run(parsed_args):
+        # simply printing is creating issues for concurrent CDK as Python uses different two instructions to print: one for the message and
+        # the other for the break line. Adding `\n` to the message ensure that both are printed at the same time
+        print(f"{message}\n", end="", flush=True)
 
 
 def _init_internal_request_filter() -> None:

{airbyte_cdk-6.38.0.dev0 → airbyte_cdk-6.38.1}/airbyte_cdk/logger.py

@@ -16,11 +16,8 @@ from airbyte_cdk.models import (
     Level,
     Type,
 )
-from airbyte_cdk.utils import PrintBuffer
 from airbyte_cdk.utils.airbyte_secrets_utils import filter_secrets
 
-PRINT_BUFFER = PrintBuffer(flush_interval=0.1)
-
 LOGGING_CONFIG = {
     "version": 1,
     "disable_existing_loggers": False,
@@ -30,7 +27,7 @@ LOGGING_CONFIG = {
     "handlers": {
         "console": {
             "class": "logging.StreamHandler",
-            "stream": PRINT_BUFFER,
+            "stream": "ext://sys.stdout",
             "formatter": "airbyte",
         },
     },

{airbyte_cdk-6.38.0.dev0 → airbyte_cdk-6.38.1}/airbyte_cdk/sources/declarative/datetime/datetime_parser.py

@@ -29,6 +29,8 @@ class DatetimeParser:
             return datetime.datetime.fromtimestamp(int(date), tz=datetime.timezone.utc)
         elif format == "%s_as_float":
             return datetime.datetime.fromtimestamp(float(date), tz=datetime.timezone.utc)
+        elif format == "%epoch_microseconds":
+            return self._UNIX_EPOCH + datetime.timedelta(microseconds=int(date))
         elif format == "%ms":
             return self._UNIX_EPOCH + datetime.timedelta(milliseconds=int(date))
         elif "%_ms" in format:
@@ -46,6 +48,8 @@ class DatetimeParser:
             return str(int(dt.timestamp()))
         if format == "%s_as_float":
             return str(float(dt.timestamp()))
+        if format == "%epoch_microseconds":
+            return str(int(dt.timestamp() * 1_000_000))
         if format == "%ms":
             # timstamp() returns a float representing the number of seconds since the unix epoch
             return str(int(dt.timestamp() * 1000))

{airbyte_cdk-6.38.0.dev0 → airbyte_cdk-6.38.1}/airbyte_cdk/sources/declarative/declarative_component_schema.yaml

@@ -1794,7 +1794,6 @@ definitions:
     type: object
     required:
       - type
-      - path
       - url_base
     properties:
       type:
@@ -1806,9 +1805,18 @@ definitions:
         type: string
         interpolation_context:
           - config
+          - next_page_token
+          - stream_interval
+          - stream_partition
+          - stream_slice
+          - creation_response
+          - polling_response
+          - download_target
         examples:
           - "https://connect.squareup.com/v2"
-          - "{{ config['base_url'] or 'https://app.posthog.com'}}/api/"
+          - "{{ config['base_url'] or 'https://app.posthog.com'}}/api"
+          - "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups"
+          - "https://example.com/api/v1/resource/{{ next_page_token['id'] }}"
       path:
         title: URL Path
         description: Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.

{airbyte_cdk-6.38.0.dev0 → airbyte_cdk-6.38.1}/airbyte_cdk/sources/declarative/decoders/composite_raw_decoder.py

@@ -151,6 +151,10 @@ class CompositeRawDecoder(Decoder):
         self, response: requests.Response
     ) -> Generator[MutableMapping[str, Any], None, None]:
         if self.is_stream_response():
+            # urllib mentions that some interfaces don't play nice with auto_close [here](https://urllib3.readthedocs.io/en/stable/user-guide.html#using-io-wrappers-with-response-content)
+            # We have indeed observed some issues with CSV parsing. Hence, we will manage the closing of the file ourselves until we find a better solution.
+            response.raw.auto_close = False
             yield from self.parser.parse(data=response.raw)  # type: ignore[arg-type]
+            response.raw.close()
         else:
             yield from self.parser.parse(data=io.BytesIO(response.content))

{airbyte_cdk-6.38.0.dev0 → airbyte_cdk-6.38.1}/airbyte_cdk/sources/declarative/interpolation/macros.py

@@ -12,6 +12,8 @@ import pytz
 from dateutil import parser
 from isodate import parse_duration
 
+from airbyte_cdk.sources.declarative.datetime.datetime_parser import DatetimeParser
+
 """
 This file contains macros that can be evaluated by a `JinjaInterpolation` object
 """
@@ -171,11 +173,7 @@ def format_datetime(
     dt_datetime = (
         datetime.datetime.strptime(dt, input_format) if input_format else str_to_datetime(dt)
     )
-    if format == "%s":
-        return str(int(dt_datetime.timestamp()))
-    elif format == "%ms":
-        return str(int(dt_datetime.timestamp() * 1_000_000))
-    return dt_datetime.strftime(format)
+    return DatetimeParser().format(dt=dt_datetime, format=format)
 
 
 _macros_list = [

{airbyte_cdk-6.38.0.dev0 → airbyte_cdk-6.38.1}/airbyte_cdk/sources/declarative/models/declarative_component_schema.py

@@ -939,7 +939,7 @@ class MinMaxDatetime(BaseModel):
     )
     datetime_format: Optional[str] = Field(
         "",
-        description='Format of the datetime value. Defaults to "%Y-%m-%dT%H:%M:%S.%f%z" if left empty. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available:\n  * **%s**: Epoch unix timestamp - `1686218963`\n  * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n  * **%ms**: Epoch unix timestamp - `1686218963123`\n  * **%a**: Weekday (abbreviated) - `Sun`\n  * **%A**: Weekday (full) - `Sunday`\n  * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n  * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n  * **%b**: Month (abbreviated) - `Jan`\n  * **%B**: Month (full) - `January`\n  * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n  * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n  * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n  * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n  * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n  * **%p**: AM/PM indicator\n  * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n  * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n  * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n  * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n  * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n  * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n  * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n  * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n  * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n  * **%x**: Date representation - `08/16/1988`\n  * **%X**: Time representation - `21:30:00`\n  * **%%**: Literal \'%\' character\n\n  Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n',
+        description='Format of the datetime value. Defaults to "%Y-%m-%dT%H:%M:%S.%f%z" if left empty. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available:\n  * **%s**: Epoch unix timestamp - `1686218963`\n  * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n  * **%ms**: Epoch unix timestamp - `1686218963123`\n  * **%a**: Weekday (abbreviated) - `Sun`\n  * **%A**: Weekday (full) - `Sunday`\n  * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n  * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n  * **%b**: Month (abbreviated) - `Jan`\n  * **%B**: Month (full) - `January`\n  * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n  * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n  * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n  * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n  * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n  * **%p**: AM/PM indicator\n  * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n  * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n  * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n  * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`, `001`, ..., `999`\n  * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n  * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n  * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n  * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n  * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n  * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n  * **%x**: Date representation - `08/16/1988`\n  * **%X**: Time representation - `21:30:00`\n  * **%%**: Literal \'%\' character\n\n  Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n',
         examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s"],
         title="Datetime Format",
     )
@@ -1545,7 +1545,7 @@ class DatetimeBasedCursor(BaseModel):
     )
     datetime_format: str = Field(
         ...,
-        description="The datetime format used to format the datetime values that are sent in outgoing requests to the API. Use placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n  * **%s**: Epoch unix timestamp - `1686218963`\n  * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n  * **%ms**: Epoch unix timestamp (milliseconds) - `1686218963123`\n  * **%a**: Weekday (abbreviated) - `Sun`\n  * **%A**: Weekday (full) - `Sunday`\n  * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n  * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n  * **%b**: Month (abbreviated) - `Jan`\n  * **%B**: Month (full) - `January`\n  * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n  * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n  * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n  * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n  * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n  * **%p**: AM/PM indicator\n  * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n  * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n  * **%f**: Microsecond (zero-padded to 6 digits) - `000000`\n  * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n  * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n  * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n  * **%U**: Week number of the year (starting Sunday) - `00`, ..., `53`\n  * **%W**: Week number of the year (starting Monday) - `00`, ..., `53`\n  * **%c**: Date and time - `Tue Aug 16 21:30:00 1988`\n  * **%x**: Date standard format - `08/16/1988`\n  * **%X**: Time standard format - `21:30:00`\n  * **%%**: Literal '%' character\n\n  Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n",
+        description="The datetime format used to format the datetime values that are sent in outgoing requests to the API. Use placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n  * **%s**: Epoch unix timestamp - `1686218963`\n  * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n  * **%ms**: Epoch unix timestamp (milliseconds) - `1686218963123`\n  * **%a**: Weekday (abbreviated) - `Sun`\n  * **%A**: Weekday (full) - `Sunday`\n  * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n  * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n  * **%b**: Month (abbreviated) - `Jan`\n  * **%B**: Month (full) - `January`\n  * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n  * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n  * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n  * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n  * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n  * **%p**: AM/PM indicator\n  * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n  * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n  * **%f**: Microsecond (zero-padded to 6 digits) - `000000`\n  * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`\n  * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n  * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n  * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n  * **%U**: Week number of the year (starting Sunday) - `00`, ..., `53`\n  * **%W**: Week number of the year (starting Monday) - `00`, ..., `53`\n  * **%c**: Date and time - `Tue Aug 16 21:30:00 1988`\n  * **%x**: Date standard format - `08/16/1988`\n  * **%X**: Time standard format - `21:30:00`\n  * **%%**: Literal '%' character\n\n  Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n",
         examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s", "%ms", "%s_as_float"],
         title="Outgoing Datetime Format",
     )
@@ -2072,12 +2072,14 @@ class HttpRequester(BaseModel):
         description="Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.",
         examples=[
             "https://connect.squareup.com/v2",
-            "{{ config['base_url'] or 'https://app.posthog.com'}}/api/",
+            "{{ config['base_url'] or 'https://app.posthog.com'}}/api",
+            "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups",
+            "https://example.com/api/v1/resource/{{ next_page_token['id'] }}",
         ],
         title="API Base URL",
     )
-    path: str = Field(
-        ...,
+    path: Optional[str] = Field(
+        None,
         description="Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.",
         examples=[
             "/products",

{airbyte_cdk-6.38.0.dev0 → airbyte_cdk-6.38.1}/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py

@@ -56,7 +56,7 @@ from airbyte_cdk.sources.declarative.auth.token_provider import (
 )
 from airbyte_cdk.sources.declarative.checks import CheckDynamicStream, CheckStream
 from airbyte_cdk.sources.declarative.concurrency_level import ConcurrencyLevel
-from airbyte_cdk.sources.declarative.datetime import MinMaxDatetime
+from airbyte_cdk.sources.declarative.datetime.min_max_datetime import MinMaxDatetime
 from airbyte_cdk.sources.declarative.declarative_stream import DeclarativeStream
 from airbyte_cdk.sources.declarative.decoders import (
     Decoder,

{airbyte_cdk-6.38.0.dev0 → airbyte_cdk-6.38.1}/airbyte_cdk/sources/declarative/requesters/http_requester.py

@@ -25,8 +25,8 @@ from airbyte_cdk.sources.message import MessageRepository, NoopMessageRepository
 from airbyte_cdk.sources.streams.call_rate import APIBudget
 from airbyte_cdk.sources.streams.http import HttpClient
 from airbyte_cdk.sources.streams.http.error_handlers import ErrorHandler
-from airbyte_cdk.sources.types import Config, StreamSlice, StreamState
-from airbyte_cdk.utils.mapping_helpers import combine_mappings
+from airbyte_cdk.sources.types import Config, EmptyString, StreamSlice, StreamState
+from airbyte_cdk.utils.mapping_helpers import combine_mappings, get_interpolation_context
 
 
 @dataclass
@@ -49,9 +49,10 @@ class HttpRequester(Requester):
 
     name: str
     url_base: Union[InterpolatedString, str]
-    path: Union[InterpolatedString, str]
     config: Config
     parameters: InitVar[Mapping[str, Any]]
+
+    path: Optional[Union[InterpolatedString, str]] = None
     authenticator: Optional[DeclarativeAuthenticator] = None
     http_method: Union[str, HttpMethod] = HttpMethod.GET
     request_options_provider: Optional[InterpolatedRequestOptionsProvider] = None
@@ -66,7 +67,9 @@ class HttpRequester(Requester):
 
     def __post_init__(self, parameters: Mapping[str, Any]) -> None:
         self._url_base = InterpolatedString.create(self.url_base, parameters=parameters)
-        self._path = InterpolatedString.create(self.path, parameters=parameters)
+        self._path = InterpolatedString.create(
+            self.path if self.path else EmptyString, parameters=parameters
+        )
         if self.request_options_provider is None:
             self._request_options_provider = InterpolatedRequestOptionsProvider(
                 config=self.config, parameters=parameters
@@ -112,27 +115,33 @@ class HttpRequester(Requester):
     def get_authenticator(self) -> DeclarativeAuthenticator:
         return self._authenticator
 
-    def get_url_base(self) -> str:
-        return os.path.join(self._url_base.eval(self.config), "")
+    def get_url_base(
+        self,
+        *,
+        stream_state: Optional[StreamState] = None,
+        stream_slice: Optional[StreamSlice] = None,
+        next_page_token: Optional[Mapping[str, Any]] = None,
+    ) -> str:
+        interpolation_context = get_interpolation_context(
+            stream_state=stream_state,
+            stream_slice=stream_slice,
+            next_page_token=next_page_token,
+        )
+        return os.path.join(self._url_base.eval(self.config, **interpolation_context), EmptyString)
 
     def get_path(
         self,
         *,
-        stream_state: Optional[StreamState],
-        stream_slice: Optional[StreamSlice],
-        next_page_token: Optional[Mapping[str, Any]],
+        stream_state: Optional[StreamState] = None,
+        stream_slice: Optional[StreamSlice] = None,
+        next_page_token: Optional[Mapping[str, Any]] = None,
     ) -> str:
-        kwargs = {
-            "stream_slice": stream_slice,
-            "next_page_token": next_page_token,
-            # update the interpolation context with extra fields, if passed.
-            **(
-                stream_slice.extra_fields
-                if stream_slice is not None and hasattr(stream_slice, "extra_fields")
-                else {}
-            ),
-        }
-        path = str(self._path.eval(self.config, **kwargs))
+        interpolation_context = get_interpolation_context(
+            stream_state=stream_state,
+            stream_slice=stream_slice,
+            next_page_token=next_page_token,
+        )
+        path = str(self._path.eval(self.config, **interpolation_context))
         return path.lstrip("/")
 
     def get_method(self) -> HttpMethod:
@@ -330,7 +339,20 @@ class HttpRequester(Requester):
 
     @classmethod
     def _join_url(cls, url_base: str, path: str) -> str:
-        return urljoin(url_base, path)
+        """
+        Joins a base URL with a given path and returns the resulting URL with any trailing slash removed.
+
+        This method ensures that there are no duplicate slashes when concatenating the base URL and the path,
+        which is useful when the full URL is provided from an interpolation context.
+
+        Args:
+            url_base (str): The base URL to which the path will be appended.
+            path (str): The path to join with the base URL.
+
+        Returns:
+            str: The concatenated URL with the trailing slash (if any) removed.
+        """
+        return urljoin(url_base, path).rstrip("/")
 
     def send_request(
         self,
@@ -347,7 +369,11 @@ class HttpRequester(Requester):
         request, response = self._http_client.send_request(
             http_method=self.get_method().value,
             url=self._join_url(
-                self.get_url_base(),
+                self.get_url_base(
+                    stream_state=stream_state,
+                    stream_slice=stream_slice,
+                    next_page_token=next_page_token,
+                ),
                 path
                 or self.get_path(
                     stream_state=stream_state,

{airbyte_cdk-6.38.0.dev0 → airbyte_cdk-6.38.1}/airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py

@@ -25,6 +25,7 @@ from airbyte_cdk.sources.declarative.requesters.request_path import RequestPath
 from airbyte_cdk.sources.types import Config, Record, StreamSlice, StreamState
 from airbyte_cdk.utils.mapping_helpers import (
     _validate_component_request_option_paths,
+    get_interpolation_context,
 )
 
 
@@ -150,11 +151,22 @@ class DefaultPaginator(Paginator):
         else:
             return None
 
-    def path(self, next_page_token: Optional[Mapping[str, Any]]) -> Optional[str]:
+    def path(
+        self,
+        next_page_token: Optional[Mapping[str, Any]],
+        stream_state: Optional[Mapping[str, Any]] = None,
+        stream_slice: Optional[StreamSlice] = None,
+    ) -> Optional[str]:
         token = next_page_token.get("next_page_token") if next_page_token else None
         if token and self.page_token_option and isinstance(self.page_token_option, RequestPath):
+            # make additional interpolation context
+            interpolation_context = get_interpolation_context(
+                stream_state=stream_state,
+                stream_slice=stream_slice,
+                next_page_token=next_page_token,
+            )
             # Replace url base to only return the path
-            return str(token).replace(self.url_base.eval(self.config), "")  # type: ignore # url_base is casted to a InterpolatedString in __post_init__
+            return str(token).replace(self.url_base.eval(self.config, **interpolation_context), "")  # type: ignore # url_base is casted to a InterpolatedString in __post_init__
         else:
             return None
 
@@ -258,8 +270,17 @@ class PaginatorTestReadDecorator(Paginator):
             response, last_page_size, last_record, last_page_token_value
         )
 
-    def path(self, next_page_token: Optional[Mapping[str, Any]]) -> Optional[str]:
-        return self._decorated.path(next_page_token)
+    def path(
+        self,
+        next_page_token: Optional[Mapping[str, Any]],
+        stream_state: Optional[Mapping[str, Any]] = None,
+        stream_slice: Optional[StreamSlice] = None,
+    ) -> Optional[str]:
+        return self._decorated.path(
+            next_page_token=next_page_token,
+            stream_state=stream_state,
+            stream_slice=stream_slice,
+        )
 
     def get_request_params(
         self,

{airbyte_cdk-6.38.0.dev0 → airbyte_cdk-6.38.1}/airbyte_cdk/sources/declarative/requesters/paginators/no_pagination.py

@@ -19,7 +19,12 @@ class NoPagination(Paginator):
 
     parameters: InitVar[Mapping[str, Any]]
 
-    def path(self, next_page_token: Optional[Mapping[str, Any]]) -> Optional[str]:
+    def path(
+        self,
+        next_page_token: Optional[Mapping[str, Any]],
+        stream_state: Optional[Mapping[str, Any]] = None,
+        stream_slice: Optional[StreamSlice] = None,
+    ) -> Optional[str]:
         return None
 
     def get_request_params(

{airbyte_cdk-6.38.0.dev0 → airbyte_cdk-6.38.1}/airbyte_cdk/sources/declarative/requesters/paginators/paginator.py

@@ -11,7 +11,7 @@ import requests
 from airbyte_cdk.sources.declarative.requesters.request_options.request_options_provider import (
     RequestOptionsProvider,
 )
-from airbyte_cdk.sources.types import Record
+from airbyte_cdk.sources.types import Record, StreamSlice
 
 
 @dataclass
@@ -49,7 +49,12 @@ class Paginator(ABC, RequestOptionsProvider):
         pass
 
     @abstractmethod
-    def path(self, next_page_token: Optional[Mapping[str, Any]]) -> Optional[str]:
+    def path(
+        self,
+        next_page_token: Optional[Mapping[str, Any]],
+        stream_state: Optional[Mapping[str, Any]] = None,
+        stream_slice: Optional[StreamSlice] = None,
+    ) -> Optional[str]:
         """
         Returns the URL path to hit to fetch the next page of records
 

{airbyte_cdk-6.38.0.dev0 → airbyte_cdk-6.38.1}/airbyte_cdk/sources/declarative/requesters/requester.py

@@ -35,7 +35,13 @@ class Requester(RequestOptionsProvider):
         pass
 
     @abstractmethod
-    def get_url_base(self) -> str:
+    def get_url_base(
+        self,
+        *,
+        stream_state: Optional[StreamState],
+        stream_slice: Optional[StreamSlice],
+        next_page_token: Optional[Mapping[str, Any]],
+    ) -> str:
         """
         :return: URL base for the  API endpoint e.g: if you wanted to hit https://myapi.com/v1/some_entity then this should return "https://myapi.com/v1/"
         """

{airbyte_cdk-6.38.0.dev0 → airbyte_cdk-6.38.1}/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py

@@ -234,13 +234,22 @@ class SimpleRetriever(Retriever):
             raise ValueError("Request body json cannot be a string")
         return body_json
 
-    def _paginator_path(self, next_page_token: Optional[Mapping[str, Any]] = None) -> Optional[str]:
+    def _paginator_path(
+        self,
+        next_page_token: Optional[Mapping[str, Any]] = None,
+        stream_state: Optional[Mapping[str, Any]] = None,
+        stream_slice: Optional[StreamSlice] = None,
+    ) -> Optional[str]:
         """
         If the paginator points to a path, follow it, else return nothing so the requester is used.
         :param next_page_token:
         :return:
         """
-        return self._paginator.path(next_page_token=next_page_token)
+        return self._paginator.path(
+            next_page_token=next_page_token,
+            stream_state=stream_state,
+            stream_slice=stream_slice,
+        )
 
     def _parse_response(
         self,
@@ -299,7 +308,11 @@ class SimpleRetriever(Retriever):
         next_page_token: Optional[Mapping[str, Any]] = None,
     ) -> Optional[requests.Response]:
         return self.requester.send_request(
-            path=self._paginator_path(next_page_token=next_page_token),
+            path=self._paginator_path(
+                next_page_token=next_page_token,
+                stream_state=stream_state,
+                stream_slice=stream_slice,
+            ),
             stream_state=stream_state,
             stream_slice=stream_slice,
             next_page_token=next_page_token,
@@ -570,7 +583,11 @@ class SimpleRetrieverTestReadDecorator(SimpleRetriever):
         next_page_token: Optional[Mapping[str, Any]] = None,
     ) -> Optional[requests.Response]:
         return self.requester.send_request(
-            path=self._paginator_path(next_page_token=next_page_token),
+            path=self._paginator_path(
+                next_page_token=next_page_token,
+                stream_state=stream_state,
+                stream_slice=stream_slice,
+            ),
             stream_state=stream_state,
             stream_slice=stream_slice,
             next_page_token=next_page_token,

{airbyte_cdk-6.38.0.dev0 → airbyte_cdk-6.38.1}/airbyte_cdk/sources/declarative/yaml_declarative_source.py

@@ -50,7 +50,6 @@ class YamlDeclarativeSource(ConcurrentDeclarativeSource[List[AirbyteStateMessage
 
     def _emit_manifest_debug_message(self, extra_args: dict[str, Any]) -> None:
         extra_args["path_to_yaml"] = self._path_to_yaml
-        self.logger.debug("declarative source created from parsed YAML manifest", extra=extra_args)
 
     @staticmethod
     def _parse(connection_definition_str: str) -> ConnectionDefinition:

{airbyte_cdk-6.38.0.dev0 → airbyte_cdk-6.38.1}/airbyte_cdk/sources/streams/http/error_handlers/default_error_mapping.py

@@ -19,9 +19,9 @@ DEFAULT_ERROR_MAPPING: Mapping[Union[int, str, Type[Exception]], ErrorResolution
         error_message="Invalid Protocol Schema: The endpoint that data is being requested from is using an invalid or insecure. Exception: requests.exceptions.InvalidSchema",
     ),
     InvalidURL: ErrorResolution(
-        response_action=ResponseAction.FAIL,
-        failure_type=FailureType.config_error,
-        error_message="Invalid URL specified: The endpoint that data is being requested from is not a valid URL. Exception: requests.exceptions.InvalidURL",
+        response_action=ResponseAction.RETRY,
+        failure_type=FailureType.transient_error,
+        error_message="Invalid URL specified or DNS error occurred: The endpoint that data is being requested from is not a valid URL. Exception: requests.exceptions.InvalidURL",
     ),
     RequestException: ErrorResolution(
         response_action=ResponseAction.RETRY,

airbyte_cdk-6.38.1/airbyte_cdk/sources/streams/utils/__init__.py

@@ -0,0 +1,3 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#

{airbyte_cdk-6.38.0.dev0 → airbyte_cdk-6.38.1}/airbyte_cdk/sources/types.py

@@ -14,6 +14,7 @@ FieldPointer = List[str]
 Config = Mapping[str, Any]
 ConnectionDefinition = Mapping[str, Any]
 StreamState = Mapping[str, Any]
+EmptyString = str()
 
 
 class Record(Mapping[str, Any]):

{airbyte_cdk-6.38.0.dev0 → airbyte_cdk-6.38.1}/airbyte_cdk/utils/mapping_helpers.py

@@ -10,7 +10,7 @@ from airbyte_cdk.sources.declarative.requesters.request_option import (
     RequestOption,
     RequestOptionType,
 )
-from airbyte_cdk.sources.types import Config
+from airbyte_cdk.sources.types import Config, StreamSlice, StreamState
 
 
 def _merge_mappings(
@@ -143,3 +143,20 @@ def _validate_component_request_option_paths(
             )
         except ValueError as error:
             raise ValueError(error)
+
+
+def get_interpolation_context(
+    stream_state: Optional[StreamState] = None,
+    stream_slice: Optional[StreamSlice] = None,
+    next_page_token: Optional[Mapping[str, Any]] = None,
+) -> Mapping[str, Any]:
+    return {
+        "stream_slice": stream_slice,
+        "next_page_token": next_page_token,
+        # update the context with extra fields, if passed.
+        **(
+            stream_slice.extra_fields
+            if stream_slice is not None and hasattr(stream_slice, "extra_fields")
+            else {}
+        ),
+    }

{airbyte_cdk-6.38.0.dev0 → airbyte_cdk-6.38.1}/pyproject.toml

@@ -23,7 +23,7 @@ classifiers = [
 keywords = ["airbyte", "connector-development-kit", "cdk"]
 
 # Python CDK uses dynamic versioning: https://github.com/mtkennerly/poetry-dynamic-versioning
-version = "6.38.0.dev0"  # Version will be calculated dynamically.
+version = "6.38.1"  # Version will be calculated dynamically.
 
 [tool.poetry-dynamic-versioning]
 enable = false
@@ -61,7 +61,7 @@ markdown = { version = "*", optional = true }
 openai = { version = "0.27.9", extras = ["embeddings"], optional = true }
 pdf2image = { version = "1.16.3", optional = true }
 "pdfminer.six" = { version = "20221105", optional = true }
-pyarrow = { version = "~15.0.0", optional = true }
+pyarrow = { version = "^19.0.0", optional = true }
 pytesseract = { version = "0.3.10", optional = true }
 python-calamine = { version = "0.2.3", optional = true }
 python-snappy = { version = "0.7.3", optional = true }
@@ -73,7 +73,7 @@ numpy = "<2"
 unstructured = { version = "0.10.27", extras = ["docx", "pptx"], optional = true }
 "unstructured.pytesseract" = { version = ">=0.3.12", optional = true }
 pyjwt = "^2.8.0"
-cryptography = ">=42.0.5,<44.0.0"
+cryptography = ">=44.0.0,<45.0.0"
 pytz = "2024.2"
 orjson = "^3.10.7"
 serpyco-rs = "^1.10.2"

airbyte_cdk-6.38.0.dev0/airbyte_cdk/sources/declarative/datetime/__init__.py

@@ -1,7 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-from airbyte_cdk.sources.declarative.datetime.min_max_datetime import MinMaxDatetime
-
-__all__ = ["MinMaxDatetime"]

airbyte_cdk-6.38.0.dev0/airbyte_cdk/sources/embedded/__init__.py

@@ -1,3 +0,0 @@
-#
-# Copyright (c) 2021 Airbyte, Inc., all rights reserved.
-#