airbyte-cdk 6.48.10.post2.dev14974028993__py3-none-any.whl → 6.48.12__py3-none-any.whl

airbyte_cdk/sources/declarative/incremental/concurrent_partition_cursor.py

@@ -229,10 +229,10 @@ class ConcurrentPerPartitionCursor(Cursor):
 
     def _throttle_state_message(self) -> Optional[float]:
         """
-        Throttles the state message emission to once every 600 seconds.
+        Throttles the state message emission to once every 60 seconds.
         """
         current_time = time.time()
-        if current_time - self._last_emission_time <= 600:
+        if current_time - self._last_emission_time <= 60:
             return None
         return current_time
 

airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py

@@ -26,6 +26,7 @@ from typing import (
 
 from isodate import parse_duration
 from pydantic.v1 import BaseModel
+from requests import Response
 
 from airbyte_cdk.connector_builder.models import (
     LogMessage as ConnectorBuilderLogMessage,
@@ -529,6 +530,7 @@ from airbyte_cdk.sources.declarative.transformations.keys_to_lower_transformatio
 from airbyte_cdk.sources.declarative.transformations.keys_to_snake_transformation import (
     KeysToSnakeCaseTransformation,
 )
+from airbyte_cdk.sources.http_logger import format_http_message
 from airbyte_cdk.sources.message import (
     InMemoryMessageRepository,
     LogAppenderMessageRepositoryDecorator,
@@ -2390,15 +2392,24 @@ class ModelToComponentFactory:
                 schema_transformations.append(
                     self._create_component_from_model(model=transformation_model, config=config)
                 )
-
+        name = "dynamic_properties"
         retriever = self._create_component_from_model(
             model=model.retriever,
             config=config,
-            name="dynamic_properties",
+            name=name,
             primary_key=None,
             stream_slicer=combined_slicers,
             transformations=[],
             use_cache=True,
+            log_formatter=(
+                lambda response: format_http_message(
+                    response,
+                    f"Schema loader '{name}' request",
+                    f"Request performed in order to extract schema.",
+                    name,
+                    is_auxiliary=True,
+                )
+            ),
         )
         schema_type_identifier = self._create_component_from_model(
             model.schema_type_identifier, config=config, parameters=model.parameters or {}
@@ -2985,6 +2996,7 @@ class ModelToComponentFactory:
             ]
         ] = None,
         use_cache: Optional[bool] = None,
+        log_formatter: Optional[Callable[[Response], Any]] = None,
         **kwargs: Any,
     ) -> SimpleRetriever:
         def _get_url() -> str:
@@ -3161,6 +3173,7 @@ class ModelToComponentFactory:
                 config=config,
                 maximum_number_of_slices=self._limit_slices_fetched or 5,
                 ignore_stream_slicer_parameters_on_paginated_requests=ignore_stream_slicer_parameters_on_paginated_requests,
+                log_formatter=log_formatter,
                 parameters=model.parameters or {},
             )
         return SimpleRetriever(

airbyte_cdk/sources/declarative/requesters/http_job_repository.py

@@ -320,14 +320,14 @@ class AsyncHttpJobRepository(AsyncJobRepository):
         return polling_response_context
 
     def _get_create_job_stream_slice(self, job: AsyncJob) -> StreamSlice:
-        stream_slice = StreamSlice(
-            partition={},
-            cursor_slice={},
-            extra_fields={
+        return StreamSlice(
+            partition=job.job_parameters().partition,
+            cursor_slice=job.job_parameters().cursor_slice,
+            extra_fields=dict(job.job_parameters().extra_fields)
+            | {
                 "creation_response": self._get_creation_response_interpolation_context(job),
             },
         )
-        return stream_slice
 
     def _get_download_targets(self, job: AsyncJob) -> Iterable[str]:
         if not self.download_target_requester:

airbyte_cdk/sources/declarative/requesters/request_options/interpolated_nested_request_input_provider.py

@@ -11,6 +11,7 @@ from airbyte_cdk.sources.declarative.interpolation.interpolated_nested_mapping i
 )
 from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
 from airbyte_cdk.sources.types import Config, StreamSlice
+from airbyte_cdk.utils.mapping_helpers import get_interpolation_context
 
 
 @dataclass
@@ -52,8 +53,8 @@ class InterpolatedNestedRequestInputProvider:
         :param next_page_token: The pagination token
         :return: The request inputs to set on an outgoing HTTP request
         """
-        kwargs = {
-            "stream_slice": stream_slice,
-            "next_page_token": next_page_token,
-        }
+        kwargs = get_interpolation_context(
+            stream_slice=stream_slice,
+            next_page_token=next_page_token,
+        )
         return self._interpolator.eval(self.config, **kwargs)  # type: ignore  # self._interpolator is always initialized with a value and will not be None

airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_input_provider.py

@@ -8,6 +8,7 @@ from typing import Any, Mapping, Optional, Tuple, Type, Union
 from airbyte_cdk.sources.declarative.interpolation.interpolated_mapping import InterpolatedMapping
 from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
 from airbyte_cdk.sources.types import Config, StreamSlice, StreamState
+from airbyte_cdk.utils.mapping_helpers import get_interpolation_context
 
 
 @dataclass
@@ -51,10 +52,10 @@ class InterpolatedRequestInputProvider:
         :param valid_value_types: A tuple of types that the interpolator should allow
         :return: The request inputs to set on an outgoing HTTP request
         """
-        kwargs = {
-            "stream_slice": stream_slice,
-            "next_page_token": next_page_token,
-        }
+        kwargs = get_interpolation_context(
+            stream_slice=stream_slice,
+            next_page_token=next_page_token,
+        )
         interpolated_value = self._interpolator.eval(  # type: ignore # self._interpolator is always initialized with a value and will not be None
             self.config,
             valid_key_types=valid_key_types,

airbyte_cdk/sources/declarative/retrievers/simple_retriever.py

@@ -10,7 +10,6 @@ from itertools import islice
 from typing import (
     Any,
     Callable,
-    Dict,
     Iterable,
     List,
     Mapping,
@@ -93,6 +92,7 @@ class SimpleRetriever(Retriever):
     cursor: Optional[DeclarativeCursor] = None
     ignore_stream_slicer_parameters_on_paginated_requests: bool = False
     additional_query_properties: Optional[QueryProperties] = None
+    log_formatter: Optional[Callable[[requests.Response], Any]] = None
 
     def __post_init__(self, parameters: Mapping[str, Any]) -> None:
         self._paginator = self.paginator or NoPagination(parameters=parameters)
@@ -353,6 +353,7 @@ class SimpleRetriever(Retriever):
                 stream_slice=stream_slice,
                 next_page_token=next_page_token,
             ),
+            log_formatter=self.log_formatter,
         )
 
     # This logic is similar to _read_pages in the HttpStream class. When making changes here, consider making changes there as well.
@@ -655,6 +656,19 @@ class SimpleRetrieverTestReadDecorator(SimpleRetriever):
 
     def __post_init__(self, options: Mapping[str, Any]) -> None:
         super().__post_init__(options)
+        self.log_formatter = (
+            (
+                lambda response: format_http_message(
+                    response,
+                    f"Stream '{self.name}' request",
+                    f"Request performed in order to extract records for stream '{self.name}'",
+                    self.name,
+                )
+            )
+            if not self.log_formatter
+            else self.log_formatter
+        )
+
         if self.maximum_number_of_slices and self.maximum_number_of_slices < 1:
             raise ValueError(
                 f"The maximum number of slices on a test read needs to be strictly positive. Got {self.maximum_number_of_slices}"
@@ -664,49 +678,6 @@ class SimpleRetrieverTestReadDecorator(SimpleRetriever):
     def stream_slices(self) -> Iterable[Optional[StreamSlice]]:  # type: ignore
         return islice(super().stream_slices(), self.maximum_number_of_slices)
 
-    def _fetch_next_page(
-        self,
-        stream_state: Mapping[str, Any],
-        stream_slice: StreamSlice,
-        next_page_token: Optional[Mapping[str, Any]] = None,
-    ) -> Optional[requests.Response]:
-        return self.requester.send_request(
-            path=self._paginator_path(
-                next_page_token=next_page_token,
-                stream_state=stream_state,
-                stream_slice=stream_slice,
-            ),
-            stream_state=stream_state,
-            stream_slice=stream_slice,
-            next_page_token=next_page_token,
-            request_headers=self._request_headers(
-                stream_state=stream_state,
-                stream_slice=stream_slice,
-                next_page_token=next_page_token,
-            ),
-            request_params=self._request_params(
-                stream_state=stream_state,
-                stream_slice=stream_slice,
-                next_page_token=next_page_token,
-            ),
-            request_body_data=self._request_body_data(
-                stream_state=stream_state,
-                stream_slice=stream_slice,
-                next_page_token=next_page_token,
-            ),
-            request_body_json=self._request_body_json(
-                stream_state=stream_state,
-                stream_slice=stream_slice,
-                next_page_token=next_page_token,
-            ),
-            log_formatter=lambda response: format_http_message(
-                response,
-                f"Stream '{self.name}' request",
-                f"Request performed in order to extract records for stream '{self.name}'",
-                self.name,
-            ),
-        )
-
 
 @deprecated(
     "This class is experimental. Use at your own risk.",

{airbyte_cdk-6.48.10.post2.dev14974028993.dist-info → airbyte_cdk-6.48.12.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 6.48.10.post2.dev14974028993
+Version: 6.48.12
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://airbyte.com
 License: MIT

{airbyte_cdk-6.48.10.post2.dev14974028993.dist-info → airbyte_cdk-6.48.12.dist-info}/RECORD

@@ -111,7 +111,7 @@ airbyte_cdk/sources/declarative/extractors/record_selector.py,sha256=vCpwX1PVRFP
 airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py,sha256=WJyA2OYIEgFpVP5Y3o0tIj69AV6IKkn9B16MeXaEItI,6513
 airbyte_cdk/sources/declarative/extractors/type_transformer.py,sha256=d6Y2Rfg8pMVEEnHllfVksWZdNVOU55yk34O03dP9muY,1626
 airbyte_cdk/sources/declarative/incremental/__init__.py,sha256=U1oZKtBaEC6IACmvziY9Wzg7Z8EgF4ZuR7NwvjlB_Sk,1255
-airbyte_cdk/sources/declarative/incremental/concurrent_partition_cursor.py,sha256=8zxfM1aXSwdfTbpL9PHwgg43ZRbNUQRcX038D7YTI_0,22768
+airbyte_cdk/sources/declarative/incremental/concurrent_partition_cursor.py,sha256=NLxDA4WOkvRhF3RwpmJ31Y0IX0b1sqGm8m7KU6qSJeo,22766
 airbyte_cdk/sources/declarative/incremental/datetime_based_cursor.py,sha256=Rbe6lJLTtZ5en33MwZiB9-H9-AwDMNHgwBZs8EqhYqk,22172
 airbyte_cdk/sources/declarative/incremental/declarative_cursor.py,sha256=5Bhw9VRPyIuCaD0wmmq_L3DZsa-rJgtKSEUzSd8YYD0,536
 airbyte_cdk/sources/declarative/incremental/global_substream_cursor.py,sha256=2tsE6FgXzemf4fZZ4uGtd8QpRBl9GJ2CRqSNJE5p0EI,16077
@@ -140,7 +140,7 @@ airbyte_cdk/sources/declarative/parsers/custom_exceptions.py,sha256=wnRUP0Xeru9R
 airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py,sha256=RUyFZS0zslLb7UfQrvqMC--k5CVLNSp7zHw6kbosvKE,9688
 airbyte_cdk/sources/declarative/parsers/manifest_normalizer.py,sha256=laBy7ebjA-PiNwc-50U4FHvMqS_mmHvnabxgFs4CjGw,17069
 airbyte_cdk/sources/declarative/parsers/manifest_reference_resolver.py,sha256=pJmg78vqE5VfUrF_KJnWjucQ4k9IWFULeAxHCowrHXE,6806
-airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py,sha256=Ma5Pfv1owMEbh58UA2GREmxp5e48zcU9ig_7K6Iw8uE,167111
+airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py,sha256=Ai-3RfSoBa3oNc34KNlUvp7KbbQuuqhtGwGOxPf8fMc,167671
 airbyte_cdk/sources/declarative/partition_routers/__init__.py,sha256=TBC9AkGaUqHm2IKHMPN6punBIcY5tWGULowcLoAVkfw,1109
 airbyte_cdk/sources/declarative/partition_routers/async_job_partition_router.py,sha256=VelO7zKqKtzMJ35jyFeg0ypJLQC0plqqIBNXoBW1G2E,3001
 airbyte_cdk/sources/declarative/partition_routers/cartesian_product_stream_slicer.py,sha256=c5cuVFM6NFkuQqG8Z5IwkBuwDrvXZN1CunUOM_L0ezg,6892
@@ -164,7 +164,7 @@ airbyte_cdk/sources/declarative/requesters/error_handlers/default_error_handler.
 airbyte_cdk/sources/declarative/requesters/error_handlers/default_http_response_filter.py,sha256=q0YkeYUUWO6iErUy0vjqiOkhg8_9d5YcCmtlpXAJJ9E,1314
 airbyte_cdk/sources/declarative/requesters/error_handlers/error_handler.py,sha256=Tan66odx8VHzfdyyXMQkXz2pJYksllGqvxmpoajgcK4,669
 airbyte_cdk/sources/declarative/requesters/error_handlers/http_response_filter.py,sha256=E-fQbt4ShfxZVoqfnmOx69C6FUPWZz8BIqI3DN9Kcjs,7935
-airbyte_cdk/sources/declarative/requesters/http_job_repository.py,sha256=uDyLvNsJ183oh3TT-O1PDOgpGt7OD1uqpLTDWTyb9PA,14271
+airbyte_cdk/sources/declarative/requesters/http_job_repository.py,sha256=pVzIDdfGs1eAZo9F6zeFYKlEmEqanhNvZLKFCHkdmNo,14348
 airbyte_cdk/sources/declarative/requesters/http_requester.py,sha256=1qUqNxJ6I_4uSkW4KYXEtygVioURIEmiaDU8GMl_Jcs,18833
 airbyte_cdk/sources/declarative/requesters/paginators/__init__.py,sha256=uArbKs9JKNCt7t9tZoeWwjDpyI1HoPp29FNW0JzvaEM,644
 airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py,sha256=SB-Af3CRb4mJwhm4EKNxzl_PK2w5QS4tqrSNNMO2IV4,12760
@@ -187,8 +187,8 @@ airbyte_cdk/sources/declarative/requesters/request_option.py,sha256=Bl0gxGWudmwT
 airbyte_cdk/sources/declarative/requesters/request_options/__init__.py,sha256=WCwpKqM4wKqy-DHJaCHbKAlFqRVOqMi9K5qonxIfi_Y,809
 airbyte_cdk/sources/declarative/requesters/request_options/datetime_based_request_options_provider.py,sha256=31nG6_0igidJFQon37-WeQkTpG3g2A5ZmlluI3ilZdE,3632
 airbyte_cdk/sources/declarative/requesters/request_options/default_request_options_provider.py,sha256=SRROdPJZ5kuqHLOlkh115pWP9nDGfDxRYPgH9oD3hPo,1798
-airbyte_cdk/sources/declarative/requesters/request_options/interpolated_nested_request_input_provider.py,sha256=86YozYuBDfu0t9NbevIvQoGU0vqTP4rt3dRSTsHz3PA,2269
-airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_input_provider.py,sha256=rR00kE64U2yL0McU1gPr4_W5_sLUqwDgL3Nvj691nRU,2884
+airbyte_cdk/sources/declarative/requesters/request_options/interpolated_nested_request_input_provider.py,sha256=STVopTmEZaxmE-w_2zL6CYXFxq-2K2wDKGtWj9OYZyU,2360
+airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_input_provider.py,sha256=-X_sncjnw3N5ZyhlLMOI6vnWy2CM8drqPFOKgzDXTQg,2975
 airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_options_provider.py,sha256=ks4rCSdEKTX2bm8obDf-PzWggel1rQEwtu35-rQvqVk,8274
 airbyte_cdk/sources/declarative/requesters/request_options/request_options_provider.py,sha256=8YRiDzjYvqJ-aMmKFcjqzv_-e8OZ5QG_TbpZ-nuCu6s,2590
 airbyte_cdk/sources/declarative/requesters/request_path.py,sha256=S3MeFvcaQrMbOkSY2W2VbXLNomqt_3eXqVd9ZhgNwUs,299
@@ -207,7 +207,7 @@ airbyte_cdk/sources/declarative/retrievers/file_uploader/file_writer.py,sha256=V
 airbyte_cdk/sources/declarative/retrievers/file_uploader/local_file_system_file_writer.py,sha256=jLpdonre1UHfbjGSD5AK_T0codLABJByTvbqepDZtEQ,422
 airbyte_cdk/sources/declarative/retrievers/file_uploader/noop_file_writer.py,sha256=1yfimzxm09d2j605cu_HhiYVDNVL1rUMi3vs_jYlIyY,330
 airbyte_cdk/sources/declarative/retrievers/retriever.py,sha256=XPLs593Xv8c5cKMc37XzUAYmzlXd1a7eSsspM-CMuWA,1696
-airbyte_cdk/sources/declarative/retrievers/simple_retriever.py,sha256=O7qpM71L1_ATIbEKa8y658jdiSJSPw0KmuGKgnaruQU,31008
+airbyte_cdk/sources/declarative/retrievers/simple_retriever.py,sha256=Ui7V_etQ-sI4GCFJqgt1bn9c5BA21dyzCJwqnKZMYTc,29876
 airbyte_cdk/sources/declarative/schema/__init__.py,sha256=xU45UvM5O4c1PSM13UHpCdh5hpW3HXy9vRRGEiAC1rg,795
 airbyte_cdk/sources/declarative/schema/composite_schema_loader.py,sha256=ymGbvxS_QyGc4nnjEyRo5ch8bVedELO41PAUxKXZyMw,1113
 airbyte_cdk/sources/declarative/schema/default_schema_loader.py,sha256=UnbzlExmwoQiVV8zDg4lhAEaqA_0pRfwbMRe8yqOuWk,1834
@@ -408,9 +408,9 @@ airbyte_cdk/utils/slice_hasher.py,sha256=EDxgROHDbfG-QKQb59m7h_7crN1tRiawdf5uU7G
 airbyte_cdk/utils/spec_schema_transformations.py,sha256=-5HTuNsnDBAhj-oLeQXwpTGA0HdcjFOf2zTEMUTTg_Y,816
 airbyte_cdk/utils/stream_status_utils.py,sha256=ZmBoiy5HVbUEHAMrUONxZvxnvfV9CesmQJLDTAIWnWw,1171
 airbyte_cdk/utils/traced_exception.py,sha256=C8uIBuCL_E4WnBAOPSxBicD06JAldoN9fGsQDp463OY,6292
-airbyte_cdk-6.48.10.post2.dev14974028993.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
-airbyte_cdk-6.48.10.post2.dev14974028993.dist-info/LICENSE_SHORT,sha256=aqF6D1NcESmpn-cqsxBtszTEnHKnlsp8L4x9wAh3Nxg,55
-airbyte_cdk-6.48.10.post2.dev14974028993.dist-info/METADATA,sha256=v8rf4G01LmISVwl0oxwetYSzQUbV5vjsuXZEa1587W8,6365
-airbyte_cdk-6.48.10.post2.dev14974028993.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-airbyte_cdk-6.48.10.post2.dev14974028993.dist-info/entry_points.txt,sha256=AKWbEkHfpzzk9nF9tqBUaw1MbvTM4mGtEzmZQm0ZWvM,139
-airbyte_cdk-6.48.10.post2.dev14974028993.dist-info/RECORD,,
+airbyte_cdk-6.48.12.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
+airbyte_cdk-6.48.12.dist-info/LICENSE_SHORT,sha256=aqF6D1NcESmpn-cqsxBtszTEnHKnlsp8L4x9wAh3Nxg,55
+airbyte_cdk-6.48.12.dist-info/METADATA,sha256=jqI_OETxlvr-K862CNigYQtRBNP7grACKsJZXkxFnH0,6344
+airbyte_cdk-6.48.12.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+airbyte_cdk-6.48.12.dist-info/entry_points.txt,sha256=AKWbEkHfpzzk9nF9tqBUaw1MbvTM4mGtEzmZQm0ZWvM,139
+airbyte_cdk-6.48.12.dist-info/RECORD,,