airbyte-cdk 6.37.0.dev1__py3-none-any.whl → 6.37.2.dev1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airbyte_cdk/connector_builder/models.py +16 -14
- airbyte_cdk/connector_builder/test_reader/helpers.py +120 -22
- airbyte_cdk/connector_builder/test_reader/message_grouper.py +16 -3
- airbyte_cdk/connector_builder/test_reader/types.py +9 -1
- airbyte_cdk/sources/declarative/auth/token_provider.py +1 -0
- airbyte_cdk/sources/declarative/concurrent_declarative_source.py +15 -0
- airbyte_cdk/sources/declarative/declarative_component_schema.yaml +5 -43
- airbyte_cdk/sources/declarative/decoders/composite_raw_decoder.py +16 -4
- airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py +1 -0
- airbyte_cdk/sources/declarative/incremental/concurrent_partition_cursor.py +83 -17
- airbyte_cdk/sources/declarative/models/declarative_component_schema.py +3 -42
- airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +52 -63
- airbyte_cdk/sources/declarative/partition_routers/__init__.py +0 -4
- airbyte_cdk/sources/declarative/requesters/http_job_repository.py +42 -4
- airbyte_cdk/sources/declarative/retrievers/async_retriever.py +10 -3
- airbyte_cdk/sources/http_logger.py +3 -0
- airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py +1 -0
- {airbyte_cdk-6.37.0.dev1.dist-info → airbyte_cdk-6.37.2.dev1.dist-info}/METADATA +1 -1
- {airbyte_cdk-6.37.0.dev1.dist-info → airbyte_cdk-6.37.2.dev1.dist-info}/RECORD +23 -24
- airbyte_cdk/sources/declarative/partition_routers/grouping_partition_router.py +0 -136
- {airbyte_cdk-6.37.0.dev1.dist-info → airbyte_cdk-6.37.2.dev1.dist-info}/LICENSE.txt +0 -0
- {airbyte_cdk-6.37.0.dev1.dist-info → airbyte_cdk-6.37.2.dev1.dist-info}/LICENSE_SHORT +0 -0
- {airbyte_cdk-6.37.0.dev1.dist-info → airbyte_cdk-6.37.2.dev1.dist-info}/WHEEL +0 -0
- {airbyte_cdk-6.37.0.dev1.dist-info → airbyte_cdk-6.37.2.dev1.dist-info}/entry_points.txt +0 -0
@@ -646,7 +646,7 @@ class Rate(BaseModel):
|
|
646
646
|
class Config:
|
647
647
|
extra = Extra.allow
|
648
648
|
|
649
|
-
limit: int = Field(
|
649
|
+
limit: Union[int, str] = Field(
|
650
650
|
...,
|
651
651
|
description="The maximum number of calls allowed within the interval.",
|
652
652
|
title="Limit",
|
@@ -2225,15 +2225,7 @@ class SimpleRetriever(BaseModel):
|
|
2225
2225
|
CustomPartitionRouter,
|
2226
2226
|
ListPartitionRouter,
|
2227
2227
|
SubstreamPartitionRouter,
|
2228
|
-
|
2229
|
-
List[
|
2230
|
-
Union[
|
2231
|
-
CustomPartitionRouter,
|
2232
|
-
ListPartitionRouter,
|
2233
|
-
SubstreamPartitionRouter,
|
2234
|
-
GroupingPartitionRouter,
|
2235
|
-
]
|
2236
|
-
],
|
2228
|
+
List[Union[CustomPartitionRouter, ListPartitionRouter, SubstreamPartitionRouter]],
|
2237
2229
|
]
|
2238
2230
|
] = Field(
|
2239
2231
|
[],
|
@@ -2311,15 +2303,7 @@ class AsyncRetriever(BaseModel):
|
|
2311
2303
|
CustomPartitionRouter,
|
2312
2304
|
ListPartitionRouter,
|
2313
2305
|
SubstreamPartitionRouter,
|
2314
|
-
|
2315
|
-
List[
|
2316
|
-
Union[
|
2317
|
-
CustomPartitionRouter,
|
2318
|
-
ListPartitionRouter,
|
2319
|
-
SubstreamPartitionRouter,
|
2320
|
-
GroupingPartitionRouter,
|
2321
|
-
]
|
2322
|
-
],
|
2306
|
+
List[Union[CustomPartitionRouter, ListPartitionRouter, SubstreamPartitionRouter]],
|
2323
2307
|
]
|
2324
2308
|
] = Field(
|
2325
2309
|
[],
|
@@ -2371,29 +2355,6 @@ class SubstreamPartitionRouter(BaseModel):
|
|
2371
2355
|
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
2372
2356
|
|
2373
2357
|
|
2374
|
-
class GroupingPartitionRouter(BaseModel):
|
2375
|
-
type: Literal["GroupingPartitionRouter"]
|
2376
|
-
group_size: int = Field(
|
2377
|
-
...,
|
2378
|
-
description="The number of partitions to include in each group. This determines how many partition values are batched together in a single slice.",
|
2379
|
-
examples=[10, 50],
|
2380
|
-
title="Group Size",
|
2381
|
-
)
|
2382
|
-
underlying_partition_router: Union[
|
2383
|
-
CustomPartitionRouter, ListPartitionRouter, SubstreamPartitionRouter
|
2384
|
-
] = Field(
|
2385
|
-
...,
|
2386
|
-
description="The partition router whose output will be grouped. This can be any valid partition router component.",
|
2387
|
-
title="Underlying Partition Router",
|
2388
|
-
)
|
2389
|
-
deduplicate: Optional[bool] = Field(
|
2390
|
-
True,
|
2391
|
-
description="If true, ensures that partitions are unique within each group by removing duplicates based on the partition key.",
|
2392
|
-
title="Deduplicate Partitions",
|
2393
|
-
)
|
2394
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
2395
|
-
|
2396
|
-
|
2397
2358
|
class HttpComponentsResolver(BaseModel):
|
2398
2359
|
type: Literal["HttpComponentsResolver"]
|
2399
2360
|
retriever: Union[AsyncRetriever, CustomRetriever, SimpleRetriever] = Field(
|
@@ -227,9 +227,6 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import
|
|
227
227
|
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
228
228
|
FlattenFields as FlattenFieldsModel,
|
229
229
|
)
|
230
|
-
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
231
|
-
GroupingPartitionRouter as GroupingPartitionRouterModel,
|
232
|
-
)
|
233
230
|
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
234
231
|
GzipDecoder as GzipDecoderModel,
|
235
232
|
)
|
@@ -382,7 +379,6 @@ from airbyte_cdk.sources.declarative.parsers.custom_code_compiler import (
|
|
382
379
|
)
|
383
380
|
from airbyte_cdk.sources.declarative.partition_routers import (
|
384
381
|
CartesianProductStreamSlicer,
|
385
|
-
GroupingPartitionRouter,
|
386
382
|
ListPartitionRouter,
|
387
383
|
PartitionRouter,
|
388
384
|
SinglePartitionRouter,
|
@@ -628,7 +624,6 @@ class ModelToComponentFactory:
|
|
628
624
|
UnlimitedCallRatePolicyModel: self.create_unlimited_call_rate_policy,
|
629
625
|
RateModel: self.create_rate,
|
630
626
|
HttpRequestRegexMatcherModel: self.create_http_request_matcher,
|
631
|
-
GroupingPartitionRouterModel: self.create_grouping_partition_router,
|
632
627
|
}
|
633
628
|
|
634
629
|
# Needed for the case where we need to perform a second parse on the fields of a custom component
|
@@ -2096,10 +2091,10 @@ class ModelToComponentFactory:
|
|
2096
2091
|
def create_json_decoder(model: JsonDecoderModel, config: Config, **kwargs: Any) -> Decoder:
|
2097
2092
|
return JsonDecoder(parameters={})
|
2098
2093
|
|
2099
|
-
|
2100
|
-
def create_csv_decoder(model: CsvDecoderModel, config: Config, **kwargs: Any) -> Decoder:
|
2094
|
+
def create_csv_decoder(self, model: CsvDecoderModel, config: Config, **kwargs: Any) -> Decoder:
|
2101
2095
|
return CompositeRawDecoder(
|
2102
|
-
parser=ModelToComponentFactory._get_parser(model, config),
|
2096
|
+
parser=ModelToComponentFactory._get_parser(model, config),
|
2097
|
+
stream_response=False if self._emit_connector_builder_messages else True,
|
2103
2098
|
)
|
2104
2099
|
|
2105
2100
|
@staticmethod
|
@@ -2108,10 +2103,12 @@ class ModelToComponentFactory:
|
|
2108
2103
|
parser=ModelToComponentFactory._get_parser(model, config), stream_response=True
|
2109
2104
|
)
|
2110
2105
|
|
2111
|
-
|
2112
|
-
|
2106
|
+
def create_gzip_decoder(
|
2107
|
+
self, model: GzipDecoderModel, config: Config, **kwargs: Any
|
2108
|
+
) -> Decoder:
|
2113
2109
|
return CompositeRawDecoder(
|
2114
|
-
parser=ModelToComponentFactory._get_parser(model, config),
|
2110
|
+
parser=ModelToComponentFactory._get_parser(model, config),
|
2111
|
+
stream_response=False if self._emit_connector_builder_messages else True,
|
2115
2112
|
)
|
2116
2113
|
|
2117
2114
|
@staticmethod
|
@@ -2632,6 +2629,47 @@ class ModelToComponentFactory:
|
|
2632
2629
|
transformations: List[RecordTransformation],
|
2633
2630
|
**kwargs: Any,
|
2634
2631
|
) -> AsyncRetriever:
|
2632
|
+
def _get_download_retriever() -> SimpleRetrieverTestReadDecorator | SimpleRetriever:
|
2633
|
+
record_selector = RecordSelector(
|
2634
|
+
extractor=download_extractor,
|
2635
|
+
name=name,
|
2636
|
+
record_filter=None,
|
2637
|
+
transformations=transformations,
|
2638
|
+
schema_normalization=TypeTransformer(TransformConfig.NoTransform),
|
2639
|
+
config=config,
|
2640
|
+
parameters={},
|
2641
|
+
)
|
2642
|
+
paginator = (
|
2643
|
+
self._create_component_from_model(
|
2644
|
+
model=model.download_paginator, decoder=decoder, config=config, url_base=""
|
2645
|
+
)
|
2646
|
+
if model.download_paginator
|
2647
|
+
else NoPagination(parameters={})
|
2648
|
+
)
|
2649
|
+
maximum_number_of_slices = self._limit_slices_fetched or 5
|
2650
|
+
|
2651
|
+
if self._limit_slices_fetched or self._emit_connector_builder_messages:
|
2652
|
+
return SimpleRetrieverTestReadDecorator(
|
2653
|
+
requester=download_requester,
|
2654
|
+
record_selector=record_selector,
|
2655
|
+
primary_key=None,
|
2656
|
+
name=job_download_components_name,
|
2657
|
+
paginator=paginator,
|
2658
|
+
config=config,
|
2659
|
+
parameters={},
|
2660
|
+
maximum_number_of_slices=maximum_number_of_slices,
|
2661
|
+
)
|
2662
|
+
|
2663
|
+
return SimpleRetriever(
|
2664
|
+
requester=download_requester,
|
2665
|
+
record_selector=record_selector,
|
2666
|
+
primary_key=None,
|
2667
|
+
name=job_download_components_name,
|
2668
|
+
paginator=paginator,
|
2669
|
+
config=config,
|
2670
|
+
parameters={},
|
2671
|
+
)
|
2672
|
+
|
2635
2673
|
decoder = (
|
2636
2674
|
self._create_component_from_model(model=model.decoder, config=config)
|
2637
2675
|
if model.decoder
|
@@ -2685,29 +2723,7 @@ class ModelToComponentFactory:
|
|
2685
2723
|
config=config,
|
2686
2724
|
name=job_download_components_name,
|
2687
2725
|
)
|
2688
|
-
download_retriever =
|
2689
|
-
requester=download_requester,
|
2690
|
-
record_selector=RecordSelector(
|
2691
|
-
extractor=download_extractor,
|
2692
|
-
name=name,
|
2693
|
-
record_filter=None,
|
2694
|
-
transformations=transformations,
|
2695
|
-
schema_normalization=TypeTransformer(TransformConfig.NoTransform),
|
2696
|
-
config=config,
|
2697
|
-
parameters={},
|
2698
|
-
),
|
2699
|
-
primary_key=None,
|
2700
|
-
name=job_download_components_name,
|
2701
|
-
paginator=(
|
2702
|
-
self._create_component_from_model(
|
2703
|
-
model=model.download_paginator, decoder=decoder, config=config, url_base=""
|
2704
|
-
)
|
2705
|
-
if model.download_paginator
|
2706
|
-
else NoPagination(parameters={})
|
2707
|
-
),
|
2708
|
-
config=config,
|
2709
|
-
parameters={},
|
2710
|
-
)
|
2726
|
+
download_retriever = _get_download_retriever()
|
2711
2727
|
abort_requester = (
|
2712
2728
|
self._create_component_from_model(
|
2713
2729
|
model=model.abort_requester,
|
@@ -3029,8 +3045,9 @@ class ModelToComponentFactory:
|
|
3029
3045
|
)
|
3030
3046
|
|
3031
3047
|
def create_rate(self, model: RateModel, config: Config, **kwargs: Any) -> Rate:
|
3048
|
+
interpolated_limit = InterpolatedString.create(str(model.limit), parameters={})
|
3032
3049
|
return Rate(
|
3033
|
-
limit=
|
3050
|
+
limit=int(interpolated_limit.eval(config=config)),
|
3034
3051
|
interval=parse_duration(model.interval),
|
3035
3052
|
)
|
3036
3053
|
|
@@ -3049,31 +3066,3 @@ class ModelToComponentFactory:
|
|
3049
3066
|
self._api_budget = self.create_component(
|
3050
3067
|
model_type=HTTPAPIBudgetModel, component_definition=component_definition, config=config
|
3051
3068
|
)
|
3052
|
-
|
3053
|
-
def create_grouping_partition_router(
|
3054
|
-
self, model: GroupingPartitionRouterModel, config: Config, **kwargs: Any
|
3055
|
-
) -> GroupingPartitionRouter:
|
3056
|
-
underlying_router = self._create_component_from_model(
|
3057
|
-
model=model.underlying_partition_router, config=config
|
3058
|
-
)
|
3059
|
-
if model.group_size < 1:
|
3060
|
-
raise ValueError(f"Group size must be greater than 0, got {model.group_size}")
|
3061
|
-
|
3062
|
-
if not isinstance(underlying_router, PartitionRouter):
|
3063
|
-
raise ValueError(
|
3064
|
-
f"Underlying partition router must be a PartitionRouter subclass, got {type(underlying_router)}"
|
3065
|
-
)
|
3066
|
-
|
3067
|
-
if isinstance(underlying_router, SubstreamPartitionRouter):
|
3068
|
-
if any(
|
3069
|
-
parent_config.request_option
|
3070
|
-
for parent_config in underlying_router.parent_stream_configs
|
3071
|
-
):
|
3072
|
-
raise ValueError("Request options are not supported for GroupingPartitionRouter.")
|
3073
|
-
|
3074
|
-
return GroupingPartitionRouter(
|
3075
|
-
group_size=model.group_size,
|
3076
|
-
underlying_partition_router=underlying_router,
|
3077
|
-
deduplicate=model.deduplicate if model.deduplicate is not None else True,
|
3078
|
-
config=config,
|
3079
|
-
)
|
@@ -8,9 +8,6 @@ from airbyte_cdk.sources.declarative.partition_routers.async_job_partition_route
|
|
8
8
|
from airbyte_cdk.sources.declarative.partition_routers.cartesian_product_stream_slicer import (
|
9
9
|
CartesianProductStreamSlicer,
|
10
10
|
)
|
11
|
-
from airbyte_cdk.sources.declarative.partition_routers.grouping_partition_router import (
|
12
|
-
GroupingPartitionRouter,
|
13
|
-
)
|
14
11
|
from airbyte_cdk.sources.declarative.partition_routers.list_partition_router import (
|
15
12
|
ListPartitionRouter,
|
16
13
|
)
|
@@ -25,7 +22,6 @@ from airbyte_cdk.sources.declarative.partition_routers.substream_partition_route
|
|
25
22
|
__all__ = [
|
26
23
|
"AsyncJobPartitionRouter",
|
27
24
|
"CartesianProductStreamSlicer",
|
28
|
-
"GroupingPartitionRouter",
|
29
25
|
"ListPartitionRouter",
|
30
26
|
"SinglePartitionRouter",
|
31
27
|
"SubstreamPartitionRouter",
|
@@ -23,6 +23,7 @@ from airbyte_cdk.sources.declarative.extractors.response_to_file_extractor impor
|
|
23
23
|
)
|
24
24
|
from airbyte_cdk.sources.declarative.requesters.requester import Requester
|
25
25
|
from airbyte_cdk.sources.declarative.retrievers.simple_retriever import SimpleRetriever
|
26
|
+
from airbyte_cdk.sources.http_logger import format_http_message
|
26
27
|
from airbyte_cdk.sources.types import Record, StreamSlice
|
27
28
|
from airbyte_cdk.utils import AirbyteTracedException
|
28
29
|
|
@@ -71,7 +72,15 @@ class AsyncHttpJobRepository(AsyncJobRepository):
|
|
71
72
|
"""
|
72
73
|
|
73
74
|
polling_response: Optional[requests.Response] = self.polling_requester.send_request(
|
74
|
-
stream_slice=stream_slice
|
75
|
+
stream_slice=stream_slice,
|
76
|
+
log_formatter=lambda polling_response: format_http_message(
|
77
|
+
response=polling_response,
|
78
|
+
title="Async Job -- Polling",
|
79
|
+
description="Poll the status of the server-side async job.",
|
80
|
+
stream_name=None,
|
81
|
+
is_auxiliary=True,
|
82
|
+
type="ASYNC_POLL",
|
83
|
+
),
|
75
84
|
)
|
76
85
|
if polling_response is None:
|
77
86
|
raise AirbyteTracedException(
|
@@ -118,8 +127,17 @@ class AsyncHttpJobRepository(AsyncJobRepository):
|
|
118
127
|
"""
|
119
128
|
|
120
129
|
response: Optional[requests.Response] = self.creation_requester.send_request(
|
121
|
-
stream_slice=stream_slice
|
130
|
+
stream_slice=stream_slice,
|
131
|
+
log_formatter=lambda response: format_http_message(
|
132
|
+
response=response,
|
133
|
+
title="Async Job -- Create",
|
134
|
+
description="Create the server-side async job.",
|
135
|
+
stream_name=None,
|
136
|
+
is_auxiliary=True,
|
137
|
+
type="ASYNC_CREATE",
|
138
|
+
),
|
122
139
|
)
|
140
|
+
|
123
141
|
if not response:
|
124
142
|
raise AirbyteTracedException(
|
125
143
|
internal_message="Always expect a response or an exception from creation_requester",
|
@@ -217,13 +235,33 @@ class AsyncHttpJobRepository(AsyncJobRepository):
|
|
217
235
|
if not self.abort_requester:
|
218
236
|
return
|
219
237
|
|
220
|
-
self.abort_requester.send_request(
|
238
|
+
abort_response = self.abort_requester.send_request(
|
239
|
+
stream_slice=self._get_create_job_stream_slice(job),
|
240
|
+
log_formatter=lambda abort_response: format_http_message(
|
241
|
+
response=abort_response,
|
242
|
+
title="Async Job -- Abort",
|
243
|
+
description="Abort the running server-side async job.",
|
244
|
+
stream_name=None,
|
245
|
+
is_auxiliary=True,
|
246
|
+
type="ASYNC_ABORT",
|
247
|
+
),
|
248
|
+
)
|
221
249
|
|
222
250
|
def delete(self, job: AsyncJob) -> None:
|
223
251
|
if not self.delete_requester:
|
224
252
|
return
|
225
253
|
|
226
|
-
self.delete_requester.send_request(
|
254
|
+
delete_job_reponse = self.delete_requester.send_request(
|
255
|
+
stream_slice=self._get_create_job_stream_slice(job),
|
256
|
+
log_formatter=lambda delete_job_reponse: format_http_message(
|
257
|
+
response=delete_job_reponse,
|
258
|
+
title="Async Job -- Delete",
|
259
|
+
description="Delete the specified job from the list of Jobs.",
|
260
|
+
stream_name=None,
|
261
|
+
is_auxiliary=True,
|
262
|
+
type="ASYNC_DELETE",
|
263
|
+
),
|
264
|
+
)
|
227
265
|
self._clean_up_job(job.api_job_id())
|
228
266
|
|
229
267
|
def _clean_up_job(self, job_id: str) -> None:
|
@@ -1,13 +1,12 @@
|
|
1
1
|
# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
2
2
|
|
3
3
|
|
4
|
-
from dataclasses import InitVar, dataclass
|
4
|
+
from dataclasses import InitVar, dataclass, field
|
5
5
|
from typing import Any, Iterable, Mapping, Optional
|
6
6
|
|
7
7
|
from typing_extensions import deprecated
|
8
8
|
|
9
9
|
from airbyte_cdk.sources.declarative.async_job.job import AsyncJob
|
10
|
-
from airbyte_cdk.sources.declarative.async_job.job_orchestrator import AsyncPartition
|
11
10
|
from airbyte_cdk.sources.declarative.extractors.record_selector import RecordSelector
|
12
11
|
from airbyte_cdk.sources.declarative.partition_routers.async_job_partition_router import (
|
13
12
|
AsyncJobPartitionRouter,
|
@@ -16,6 +15,7 @@ from airbyte_cdk.sources.declarative.retrievers.retriever import Retriever
|
|
16
15
|
from airbyte_cdk.sources.source import ExperimentalClassWarning
|
17
16
|
from airbyte_cdk.sources.streams.core import StreamData
|
18
17
|
from airbyte_cdk.sources.types import Config, StreamSlice, StreamState
|
18
|
+
from airbyte_cdk.sources.utils.slice_logger import AlwaysLogSliceLogger
|
19
19
|
|
20
20
|
|
21
21
|
@deprecated(
|
@@ -28,6 +28,10 @@ class AsyncRetriever(Retriever):
|
|
28
28
|
parameters: InitVar[Mapping[str, Any]]
|
29
29
|
record_selector: RecordSelector
|
30
30
|
stream_slicer: AsyncJobPartitionRouter
|
31
|
+
slice_logger: AlwaysLogSliceLogger = field(
|
32
|
+
init=False,
|
33
|
+
default_factory=lambda: AlwaysLogSliceLogger(),
|
34
|
+
)
|
31
35
|
|
32
36
|
def __post_init__(self, parameters: Mapping[str, Any]) -> None:
|
33
37
|
self._parameters = parameters
|
@@ -75,13 +79,16 @@ class AsyncRetriever(Retriever):
|
|
75
79
|
return stream_slice.extra_fields.get("jobs", []) if stream_slice else []
|
76
80
|
|
77
81
|
def stream_slices(self) -> Iterable[Optional[StreamSlice]]:
|
78
|
-
|
82
|
+
yield from self.stream_slicer.stream_slices()
|
79
83
|
|
80
84
|
def read_records(
|
81
85
|
self,
|
82
86
|
records_schema: Mapping[str, Any],
|
83
87
|
stream_slice: Optional[StreamSlice] = None,
|
84
88
|
) -> Iterable[StreamData]:
|
89
|
+
# emit the slice_descriptor log message, for connector builder TestRead
|
90
|
+
yield self.slice_logger.create_slice_log_message(stream_slice.cursor_slice) # type: ignore
|
91
|
+
|
85
92
|
stream_state: StreamState = self._get_stream_state()
|
86
93
|
jobs: Iterable[AsyncJob] = self._validate_and_get_stream_slice_jobs(stream_slice)
|
87
94
|
records: Iterable[Mapping[str, Any]] = self.stream_slicer.fetch_records(jobs)
|
@@ -15,11 +15,14 @@ def format_http_message(
|
|
15
15
|
description: str,
|
16
16
|
stream_name: Optional[str],
|
17
17
|
is_auxiliary: bool | None = None,
|
18
|
+
type: Optional[str] = None,
|
18
19
|
) -> LogMessage:
|
20
|
+
request_type: str = type if type else "HTTP"
|
19
21
|
request = response.request
|
20
22
|
log_message = {
|
21
23
|
"http": {
|
22
24
|
"title": title,
|
25
|
+
"type": request_type,
|
23
26
|
"description": description,
|
24
27
|
"request": {
|
25
28
|
"method": request.method,
|
@@ -9,12 +9,12 @@ airbyte_cdk/connector_builder/README.md,sha256=Hw3wvVewuHG9-QgsAq1jDiKuLlStDxKBz
|
|
9
9
|
airbyte_cdk/connector_builder/__init__.py,sha256=4Hw-PX1-VgESLF16cDdvuYCzGJtHntThLF4qIiULWeo,61
|
10
10
|
airbyte_cdk/connector_builder/connector_builder_handler.py,sha256=BntqkP63RBPvGCtB3CrLLtYplfSlBR42kwXyyk4YGas,4268
|
11
11
|
airbyte_cdk/connector_builder/main.py,sha256=ubAPE0Oo5gjZOa-KMtLLJQkc8_inUpFR3sIb2DEh2No,3722
|
12
|
-
airbyte_cdk/connector_builder/models.py,sha256=
|
12
|
+
airbyte_cdk/connector_builder/models.py,sha256=9pIZ98LW_d6fRS39VdnUOf3cxGt4TkC5MJ0_OrzcCRk,1578
|
13
13
|
airbyte_cdk/connector_builder/test_reader/__init__.py,sha256=iTwBMoI9vaJotEgpqZbFjlxRcbxXYypSVJ9YxeHk7wc,120
|
14
|
-
airbyte_cdk/connector_builder/test_reader/helpers.py,sha256=
|
15
|
-
airbyte_cdk/connector_builder/test_reader/message_grouper.py,sha256=
|
14
|
+
airbyte_cdk/connector_builder/test_reader/helpers.py,sha256=Iczn-_iczS2CaIAunWwyFcX0uLTra8Wh9JVfzm1Gfxo,26765
|
15
|
+
airbyte_cdk/connector_builder/test_reader/message_grouper.py,sha256=84BAEPIBHMq3WCfO14WNvh_q7OsjGgDt0q1FTu8eW-w,6918
|
16
16
|
airbyte_cdk/connector_builder/test_reader/reader.py,sha256=GurMB4ITO_PntvhIHSJkXbhynLilI4DObY5A2axavXo,20667
|
17
|
-
airbyte_cdk/connector_builder/test_reader/types.py,sha256=
|
17
|
+
airbyte_cdk/connector_builder/test_reader/types.py,sha256=hPZG3jO03kBaPyW94NI3JHRS1jxXGSNBcN1HFzOxo5Y,2528
|
18
18
|
airbyte_cdk/destinations/__init__.py,sha256=FyDp28PT_YceJD5HDFhA-mrGfX9AONIyMQ4d68CHNxQ,213
|
19
19
|
airbyte_cdk/destinations/destination.py,sha256=CIq-yb8C_0QvcKCtmStaHfiqn53GEfRAIGGCkJhKP1Q,5880
|
20
20
|
airbyte_cdk/destinations/vector_db_based/README.md,sha256=QAe8c_1Afme4r2TCE10cTSaxUE3zgCBuArSuRQqK8tA,2115
|
@@ -60,22 +60,22 @@ airbyte_cdk/sources/declarative/auth/jwt.py,sha256=SICqNsN2Cn_EgKadIgWuZpQxuMHyz
|
|
60
60
|
airbyte_cdk/sources/declarative/auth/oauth.py,sha256=SUfib1oSzlyRRnOSg8Bui73mfyrcyr9OssdchbKdu4s,14162
|
61
61
|
airbyte_cdk/sources/declarative/auth/selective_authenticator.py,sha256=qGwC6YsCldr1bIeKG6Qo-A9a5cTdHw-vcOn3OtQrS4c,1540
|
62
62
|
airbyte_cdk/sources/declarative/auth/token.py,sha256=2EnE78EhBOY9hbeZnQJ9AuFaM-G7dccU-oKo_LThRQk,11070
|
63
|
-
airbyte_cdk/sources/declarative/auth/token_provider.py,sha256=
|
63
|
+
airbyte_cdk/sources/declarative/auth/token_provider.py,sha256=Jzuxlmt1_-_aFC_n0OmP8L1nDOacLzbEVVx3kjdX_W8,3104
|
64
64
|
airbyte_cdk/sources/declarative/checks/__init__.py,sha256=nsVV5Bo0E_tBNd8A4Xdsdb-75PpcLo5RQu2RQ_Gv-ME,806
|
65
65
|
airbyte_cdk/sources/declarative/checks/check_dynamic_stream.py,sha256=HUktywjI8pqOeED08UGqponUSwxs2TOAECTowlWlrRE,2138
|
66
66
|
airbyte_cdk/sources/declarative/checks/check_stream.py,sha256=dAA-UhmMj0WLXCkRQrilWCfJmncBzXCZ18ptRNip3XA,2139
|
67
67
|
airbyte_cdk/sources/declarative/checks/connection_checker.py,sha256=MBRJo6WJlZQHpIfOGaNOkkHUmgUl_4wDM6VPo41z5Ss,1383
|
68
68
|
airbyte_cdk/sources/declarative/concurrency_level/__init__.py,sha256=5XUqrmlstYlMM0j6crktlKQwALek0uiz2D3WdM46MyA,191
|
69
69
|
airbyte_cdk/sources/declarative/concurrency_level/concurrency_level.py,sha256=YIwCTCpOr_QSNW4ltQK0yUGWInI8PKNY216HOOegYLk,2101
|
70
|
-
airbyte_cdk/sources/declarative/concurrent_declarative_source.py,sha256=
|
70
|
+
airbyte_cdk/sources/declarative/concurrent_declarative_source.py,sha256=KBF9wdPC5KauFwg9dv4pFHLz01ZMwbMvN5ZCcZgiBEE,25424
|
71
71
|
airbyte_cdk/sources/declarative/datetime/__init__.py,sha256=l9LG7Qm6e5r_qgqfVKnx3mXYtg1I9MmMjomVIPfU4XA,177
|
72
72
|
airbyte_cdk/sources/declarative/datetime/datetime_parser.py,sha256=SX9JjdesN1edN2WVUVMzU_ptqp2QB1OnsnjZ4mwcX7w,2579
|
73
73
|
airbyte_cdk/sources/declarative/datetime/min_max_datetime.py,sha256=0BHBtDNQZfvwM45-tY5pNlTcKAFSGGNxemoi0Jic-0E,5785
|
74
|
-
airbyte_cdk/sources/declarative/declarative_component_schema.yaml,sha256=
|
74
|
+
airbyte_cdk/sources/declarative/declarative_component_schema.yaml,sha256=5o5GsltzbVL2jyXvjWzUoV_r5xpwG_YdLSVUuG_d_34,144548
|
75
75
|
airbyte_cdk/sources/declarative/declarative_source.py,sha256=nF7wBqFd3AQmEKAm4CnIo29CJoQL562cJGSCeL8U8bA,1531
|
76
76
|
airbyte_cdk/sources/declarative/declarative_stream.py,sha256=venZjfpvtqr3oFSuvMBWtn4h9ayLhD4L65ACuXCDZ64,10445
|
77
77
|
airbyte_cdk/sources/declarative/decoders/__init__.py,sha256=JHb_0d3SE6kNY10mxA5YBEKPeSbsWYjByq1gUQxepoE,953
|
78
|
-
airbyte_cdk/sources/declarative/decoders/composite_raw_decoder.py,sha256=
|
78
|
+
airbyte_cdk/sources/declarative/decoders/composite_raw_decoder.py,sha256=dQtXYK2ngM8JnVsQi_UXQugWYjotMrzRJfc6kNz7Sx4,5003
|
79
79
|
airbyte_cdk/sources/declarative/decoders/decoder.py,sha256=sl-Gt8lXi7yD2Q-sD8je5QS2PbgrgsYjxRLWsay7DMc,826
|
80
80
|
airbyte_cdk/sources/declarative/decoders/json_decoder.py,sha256=BdWpXXPhEGf_zknggJmhojLosmxuw51RBVTS0jvdCPc,2080
|
81
81
|
airbyte_cdk/sources/declarative/decoders/noop_decoder.py,sha256=iZh0yKY_JzgBnJWiubEusf5c0o6Khd-8EWFWT-8EgFo,542
|
@@ -89,10 +89,10 @@ airbyte_cdk/sources/declarative/extractors/http_selector.py,sha256=2zWZ4ewTqQC8V
|
|
89
89
|
airbyte_cdk/sources/declarative/extractors/record_extractor.py,sha256=XJELMjahAsaomlvQgN2zrNO0DJX0G0fr9r682gUz7Pg,691
|
90
90
|
airbyte_cdk/sources/declarative/extractors/record_filter.py,sha256=yTdEkyDUSW2KbFkEwJJMlS963C955LgCCOVfTmmScpQ,3367
|
91
91
|
airbyte_cdk/sources/declarative/extractors/record_selector.py,sha256=HCqx7IyENM_aRF4it2zJN26_vDu6WeP8XgCxQWHUvcY,6934
|
92
|
-
airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py,sha256=
|
92
|
+
airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py,sha256=L6hQV7bdwEp8y-TBMeQY-xmrNyRggL14lKXdWnzYFfA,6622
|
93
93
|
airbyte_cdk/sources/declarative/extractors/type_transformer.py,sha256=d6Y2Rfg8pMVEEnHllfVksWZdNVOU55yk34O03dP9muY,1626
|
94
94
|
airbyte_cdk/sources/declarative/incremental/__init__.py,sha256=U1oZKtBaEC6IACmvziY9Wzg7Z8EgF4ZuR7NwvjlB_Sk,1255
|
95
|
-
airbyte_cdk/sources/declarative/incremental/concurrent_partition_cursor.py,sha256=
|
95
|
+
airbyte_cdk/sources/declarative/incremental/concurrent_partition_cursor.py,sha256=MT5JbdEbnPzk3VWZGGvThe4opoX5dHhSXFrnTRYC6dg,22210
|
96
96
|
airbyte_cdk/sources/declarative/incremental/datetime_based_cursor.py,sha256=Rbe6lJLTtZ5en33MwZiB9-H9-AwDMNHgwBZs8EqhYqk,22172
|
97
97
|
airbyte_cdk/sources/declarative/incremental/declarative_cursor.py,sha256=5Bhw9VRPyIuCaD0wmmq_L3DZsa-rJgtKSEUzSd8YYD0,536
|
98
98
|
airbyte_cdk/sources/declarative/incremental/global_substream_cursor.py,sha256=2tsE6FgXzemf4fZZ4uGtd8QpRBl9GJ2CRqSNJE5p0EI,16077
|
@@ -113,17 +113,16 @@ airbyte_cdk/sources/declarative/migrations/__init__.py,sha256=47DEQpj8HBSa-_TImW
|
|
113
113
|
airbyte_cdk/sources/declarative/migrations/legacy_to_per_partition_state_migration.py,sha256=iemy3fKLczcU0-Aor7tx5jcT6DRedKMqyK7kCOp01hg,3924
|
114
114
|
airbyte_cdk/sources/declarative/migrations/state_migration.py,sha256=KWPjealMLKSMtajXgkdGgKg7EmTLR-CqqD7UIh0-eDU,794
|
115
115
|
airbyte_cdk/sources/declarative/models/__init__.py,sha256=nUFxNCiKeYRVXuZEKA7GD-lTHxsiKcQ8FitZjKhPIvE,100
|
116
|
-
airbyte_cdk/sources/declarative/models/declarative_component_schema.py,sha256=
|
116
|
+
airbyte_cdk/sources/declarative/models/declarative_component_schema.py,sha256=gNL9DqajD2A8UBnKAz7F7YQuYH7frQyHiPQPIMGq2xo,101958
|
117
117
|
airbyte_cdk/sources/declarative/parsers/__init__.py,sha256=ZnqYNxHsKCgO38IwB34RQyRMXTs4GTvlRi3ImKnIioo,61
|
118
118
|
airbyte_cdk/sources/declarative/parsers/custom_code_compiler.py,sha256=958MMX6_ZOJUlDDdNr9Krosgi2bCKGx2Z765M2Woz18,5505
|
119
119
|
airbyte_cdk/sources/declarative/parsers/custom_exceptions.py,sha256=Rir9_z3Kcd5Es0-LChrzk-0qubAsiK_RSEnLmK2OXm8,553
|
120
120
|
airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py,sha256=CXwTfD3wSQq3okcqwigpprbHhSURUokh4GK2OmOyKC8,9132
|
121
121
|
airbyte_cdk/sources/declarative/parsers/manifest_reference_resolver.py,sha256=IWUOdF03o-aQn0Occo1BJCxU0Pz-QILk5L67nzw2thw,6803
|
122
|
-
airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py,sha256=
|
123
|
-
airbyte_cdk/sources/declarative/partition_routers/__init__.py,sha256=
|
122
|
+
airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py,sha256=dj7b3s8jaDV7Tb7EXVSzkvC8QY-mxyOf48rxkSMws6A,134851
|
123
|
+
airbyte_cdk/sources/declarative/partition_routers/__init__.py,sha256=HJ-Syp3p7RpyR_OK0X_a2kSyISfu3W-PKrRI16iY0a8,957
|
124
124
|
airbyte_cdk/sources/declarative/partition_routers/async_job_partition_router.py,sha256=VelO7zKqKtzMJ35jyFeg0ypJLQC0plqqIBNXoBW1G2E,3001
|
125
125
|
airbyte_cdk/sources/declarative/partition_routers/cartesian_product_stream_slicer.py,sha256=c5cuVFM6NFkuQqG8Z5IwkBuwDrvXZN1CunUOM_L0ezg,6892
|
126
|
-
airbyte_cdk/sources/declarative/partition_routers/grouping_partition_router.py,sha256=PFoY931wC1i7Elphrd7LCFUPYKOTPEovLXC-mvkQow0,5531
|
127
126
|
airbyte_cdk/sources/declarative/partition_routers/list_partition_router.py,sha256=tmGGpMoOBmaMfhVZq53AEWxoHm2lmNVi6hA2_IVEnAA,4882
|
128
127
|
airbyte_cdk/sources/declarative/partition_routers/partition_router.py,sha256=YyEIzdmLd1FjbVP3QbQ2VFCLW_P-OGbVh6VpZShp54k,2218
|
129
128
|
airbyte_cdk/sources/declarative/partition_routers/single_partition_router.py,sha256=SKzKjSyfccq4dxGIh-J6ejrgkCHzaiTIazmbmeQiRD4,1942
|
@@ -143,7 +142,7 @@ airbyte_cdk/sources/declarative/requesters/error_handlers/default_error_handler.
|
|
143
142
|
airbyte_cdk/sources/declarative/requesters/error_handlers/default_http_response_filter.py,sha256=q0YkeYUUWO6iErUy0vjqiOkhg8_9d5YcCmtlpXAJJ9E,1314
|
144
143
|
airbyte_cdk/sources/declarative/requesters/error_handlers/error_handler.py,sha256=Tan66odx8VHzfdyyXMQkXz2pJYksllGqvxmpoajgcK4,669
|
145
144
|
airbyte_cdk/sources/declarative/requesters/error_handlers/http_response_filter.py,sha256=E-fQbt4ShfxZVoqfnmOx69C6FUPWZz8BIqI3DN9Kcjs,7935
|
146
|
-
airbyte_cdk/sources/declarative/requesters/http_job_repository.py,sha256=
|
145
|
+
airbyte_cdk/sources/declarative/requesters/http_job_repository.py,sha256=Wjh5ARW1kcuRfihfFEloQ_l1N6yY1NN_4qFZJiqlH4o,11612
|
147
146
|
airbyte_cdk/sources/declarative/requesters/http_requester.py,sha256=pR2uR5b9eGyvYIOYwus3mz3OaqRu1ozwja_ys1SE7hc,14952
|
148
147
|
airbyte_cdk/sources/declarative/requesters/paginators/__init__.py,sha256=uArbKs9JKNCt7t9tZoeWwjDpyI1HoPp29FNW0JzvaEM,644
|
149
148
|
airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py,sha256=ZW4lwWNAzb4zL0jKc-HjowP5-y0Zg9xi0YlK6tkx_XY,12057
|
@@ -170,7 +169,7 @@ airbyte_cdk/sources/declarative/resolvers/components_resolver.py,sha256=KPjKc0yb
|
|
170
169
|
airbyte_cdk/sources/declarative/resolvers/config_components_resolver.py,sha256=dz4iJV9liD_LzY_Mn4XmAStoUll60R3MIGWV4aN3pgg,5223
|
171
170
|
airbyte_cdk/sources/declarative/resolvers/http_components_resolver.py,sha256=AiojNs8wItJFrENZBFUaDvau3sgwudO6Wkra36upSPo,4639
|
172
171
|
airbyte_cdk/sources/declarative/retrievers/__init__.py,sha256=ix9m1dkR69DcXCXUKC5RK_ZZM7ojTLBQ4IkWQTfmfCk,456
|
173
|
-
airbyte_cdk/sources/declarative/retrievers/async_retriever.py,sha256=
|
172
|
+
airbyte_cdk/sources/declarative/retrievers/async_retriever.py,sha256=dwYZ70eg9DKHEqZydHhMFPkEILbNcXu7E-djOCikNgI,3530
|
174
173
|
airbyte_cdk/sources/declarative/retrievers/retriever.py,sha256=XPLs593Xv8c5cKMc37XzUAYmzlXd1a7eSsspM-CMuWA,1696
|
175
174
|
airbyte_cdk/sources/declarative/retrievers/simple_retriever.py,sha256=bOAKQLgMv1Vca-ozMPRVAg1V5nkyUoPwqC02lKpnLiM,24575
|
176
175
|
airbyte_cdk/sources/declarative/schema/__init__.py,sha256=xU45UvM5O4c1PSM13UHpCdh5hpW3HXy9vRRGEiAC1rg,795
|
@@ -251,7 +250,7 @@ airbyte_cdk/sources/file_based/stream/identities_stream.py,sha256=DwgNU-jDp5vZ_W
|
|
251
250
|
airbyte_cdk/sources/file_based/stream/permissions_file_based_stream.py,sha256=i0Jn0zuAPomLa4pHSu9TQ3gAN5xXhNzPTYVwUDiDEyE,3523
|
252
251
|
airbyte_cdk/sources/file_based/types.py,sha256=INxG7OPnkdUP69oYNKMAbwhvV1AGvLRHs1J6pIia2FI,218
|
253
252
|
airbyte_cdk/sources/http_config.py,sha256=OBZeuyFilm6NlDlBhFQvHhTWabEvZww6OHDIlZujIS0,730
|
254
|
-
airbyte_cdk/sources/http_logger.py,sha256=
|
253
|
+
airbyte_cdk/sources/http_logger.py,sha256=H93kPAujHhPmXNX0JSFG3D-SL6yEFA5PtKot9Hu3TYA,1690
|
255
254
|
airbyte_cdk/sources/message/__init__.py,sha256=y98fzHsQBwXwp2zEa4K5mxGFqjnx9lDn9O0pTk-VS4U,395
|
256
255
|
airbyte_cdk/sources/message/repository.py,sha256=SG7avgti_-dj8FcRHTTrhgLLGJbElv14_zIB0SH8AIc,4763
|
257
256
|
airbyte_cdk/sources/source.py,sha256=KIBBH5VLEb8BZ8B9aROlfaI6OLoJqKDPMJ10jkAR7nk,3611
|
@@ -304,7 +303,7 @@ airbyte_cdk/sources/streams/http/http.py,sha256=0uariNq8OFnlX7iqOHwBhecxA-Hfd5hS
|
|
304
303
|
airbyte_cdk/sources/streams/http/http_client.py,sha256=tDE0ROtxjGMVphvsw8INvGMtZ97hIF-v47pZ3jIyiwc,23011
|
305
304
|
airbyte_cdk/sources/streams/http/rate_limiting.py,sha256=IwdjrHKUnU97XO4qONgYRv4YYW51xQ8SJm4WLafXDB8,6351
|
306
305
|
airbyte_cdk/sources/streams/http/requests_native_auth/__init__.py,sha256=RN0D3nOX1xLgwEwKWu6pkGy3XqBFzKSNZ8Lf6umU2eY,413
|
307
|
-
airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py,sha256=
|
306
|
+
airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py,sha256=P9U8vtcrZ3m0InSG2W0H4gTYTxjQxkIe6mhF9xvO8Ug,18824
|
308
307
|
airbyte_cdk/sources/streams/http/requests_native_auth/abstract_token.py,sha256=Y3n7J-sk5yGjv_OxtY6Z6k0PEsFZmtIRi-x0KCbaHdA,1010
|
309
308
|
airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py,sha256=C2j2uVfi9d-3KgHO3NGxIiFdfASjHOtsd6g_LWPYOAs,20311
|
310
309
|
airbyte_cdk/sources/streams/http/requests_native_auth/token.py,sha256=h5PTzcdH-RQLeCg7xZ45w_484OPUDSwNWl_iMJQmZoI,2526
|
@@ -361,9 +360,9 @@ airbyte_cdk/utils/slice_hasher.py,sha256=EDxgROHDbfG-QKQb59m7h_7crN1tRiawdf5uU7G
|
|
361
360
|
airbyte_cdk/utils/spec_schema_transformations.py,sha256=-5HTuNsnDBAhj-oLeQXwpTGA0HdcjFOf2zTEMUTTg_Y,816
|
362
361
|
airbyte_cdk/utils/stream_status_utils.py,sha256=ZmBoiy5HVbUEHAMrUONxZvxnvfV9CesmQJLDTAIWnWw,1171
|
363
362
|
airbyte_cdk/utils/traced_exception.py,sha256=C8uIBuCL_E4WnBAOPSxBicD06JAldoN9fGsQDp463OY,6292
|
364
|
-
airbyte_cdk-6.37.
|
365
|
-
airbyte_cdk-6.37.
|
366
|
-
airbyte_cdk-6.37.
|
367
|
-
airbyte_cdk-6.37.
|
368
|
-
airbyte_cdk-6.37.
|
369
|
-
airbyte_cdk-6.37.
|
363
|
+
airbyte_cdk-6.37.2.dev1.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
|
364
|
+
airbyte_cdk-6.37.2.dev1.dist-info/LICENSE_SHORT,sha256=aqF6D1NcESmpn-cqsxBtszTEnHKnlsp8L4x9wAh3Nxg,55
|
365
|
+
airbyte_cdk-6.37.2.dev1.dist-info/METADATA,sha256=-nhNtzFqhzx4jLmK3XuDXywoS33ZGSuy9PM0PsRC97g,6015
|
366
|
+
airbyte_cdk-6.37.2.dev1.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
|
367
|
+
airbyte_cdk-6.37.2.dev1.dist-info/entry_points.txt,sha256=fj-e3PAQvsxsQzyyq8UkG1k8spunWnD4BAH2AwlR6NM,95
|
368
|
+
airbyte_cdk-6.37.2.dev1.dist-info/RECORD,,
|