airbyte-cdk 6.36.1__py3-none-any.whl → 6.36.3.dev0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airbyte_cdk/connector_builder/models.py +16 -14
- airbyte_cdk/connector_builder/test_reader/helpers.py +120 -22
- airbyte_cdk/connector_builder/test_reader/message_grouper.py +16 -3
- airbyte_cdk/connector_builder/test_reader/types.py +9 -1
- airbyte_cdk/sources/declarative/auth/token_provider.py +1 -0
- airbyte_cdk/sources/declarative/decoders/composite_raw_decoder.py +5 -2
- airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py +1 -0
- airbyte_cdk/sources/declarative/incremental/concurrent_partition_cursor.py +83 -17
- airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +42 -23
- airbyte_cdk/sources/declarative/requesters/http_job_repository.py +42 -4
- airbyte_cdk/sources/declarative/retrievers/async_retriever.py +10 -3
- airbyte_cdk/sources/http_logger.py +3 -0
- airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py +1 -0
- {airbyte_cdk-6.36.1.dist-info → airbyte_cdk-6.36.3.dev0.dist-info}/METADATA +1 -1
- {airbyte_cdk-6.36.1.dist-info → airbyte_cdk-6.36.3.dev0.dist-info}/RECORD +19 -19
- {airbyte_cdk-6.36.1.dist-info → airbyte_cdk-6.36.3.dev0.dist-info}/LICENSE.txt +0 -0
- {airbyte_cdk-6.36.1.dist-info → airbyte_cdk-6.36.3.dev0.dist-info}/LICENSE_SHORT +0 -0
- {airbyte_cdk-6.36.1.dist-info → airbyte_cdk-6.36.3.dev0.dist-info}/WHEEL +0 -0
- {airbyte_cdk-6.36.1.dist-info → airbyte_cdk-6.36.3.dev0.dist-info}/entry_points.txt +0 -0
@@ -21,20 +21,6 @@ class HttpRequest:
|
|
21
21
|
body: Optional[str] = None
|
22
22
|
|
23
23
|
|
24
|
-
@dataclass
|
25
|
-
class StreamReadPages:
|
26
|
-
records: List[object]
|
27
|
-
request: Optional[HttpRequest] = None
|
28
|
-
response: Optional[HttpResponse] = None
|
29
|
-
|
30
|
-
|
31
|
-
@dataclass
|
32
|
-
class StreamReadSlices:
|
33
|
-
pages: List[StreamReadPages]
|
34
|
-
slice_descriptor: Optional[Dict[str, Any]]
|
35
|
-
state: Optional[List[Dict[str, Any]]] = None
|
36
|
-
|
37
|
-
|
38
24
|
@dataclass
|
39
25
|
class LogMessage:
|
40
26
|
message: str
|
@@ -46,11 +32,27 @@ class LogMessage:
|
|
46
32
|
@dataclass
|
47
33
|
class AuxiliaryRequest:
|
48
34
|
title: str
|
35
|
+
type: str
|
49
36
|
description: str
|
50
37
|
request: HttpRequest
|
51
38
|
response: HttpResponse
|
52
39
|
|
53
40
|
|
41
|
+
@dataclass
|
42
|
+
class StreamReadPages:
|
43
|
+
records: List[object]
|
44
|
+
request: Optional[HttpRequest] = None
|
45
|
+
response: Optional[HttpResponse] = None
|
46
|
+
|
47
|
+
|
48
|
+
@dataclass
|
49
|
+
class StreamReadSlices:
|
50
|
+
pages: List[StreamReadPages]
|
51
|
+
slice_descriptor: Optional[Dict[str, Any]]
|
52
|
+
state: Optional[List[Dict[str, Any]]] = None
|
53
|
+
auxiliary_requests: Optional[List[AuxiliaryRequest]] = None
|
54
|
+
|
55
|
+
|
54
56
|
@dataclass
|
55
57
|
class StreamRead(object):
|
56
58
|
logs: List[LogMessage]
|
@@ -28,7 +28,7 @@ from airbyte_cdk.utils.schema_inferrer import (
|
|
28
28
|
SchemaInferrer,
|
29
29
|
)
|
30
30
|
|
31
|
-
from .types import LOG_MESSAGES_OUTPUT_TYPE
|
31
|
+
from .types import ASYNC_AUXILIARY_REQUEST_TYPES, LOG_MESSAGES_OUTPUT_TYPE
|
32
32
|
|
33
33
|
# -------
|
34
34
|
# Parsers
|
@@ -226,7 +226,8 @@ def should_close_page(
|
|
226
226
|
at_least_one_page_in_group
|
227
227
|
and is_log_message(message)
|
228
228
|
and (
|
229
|
-
is_page_http_request(json_message)
|
229
|
+
is_page_http_request(json_message)
|
230
|
+
or message.log.message.startswith(SliceLogger.SLICE_LOG_PREFIX) # type: ignore[union-attr] # AirbyteMessage with MessageType.LOG has log.message
|
230
231
|
)
|
231
232
|
)
|
232
233
|
|
@@ -330,6 +331,10 @@ def is_auxiliary_http_request(message: Optional[Dict[str, Any]]) -> bool:
|
|
330
331
|
return is_http_log(message) and message.get("http", {}).get("is_auxiliary", False)
|
331
332
|
|
332
333
|
|
334
|
+
def is_async_auxiliary_request(message: AuxiliaryRequest) -> bool:
|
335
|
+
return message.type in ASYNC_AUXILIARY_REQUEST_TYPES
|
336
|
+
|
337
|
+
|
333
338
|
def is_log_message(message: AirbyteMessage) -> bool:
|
334
339
|
"""
|
335
340
|
Determines whether the provided message is of type LOG.
|
@@ -413,6 +418,7 @@ def handle_current_slice(
|
|
413
418
|
current_slice_pages: List[StreamReadPages],
|
414
419
|
current_slice_descriptor: Optional[Dict[str, Any]] = None,
|
415
420
|
latest_state_message: Optional[Dict[str, Any]] = None,
|
421
|
+
auxiliary_requests: Optional[List[AuxiliaryRequest]] = None,
|
416
422
|
) -> StreamReadSlices:
|
417
423
|
"""
|
418
424
|
Handles the current slice by packaging its pages, descriptor, and state into a StreamReadSlices instance.
|
@@ -421,6 +427,7 @@ def handle_current_slice(
|
|
421
427
|
current_slice_pages (List[StreamReadPages]): The pages to be included in the slice.
|
422
428
|
current_slice_descriptor (Optional[Dict[str, Any]]): Descriptor for the current slice, optional.
|
423
429
|
latest_state_message (Optional[Dict[str, Any]]): The latest state message, optional.
|
430
|
+
auxiliary_requests (Optional[List[AuxiliaryRequest]]): The auxiliary requests to include, optional.
|
424
431
|
|
425
432
|
Returns:
|
426
433
|
StreamReadSlices: An object containing the current slice's pages, descriptor, and state.
|
@@ -429,6 +436,7 @@ def handle_current_slice(
|
|
429
436
|
pages=current_slice_pages,
|
430
437
|
slice_descriptor=current_slice_descriptor,
|
431
438
|
state=[latest_state_message] if latest_state_message else [],
|
439
|
+
auxiliary_requests=auxiliary_requests if auxiliary_requests else [],
|
432
440
|
)
|
433
441
|
|
434
442
|
|
@@ -486,29 +494,24 @@ def handle_auxiliary_request(json_message: Dict[str, JsonType]) -> AuxiliaryRequ
|
|
486
494
|
Raises:
|
487
495
|
ValueError: If any of the "airbyte_cdk", "stream", or "http" fields is not a dictionary.
|
488
496
|
"""
|
489
|
-
airbyte_cdk = json_message.get("airbyte_cdk", {})
|
490
|
-
|
491
|
-
if not isinstance(airbyte_cdk, dict):
|
492
|
-
raise ValueError(
|
493
|
-
f"Expected airbyte_cdk to be a dict, got {airbyte_cdk} of type {type(airbyte_cdk)}"
|
494
|
-
)
|
495
|
-
|
496
|
-
stream = airbyte_cdk.get("stream", {})
|
497
497
|
|
498
|
-
|
499
|
-
|
498
|
+
airbyte_cdk = get_airbyte_cdk_from_message(json_message)
|
499
|
+
stream = get_stream_from_airbyte_cdk(airbyte_cdk)
|
500
|
+
title_prefix = get_auxiliary_request_title_prefix(stream)
|
501
|
+
http = get_http_property_from_message(json_message)
|
502
|
+
request_type = get_auxiliary_request_type(stream, http)
|
500
503
|
|
501
|
-
|
502
|
-
|
503
|
-
|
504
|
-
|
505
|
-
raise ValueError(f"Expected http to be a dict, got {http} of type {type(http)}")
|
504
|
+
title = title_prefix + str(http.get("title", None))
|
505
|
+
description = str(http.get("description", None))
|
506
|
+
request = create_request_from_log_message(json_message)
|
507
|
+
response = create_response_from_log_message(json_message)
|
506
508
|
|
507
509
|
return AuxiliaryRequest(
|
508
|
-
title=
|
509
|
-
|
510
|
-
|
511
|
-
|
510
|
+
title=title,
|
511
|
+
type=request_type,
|
512
|
+
description=description,
|
513
|
+
request=request,
|
514
|
+
response=response,
|
512
515
|
)
|
513
516
|
|
514
517
|
|
@@ -558,7 +561,8 @@ def handle_log_message(
|
|
558
561
|
at_least_one_page_in_group,
|
559
562
|
current_page_request,
|
560
563
|
current_page_response,
|
561
|
-
auxiliary_request
|
564
|
+
auxiliary_request,
|
565
|
+
log_message,
|
562
566
|
)
|
563
567
|
|
564
568
|
|
@@ -589,3 +593,97 @@ def handle_record_message(
|
|
589
593
|
datetime_format_inferrer.accumulate(message.record) # type: ignore
|
590
594
|
|
591
595
|
return records_count
|
596
|
+
|
597
|
+
|
598
|
+
# -------
|
599
|
+
# Reusable Getters
|
600
|
+
# -------
|
601
|
+
|
602
|
+
|
603
|
+
def get_airbyte_cdk_from_message(json_message: Dict[str, JsonType]) -> dict: # type: ignore
|
604
|
+
"""
|
605
|
+
Retrieves the "airbyte_cdk" dictionary from the provided JSON message.
|
606
|
+
|
607
|
+
This function validates that the extracted "airbyte_cdk" is of type dict,
|
608
|
+
raising a ValueError if the validation fails.
|
609
|
+
|
610
|
+
Parameters:
|
611
|
+
json_message (Dict[str, JsonType]): A dictionary representing the JSON message.
|
612
|
+
|
613
|
+
Returns:
|
614
|
+
dict: The "airbyte_cdk" dictionary extracted from the JSON message.
|
615
|
+
|
616
|
+
Raises:
|
617
|
+
ValueError: If the "airbyte_cdk" field is not a dictionary.
|
618
|
+
"""
|
619
|
+
airbyte_cdk = json_message.get("airbyte_cdk", {})
|
620
|
+
|
621
|
+
if not isinstance(airbyte_cdk, dict):
|
622
|
+
raise ValueError(
|
623
|
+
f"Expected airbyte_cdk to be a dict, got {airbyte_cdk} of type {type(airbyte_cdk)}"
|
624
|
+
)
|
625
|
+
|
626
|
+
return airbyte_cdk
|
627
|
+
|
628
|
+
|
629
|
+
def get_stream_from_airbyte_cdk(airbyte_cdk: dict) -> dict: # type: ignore
|
630
|
+
"""
|
631
|
+
Retrieves the "stream" dictionary from the provided "airbyte_cdk" dictionary.
|
632
|
+
|
633
|
+
This function ensures that the extracted "stream" is of type dict,
|
634
|
+
raising a ValueError if the validation fails.
|
635
|
+
|
636
|
+
Parameters:
|
637
|
+
airbyte_cdk (dict): The dictionary representing the Airbyte CDK data.
|
638
|
+
|
639
|
+
Returns:
|
640
|
+
dict: The "stream" dictionary extracted from the Airbyte CDK data.
|
641
|
+
|
642
|
+
Raises:
|
643
|
+
ValueError: If the "stream" field is not a dictionary.
|
644
|
+
"""
|
645
|
+
|
646
|
+
stream = airbyte_cdk.get("stream", {})
|
647
|
+
|
648
|
+
if not isinstance(stream, dict):
|
649
|
+
raise ValueError(f"Expected stream to be a dict, got {stream} of type {type(stream)}")
|
650
|
+
|
651
|
+
return stream
|
652
|
+
|
653
|
+
|
654
|
+
def get_auxiliary_request_title_prefix(stream: dict) -> str: # type: ignore
|
655
|
+
"""
|
656
|
+
Generates a title prefix based on the stream type.
|
657
|
+
"""
|
658
|
+
return "Parent stream: " if stream.get("is_substream", False) else ""
|
659
|
+
|
660
|
+
|
661
|
+
def get_http_property_from_message(json_message: Dict[str, JsonType]) -> dict: # type: ignore
|
662
|
+
"""
|
663
|
+
Retrieves the "http" dictionary from the provided JSON message.
|
664
|
+
|
665
|
+
This function validates that the extracted "http" is of type dict,
|
666
|
+
raising a ValueError if the validation fails.
|
667
|
+
|
668
|
+
Parameters:
|
669
|
+
json_message (Dict[str, JsonType]): A dictionary representing the JSON message.
|
670
|
+
|
671
|
+
Returns:
|
672
|
+
dict: The "http" dictionary extracted from the JSON message.
|
673
|
+
|
674
|
+
Raises:
|
675
|
+
ValueError: If the "http" field is not a dictionary.
|
676
|
+
"""
|
677
|
+
http = json_message.get("http", {})
|
678
|
+
|
679
|
+
if not isinstance(http, dict):
|
680
|
+
raise ValueError(f"Expected http to be a dict, got {http} of type {type(http)}")
|
681
|
+
|
682
|
+
return http
|
683
|
+
|
684
|
+
|
685
|
+
def get_auxiliary_request_type(stream: dict, http: dict) -> str: # type: ignore
|
686
|
+
"""
|
687
|
+
Determines the type of the auxiliary request based on the stream and HTTP properties.
|
688
|
+
"""
|
689
|
+
return "PARENT_STREAM" if stream.get("is_substream", False) else str(http.get("type", None))
|
@@ -6,6 +6,7 @@
|
|
6
6
|
from typing import Any, Dict, Iterator, List, Mapping, Optional
|
7
7
|
|
8
8
|
from airbyte_cdk.connector_builder.models import (
|
9
|
+
AuxiliaryRequest,
|
9
10
|
HttpRequest,
|
10
11
|
HttpResponse,
|
11
12
|
StreamReadPages,
|
@@ -24,6 +25,7 @@ from .helpers import (
|
|
24
25
|
handle_current_slice,
|
25
26
|
handle_log_message,
|
26
27
|
handle_record_message,
|
28
|
+
is_async_auxiliary_request,
|
27
29
|
is_config_update_message,
|
28
30
|
is_log_message,
|
29
31
|
is_record_message,
|
@@ -89,6 +91,7 @@ def get_message_groups(
|
|
89
91
|
current_page_request: Optional[HttpRequest] = None
|
90
92
|
current_page_response: Optional[HttpResponse] = None
|
91
93
|
latest_state_message: Optional[Dict[str, Any]] = None
|
94
|
+
slice_auxiliary_requests: List[AuxiliaryRequest] = []
|
92
95
|
|
93
96
|
while records_count < limit and (message := next(messages, None)):
|
94
97
|
json_message = airbyte_message_to_json(message)
|
@@ -106,6 +109,7 @@ def get_message_groups(
|
|
106
109
|
current_slice_pages,
|
107
110
|
current_slice_descriptor,
|
108
111
|
latest_state_message,
|
112
|
+
slice_auxiliary_requests,
|
109
113
|
)
|
110
114
|
current_slice_descriptor = parse_slice_description(message.log.message) # type: ignore
|
111
115
|
current_slice_pages = []
|
@@ -118,7 +122,8 @@ def get_message_groups(
|
|
118
122
|
at_least_one_page_in_group,
|
119
123
|
current_page_request,
|
120
124
|
current_page_response,
|
121
|
-
|
125
|
+
auxiliary_request,
|
126
|
+
log_message,
|
122
127
|
) = handle_log_message(
|
123
128
|
message,
|
124
129
|
json_message,
|
@@ -126,8 +131,15 @@ def get_message_groups(
|
|
126
131
|
current_page_request,
|
127
132
|
current_page_response,
|
128
133
|
)
|
129
|
-
|
130
|
-
|
134
|
+
|
135
|
+
if auxiliary_request:
|
136
|
+
if is_async_auxiliary_request(auxiliary_request):
|
137
|
+
slice_auxiliary_requests.append(auxiliary_request)
|
138
|
+
else:
|
139
|
+
yield auxiliary_request
|
140
|
+
|
141
|
+
if log_message:
|
142
|
+
yield log_message
|
131
143
|
elif is_trace_with_error(message):
|
132
144
|
if message.trace is not None:
|
133
145
|
yield message.trace
|
@@ -157,4 +169,5 @@ def get_message_groups(
|
|
157
169
|
current_slice_pages,
|
158
170
|
current_slice_descriptor,
|
159
171
|
latest_state_message,
|
172
|
+
slice_auxiliary_requests,
|
160
173
|
)
|
@@ -71,5 +71,13 @@ LOG_MESSAGES_OUTPUT_TYPE = tuple[
|
|
71
71
|
bool,
|
72
72
|
HttpRequest | None,
|
73
73
|
HttpResponse | None,
|
74
|
-
AuxiliaryRequest |
|
74
|
+
AuxiliaryRequest | None,
|
75
|
+
AirbyteLogMessage | None,
|
76
|
+
]
|
77
|
+
|
78
|
+
ASYNC_AUXILIARY_REQUEST_TYPES = [
|
79
|
+
"ASYNC_CREATE",
|
80
|
+
"ASYNC_POLL",
|
81
|
+
"ASYNC_ABORT",
|
82
|
+
"ASYNC_DELETE",
|
75
83
|
]
|
@@ -5,7 +5,7 @@ import json
|
|
5
5
|
import logging
|
6
6
|
from abc import ABC, abstractmethod
|
7
7
|
from dataclasses import dataclass
|
8
|
-
from io import BufferedIOBase, TextIOWrapper
|
8
|
+
from io import BufferedIOBase, BytesIO, TextIOWrapper
|
9
9
|
from typing import Any, Generator, MutableMapping, Optional
|
10
10
|
|
11
11
|
import orjson
|
@@ -124,7 +124,8 @@ class CsvParser(Parser):
|
|
124
124
|
"""
|
125
125
|
Parse CSV data from decompressed bytes.
|
126
126
|
"""
|
127
|
-
|
127
|
+
bytes_data = BytesIO(data.read())
|
128
|
+
text_data = TextIOWrapper(bytes_data, encoding=self.encoding) # type: ignore
|
128
129
|
reader = csv.DictReader(text_data, delimiter=self._get_delimiter() or ",")
|
129
130
|
for row in reader:
|
130
131
|
yield row
|
@@ -151,6 +152,8 @@ class CompositeRawDecoder(Decoder):
|
|
151
152
|
self, response: requests.Response
|
152
153
|
) -> Generator[MutableMapping[str, Any], None, None]:
|
153
154
|
if self.is_stream_response():
|
155
|
+
response.raw.auto_close = False
|
154
156
|
yield from self.parser.parse(data=response.raw) # type: ignore[arg-type]
|
157
|
+
response.raw.close()
|
155
158
|
else:
|
156
159
|
yield from self.parser.parse(data=io.BytesIO(response.content))
|
@@ -136,6 +136,7 @@ class ResponseToFileExtractor(RecordExtractor):
|
|
136
136
|
"""
|
137
137
|
|
138
138
|
try:
|
139
|
+
# TODO: Add support for other file types, like `json`, with `pd.read_json()`
|
139
140
|
with open(path, "r", encoding=file_encoding) as data:
|
140
141
|
chunks = pd.read_csv(
|
141
142
|
data, chunksize=chunk_size, iterator=True, dialect="unix", dtype=object
|
@@ -95,6 +95,10 @@ class ConcurrentPerPartitionCursor(Cursor):
|
|
95
95
|
# the oldest partitions can be efficiently removed, maintaining the most recent partitions.
|
96
96
|
self._cursor_per_partition: OrderedDict[str, ConcurrentCursor] = OrderedDict()
|
97
97
|
self._semaphore_per_partition: OrderedDict[str, threading.Semaphore] = OrderedDict()
|
98
|
+
|
99
|
+
# Parent-state tracking: store each partition’s parent state in creation order
|
100
|
+
self._partition_parent_state_map: OrderedDict[str, Mapping[str, Any]] = OrderedDict()
|
101
|
+
|
98
102
|
self._finished_partitions: set[str] = set()
|
99
103
|
self._lock = threading.Lock()
|
100
104
|
self._timer = Timer()
|
@@ -155,11 +159,62 @@ class ConcurrentPerPartitionCursor(Cursor):
|
|
155
159
|
and self._semaphore_per_partition[partition_key]._value == 0
|
156
160
|
):
|
157
161
|
self._update_global_cursor(cursor.state[self.cursor_field.cursor_field_key])
|
158
|
-
|
162
|
+
|
163
|
+
self._check_and_update_parent_state()
|
164
|
+
|
165
|
+
self._emit_state_message()
|
166
|
+
|
167
|
+
def _check_and_update_parent_state(self) -> None:
|
168
|
+
"""
|
169
|
+
Pop the leftmost partition state from _partition_parent_state_map only if
|
170
|
+
*all partitions* up to (and including) that partition key in _semaphore_per_partition
|
171
|
+
are fully finished (i.e. in _finished_partitions and semaphore._value == 0).
|
172
|
+
Additionally, delete finished semaphores with a value of 0 to free up memory,
|
173
|
+
as they are only needed to track errors and completion status.
|
174
|
+
"""
|
175
|
+
last_closed_state = None
|
176
|
+
|
177
|
+
while self._partition_parent_state_map:
|
178
|
+
# Look at the earliest partition key in creation order
|
179
|
+
earliest_key = next(iter(self._partition_parent_state_map))
|
180
|
+
|
181
|
+
# Verify ALL partitions from the left up to earliest_key are finished
|
182
|
+
all_left_finished = True
|
183
|
+
for p_key, sem in list(
|
184
|
+
self._semaphore_per_partition.items()
|
185
|
+
): # Use list to allow modification during iteration
|
186
|
+
# If any earlier partition is still not finished, we must stop
|
187
|
+
if p_key not in self._finished_partitions or sem._value != 0:
|
188
|
+
all_left_finished = False
|
189
|
+
break
|
190
|
+
# Once we've reached earliest_key in the semaphore order, we can stop checking
|
191
|
+
if p_key == earliest_key:
|
192
|
+
break
|
193
|
+
|
194
|
+
# If the partitions up to earliest_key are not all finished, break the while-loop
|
195
|
+
if not all_left_finished:
|
196
|
+
break
|
197
|
+
|
198
|
+
# Pop the leftmost entry from parent-state map
|
199
|
+
_, closed_parent_state = self._partition_parent_state_map.popitem(last=False)
|
200
|
+
last_closed_state = closed_parent_state
|
201
|
+
|
202
|
+
# Clean up finished semaphores with value 0 up to and including earliest_key
|
203
|
+
for p_key in list(self._semaphore_per_partition.keys()):
|
204
|
+
sem = self._semaphore_per_partition[p_key]
|
205
|
+
if p_key in self._finished_partitions and sem._value == 0:
|
206
|
+
del self._semaphore_per_partition[p_key]
|
207
|
+
logger.debug(f"Deleted finished semaphore for partition {p_key} with value 0")
|
208
|
+
if p_key == earliest_key:
|
209
|
+
break
|
210
|
+
|
211
|
+
# Update _parent_state if we popped at least one partition
|
212
|
+
if last_closed_state is not None:
|
213
|
+
self._parent_state = last_closed_state
|
159
214
|
|
160
215
|
def ensure_at_least_one_state_emitted(self) -> None:
|
161
216
|
"""
|
162
|
-
The platform
|
217
|
+
The platform expects at least one state message on successful syncs. Hence, whatever happens, we expect this method to be
|
163
218
|
called.
|
164
219
|
"""
|
165
220
|
if not any(
|
@@ -201,13 +256,19 @@ class ConcurrentPerPartitionCursor(Cursor):
|
|
201
256
|
|
202
257
|
slices = self._partition_router.stream_slices()
|
203
258
|
self._timer.start()
|
204
|
-
for partition in
|
205
|
-
|
259
|
+
for partition, last, parent_state in iterate_with_last_flag_and_state(
|
260
|
+
slices, self._partition_router.get_stream_state
|
261
|
+
):
|
262
|
+
yield from self._generate_slices_from_partition(partition, parent_state)
|
206
263
|
|
207
|
-
def _generate_slices_from_partition(
|
264
|
+
def _generate_slices_from_partition(
|
265
|
+
self, partition: StreamSlice, parent_state: Mapping[str, Any]
|
266
|
+
) -> Iterable[StreamSlice]:
|
208
267
|
# Ensure the maximum number of partitions is not exceeded
|
209
268
|
self._ensure_partition_limit()
|
210
269
|
|
270
|
+
partition_key = self._to_partition_key(partition.partition)
|
271
|
+
|
211
272
|
cursor = self._cursor_per_partition.get(self._to_partition_key(partition.partition))
|
212
273
|
if not cursor:
|
213
274
|
cursor = self._create_cursor(
|
@@ -216,18 +277,26 @@ class ConcurrentPerPartitionCursor(Cursor):
|
|
216
277
|
)
|
217
278
|
with self._lock:
|
218
279
|
self._number_of_partitions += 1
|
219
|
-
self._cursor_per_partition[
|
220
|
-
|
221
|
-
|
222
|
-
|
280
|
+
self._cursor_per_partition[partition_key] = cursor
|
281
|
+
self._semaphore_per_partition[partition_key] = threading.Semaphore(0)
|
282
|
+
|
283
|
+
with self._lock:
|
284
|
+
if (
|
285
|
+
len(self._partition_parent_state_map) == 0
|
286
|
+
or self._partition_parent_state_map[
|
287
|
+
next(reversed(self._partition_parent_state_map))
|
288
|
+
]
|
289
|
+
!= parent_state
|
290
|
+
):
|
291
|
+
self._partition_parent_state_map[partition_key] = deepcopy(parent_state)
|
223
292
|
|
224
293
|
for cursor_slice, is_last_slice, _ in iterate_with_last_flag_and_state(
|
225
294
|
cursor.stream_slices(),
|
226
295
|
lambda: None,
|
227
296
|
):
|
228
|
-
self._semaphore_per_partition[
|
297
|
+
self._semaphore_per_partition[partition_key].release()
|
229
298
|
if is_last_slice:
|
230
|
-
self._finished_partitions.add(
|
299
|
+
self._finished_partitions.add(partition_key)
|
231
300
|
yield StreamSlice(
|
232
301
|
partition=partition, cursor_slice=cursor_slice, extra_fields=partition.extra_fields
|
233
302
|
)
|
@@ -257,9 +326,9 @@ class ConcurrentPerPartitionCursor(Cursor):
|
|
257
326
|
while len(self._cursor_per_partition) > self.DEFAULT_MAX_PARTITIONS_NUMBER - 1:
|
258
327
|
# Try removing finished partitions first
|
259
328
|
for partition_key in list(self._cursor_per_partition.keys()):
|
260
|
-
if (
|
261
|
-
partition_key in self.
|
262
|
-
|
329
|
+
if partition_key in self._finished_partitions and (
|
330
|
+
partition_key not in self._semaphore_per_partition
|
331
|
+
or self._semaphore_per_partition[partition_key]._value == 0
|
263
332
|
):
|
264
333
|
oldest_partition = self._cursor_per_partition.pop(
|
265
334
|
partition_key
|
@@ -338,9 +407,6 @@ class ConcurrentPerPartitionCursor(Cursor):
|
|
338
407
|
self._cursor_per_partition[self._to_partition_key(state["partition"])] = (
|
339
408
|
self._create_cursor(state["cursor"])
|
340
409
|
)
|
341
|
-
self._semaphore_per_partition[self._to_partition_key(state["partition"])] = (
|
342
|
-
threading.Semaphore(0)
|
343
|
-
)
|
344
410
|
|
345
411
|
# set default state for missing partitions if it is per partition with fallback to global
|
346
412
|
if self._GLOBAL_STATE_KEY in stream_state:
|
@@ -2629,6 +2629,47 @@ class ModelToComponentFactory:
|
|
2629
2629
|
transformations: List[RecordTransformation],
|
2630
2630
|
**kwargs: Any,
|
2631
2631
|
) -> AsyncRetriever:
|
2632
|
+
def _get_download_retriever() -> SimpleRetrieverTestReadDecorator | SimpleRetriever:
|
2633
|
+
record_selector = RecordSelector(
|
2634
|
+
extractor=download_extractor,
|
2635
|
+
name=name,
|
2636
|
+
record_filter=None,
|
2637
|
+
transformations=transformations,
|
2638
|
+
schema_normalization=TypeTransformer(TransformConfig.NoTransform),
|
2639
|
+
config=config,
|
2640
|
+
parameters={},
|
2641
|
+
)
|
2642
|
+
paginator = (
|
2643
|
+
self._create_component_from_model(
|
2644
|
+
model=model.download_paginator, decoder=decoder, config=config, url_base=""
|
2645
|
+
)
|
2646
|
+
if model.download_paginator
|
2647
|
+
else NoPagination(parameters={})
|
2648
|
+
)
|
2649
|
+
maximum_number_of_slices = self._limit_slices_fetched or 5
|
2650
|
+
|
2651
|
+
if self._limit_slices_fetched or self._emit_connector_builder_messages:
|
2652
|
+
return SimpleRetrieverTestReadDecorator(
|
2653
|
+
requester=download_requester,
|
2654
|
+
record_selector=record_selector,
|
2655
|
+
primary_key=None,
|
2656
|
+
name=job_download_components_name,
|
2657
|
+
paginator=paginator,
|
2658
|
+
config=config,
|
2659
|
+
parameters={},
|
2660
|
+
maximum_number_of_slices=maximum_number_of_slices,
|
2661
|
+
)
|
2662
|
+
|
2663
|
+
return SimpleRetriever(
|
2664
|
+
requester=download_requester,
|
2665
|
+
record_selector=record_selector,
|
2666
|
+
primary_key=None,
|
2667
|
+
name=job_download_components_name,
|
2668
|
+
paginator=paginator,
|
2669
|
+
config=config,
|
2670
|
+
parameters={},
|
2671
|
+
)
|
2672
|
+
|
2632
2673
|
decoder = (
|
2633
2674
|
self._create_component_from_model(model=model.decoder, config=config)
|
2634
2675
|
if model.decoder
|
@@ -2682,29 +2723,7 @@ class ModelToComponentFactory:
|
|
2682
2723
|
config=config,
|
2683
2724
|
name=job_download_components_name,
|
2684
2725
|
)
|
2685
|
-
download_retriever =
|
2686
|
-
requester=download_requester,
|
2687
|
-
record_selector=RecordSelector(
|
2688
|
-
extractor=download_extractor,
|
2689
|
-
name=name,
|
2690
|
-
record_filter=None,
|
2691
|
-
transformations=transformations,
|
2692
|
-
schema_normalization=TypeTransformer(TransformConfig.NoTransform),
|
2693
|
-
config=config,
|
2694
|
-
parameters={},
|
2695
|
-
),
|
2696
|
-
primary_key=None,
|
2697
|
-
name=job_download_components_name,
|
2698
|
-
paginator=(
|
2699
|
-
self._create_component_from_model(
|
2700
|
-
model=model.download_paginator, decoder=decoder, config=config, url_base=""
|
2701
|
-
)
|
2702
|
-
if model.download_paginator
|
2703
|
-
else NoPagination(parameters={})
|
2704
|
-
),
|
2705
|
-
config=config,
|
2706
|
-
parameters={},
|
2707
|
-
)
|
2726
|
+
download_retriever = _get_download_retriever()
|
2708
2727
|
abort_requester = (
|
2709
2728
|
self._create_component_from_model(
|
2710
2729
|
model=model.abort_requester,
|
@@ -23,6 +23,7 @@ from airbyte_cdk.sources.declarative.extractors.response_to_file_extractor impor
|
|
23
23
|
)
|
24
24
|
from airbyte_cdk.sources.declarative.requesters.requester import Requester
|
25
25
|
from airbyte_cdk.sources.declarative.retrievers.simple_retriever import SimpleRetriever
|
26
|
+
from airbyte_cdk.sources.http_logger import format_http_message
|
26
27
|
from airbyte_cdk.sources.types import Record, StreamSlice
|
27
28
|
from airbyte_cdk.utils import AirbyteTracedException
|
28
29
|
|
@@ -71,7 +72,15 @@ class AsyncHttpJobRepository(AsyncJobRepository):
|
|
71
72
|
"""
|
72
73
|
|
73
74
|
polling_response: Optional[requests.Response] = self.polling_requester.send_request(
|
74
|
-
stream_slice=stream_slice
|
75
|
+
stream_slice=stream_slice,
|
76
|
+
log_formatter=lambda polling_response: format_http_message(
|
77
|
+
response=polling_response,
|
78
|
+
title="Async Job -- Polling",
|
79
|
+
description="Poll the status of the server-side async job.",
|
80
|
+
stream_name=None,
|
81
|
+
is_auxiliary=True,
|
82
|
+
type="ASYNC_POLL",
|
83
|
+
),
|
75
84
|
)
|
76
85
|
if polling_response is None:
|
77
86
|
raise AirbyteTracedException(
|
@@ -118,8 +127,17 @@ class AsyncHttpJobRepository(AsyncJobRepository):
|
|
118
127
|
"""
|
119
128
|
|
120
129
|
response: Optional[requests.Response] = self.creation_requester.send_request(
|
121
|
-
stream_slice=stream_slice
|
130
|
+
stream_slice=stream_slice,
|
131
|
+
log_formatter=lambda response: format_http_message(
|
132
|
+
response=response,
|
133
|
+
title="Async Job -- Create",
|
134
|
+
description="Create the server-side async job.",
|
135
|
+
stream_name=None,
|
136
|
+
is_auxiliary=True,
|
137
|
+
type="ASYNC_CREATE",
|
138
|
+
),
|
122
139
|
)
|
140
|
+
|
123
141
|
if not response:
|
124
142
|
raise AirbyteTracedException(
|
125
143
|
internal_message="Always expect a response or an exception from creation_requester",
|
@@ -217,13 +235,33 @@ class AsyncHttpJobRepository(AsyncJobRepository):
|
|
217
235
|
if not self.abort_requester:
|
218
236
|
return
|
219
237
|
|
220
|
-
self.abort_requester.send_request(
|
238
|
+
abort_response = self.abort_requester.send_request(
|
239
|
+
stream_slice=self._get_create_job_stream_slice(job),
|
240
|
+
log_formatter=lambda abort_response: format_http_message(
|
241
|
+
response=abort_response,
|
242
|
+
title="Async Job -- Abort",
|
243
|
+
description="Abort the running server-side async job.",
|
244
|
+
stream_name=None,
|
245
|
+
is_auxiliary=True,
|
246
|
+
type="ASYNC_ABORT",
|
247
|
+
),
|
248
|
+
)
|
221
249
|
|
222
250
|
def delete(self, job: AsyncJob) -> None:
|
223
251
|
if not self.delete_requester:
|
224
252
|
return
|
225
253
|
|
226
|
-
self.delete_requester.send_request(
|
254
|
+
delete_job_reponse = self.delete_requester.send_request(
|
255
|
+
stream_slice=self._get_create_job_stream_slice(job),
|
256
|
+
log_formatter=lambda delete_job_reponse: format_http_message(
|
257
|
+
response=delete_job_reponse,
|
258
|
+
title="Async Job -- Delete",
|
259
|
+
description="Delete the specified job from the list of Jobs.",
|
260
|
+
stream_name=None,
|
261
|
+
is_auxiliary=True,
|
262
|
+
type="ASYNC_DELETE",
|
263
|
+
),
|
264
|
+
)
|
227
265
|
self._clean_up_job(job.api_job_id())
|
228
266
|
|
229
267
|
def _clean_up_job(self, job_id: str) -> None:
|
@@ -1,13 +1,12 @@
|
|
1
1
|
# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
2
2
|
|
3
3
|
|
4
|
-
from dataclasses import InitVar, dataclass
|
4
|
+
from dataclasses import InitVar, dataclass, field
|
5
5
|
from typing import Any, Iterable, Mapping, Optional
|
6
6
|
|
7
7
|
from typing_extensions import deprecated
|
8
8
|
|
9
9
|
from airbyte_cdk.sources.declarative.async_job.job import AsyncJob
|
10
|
-
from airbyte_cdk.sources.declarative.async_job.job_orchestrator import AsyncPartition
|
11
10
|
from airbyte_cdk.sources.declarative.extractors.record_selector import RecordSelector
|
12
11
|
from airbyte_cdk.sources.declarative.partition_routers.async_job_partition_router import (
|
13
12
|
AsyncJobPartitionRouter,
|
@@ -16,6 +15,7 @@ from airbyte_cdk.sources.declarative.retrievers.retriever import Retriever
|
|
16
15
|
from airbyte_cdk.sources.source import ExperimentalClassWarning
|
17
16
|
from airbyte_cdk.sources.streams.core import StreamData
|
18
17
|
from airbyte_cdk.sources.types import Config, StreamSlice, StreamState
|
18
|
+
from airbyte_cdk.sources.utils.slice_logger import AlwaysLogSliceLogger
|
19
19
|
|
20
20
|
|
21
21
|
@deprecated(
|
@@ -28,6 +28,10 @@ class AsyncRetriever(Retriever):
|
|
28
28
|
parameters: InitVar[Mapping[str, Any]]
|
29
29
|
record_selector: RecordSelector
|
30
30
|
stream_slicer: AsyncJobPartitionRouter
|
31
|
+
slice_logger: AlwaysLogSliceLogger = field(
|
32
|
+
init=False,
|
33
|
+
default_factory=lambda: AlwaysLogSliceLogger(),
|
34
|
+
)
|
31
35
|
|
32
36
|
def __post_init__(self, parameters: Mapping[str, Any]) -> None:
|
33
37
|
self._parameters = parameters
|
@@ -75,13 +79,16 @@ class AsyncRetriever(Retriever):
|
|
75
79
|
return stream_slice.extra_fields.get("jobs", []) if stream_slice else []
|
76
80
|
|
77
81
|
def stream_slices(self) -> Iterable[Optional[StreamSlice]]:
|
78
|
-
|
82
|
+
yield from self.stream_slicer.stream_slices()
|
79
83
|
|
80
84
|
def read_records(
|
81
85
|
self,
|
82
86
|
records_schema: Mapping[str, Any],
|
83
87
|
stream_slice: Optional[StreamSlice] = None,
|
84
88
|
) -> Iterable[StreamData]:
|
89
|
+
# emit the slice_descriptor log message, for connector builder TestRead
|
90
|
+
yield self.slice_logger.create_slice_log_message(stream_slice.cursor_slice) # type: ignore
|
91
|
+
|
85
92
|
stream_state: StreamState = self._get_stream_state()
|
86
93
|
jobs: Iterable[AsyncJob] = self._validate_and_get_stream_slice_jobs(stream_slice)
|
87
94
|
records: Iterable[Mapping[str, Any]] = self.stream_slicer.fetch_records(jobs)
|
@@ -15,11 +15,14 @@ def format_http_message(
|
|
15
15
|
description: str,
|
16
16
|
stream_name: Optional[str],
|
17
17
|
is_auxiliary: bool | None = None,
|
18
|
+
type: Optional[str] = None,
|
18
19
|
) -> LogMessage:
|
20
|
+
request_type: str = type if type else "HTTP"
|
19
21
|
request = response.request
|
20
22
|
log_message = {
|
21
23
|
"http": {
|
22
24
|
"title": title,
|
25
|
+
"type": request_type,
|
23
26
|
"description": description,
|
24
27
|
"request": {
|
25
28
|
"method": request.method,
|
@@ -9,12 +9,12 @@ airbyte_cdk/connector_builder/README.md,sha256=Hw3wvVewuHG9-QgsAq1jDiKuLlStDxKBz
|
|
9
9
|
airbyte_cdk/connector_builder/__init__.py,sha256=4Hw-PX1-VgESLF16cDdvuYCzGJtHntThLF4qIiULWeo,61
|
10
10
|
airbyte_cdk/connector_builder/connector_builder_handler.py,sha256=BntqkP63RBPvGCtB3CrLLtYplfSlBR42kwXyyk4YGas,4268
|
11
11
|
airbyte_cdk/connector_builder/main.py,sha256=ubAPE0Oo5gjZOa-KMtLLJQkc8_inUpFR3sIb2DEh2No,3722
|
12
|
-
airbyte_cdk/connector_builder/models.py,sha256=
|
12
|
+
airbyte_cdk/connector_builder/models.py,sha256=9pIZ98LW_d6fRS39VdnUOf3cxGt4TkC5MJ0_OrzcCRk,1578
|
13
13
|
airbyte_cdk/connector_builder/test_reader/__init__.py,sha256=iTwBMoI9vaJotEgpqZbFjlxRcbxXYypSVJ9YxeHk7wc,120
|
14
|
-
airbyte_cdk/connector_builder/test_reader/helpers.py,sha256=
|
15
|
-
airbyte_cdk/connector_builder/test_reader/message_grouper.py,sha256=
|
14
|
+
airbyte_cdk/connector_builder/test_reader/helpers.py,sha256=Iczn-_iczS2CaIAunWwyFcX0uLTra8Wh9JVfzm1Gfxo,26765
|
15
|
+
airbyte_cdk/connector_builder/test_reader/message_grouper.py,sha256=84BAEPIBHMq3WCfO14WNvh_q7OsjGgDt0q1FTu8eW-w,6918
|
16
16
|
airbyte_cdk/connector_builder/test_reader/reader.py,sha256=GurMB4ITO_PntvhIHSJkXbhynLilI4DObY5A2axavXo,20667
|
17
|
-
airbyte_cdk/connector_builder/test_reader/types.py,sha256=
|
17
|
+
airbyte_cdk/connector_builder/test_reader/types.py,sha256=hPZG3jO03kBaPyW94NI3JHRS1jxXGSNBcN1HFzOxo5Y,2528
|
18
18
|
airbyte_cdk/destinations/__init__.py,sha256=FyDp28PT_YceJD5HDFhA-mrGfX9AONIyMQ4d68CHNxQ,213
|
19
19
|
airbyte_cdk/destinations/destination.py,sha256=CIq-yb8C_0QvcKCtmStaHfiqn53GEfRAIGGCkJhKP1Q,5880
|
20
20
|
airbyte_cdk/destinations/vector_db_based/README.md,sha256=QAe8c_1Afme4r2TCE10cTSaxUE3zgCBuArSuRQqK8tA,2115
|
@@ -60,7 +60,7 @@ airbyte_cdk/sources/declarative/auth/jwt.py,sha256=SICqNsN2Cn_EgKadIgWuZpQxuMHyz
|
|
60
60
|
airbyte_cdk/sources/declarative/auth/oauth.py,sha256=SUfib1oSzlyRRnOSg8Bui73mfyrcyr9OssdchbKdu4s,14162
|
61
61
|
airbyte_cdk/sources/declarative/auth/selective_authenticator.py,sha256=qGwC6YsCldr1bIeKG6Qo-A9a5cTdHw-vcOn3OtQrS4c,1540
|
62
62
|
airbyte_cdk/sources/declarative/auth/token.py,sha256=2EnE78EhBOY9hbeZnQJ9AuFaM-G7dccU-oKo_LThRQk,11070
|
63
|
-
airbyte_cdk/sources/declarative/auth/token_provider.py,sha256=
|
63
|
+
airbyte_cdk/sources/declarative/auth/token_provider.py,sha256=Jzuxlmt1_-_aFC_n0OmP8L1nDOacLzbEVVx3kjdX_W8,3104
|
64
64
|
airbyte_cdk/sources/declarative/checks/__init__.py,sha256=nsVV5Bo0E_tBNd8A4Xdsdb-75PpcLo5RQu2RQ_Gv-ME,806
|
65
65
|
airbyte_cdk/sources/declarative/checks/check_dynamic_stream.py,sha256=HUktywjI8pqOeED08UGqponUSwxs2TOAECTowlWlrRE,2138
|
66
66
|
airbyte_cdk/sources/declarative/checks/check_stream.py,sha256=dAA-UhmMj0WLXCkRQrilWCfJmncBzXCZ18ptRNip3XA,2139
|
@@ -75,7 +75,7 @@ airbyte_cdk/sources/declarative/declarative_component_schema.yaml,sha256=5o5Gslt
|
|
75
75
|
airbyte_cdk/sources/declarative/declarative_source.py,sha256=nF7wBqFd3AQmEKAm4CnIo29CJoQL562cJGSCeL8U8bA,1531
|
76
76
|
airbyte_cdk/sources/declarative/declarative_stream.py,sha256=venZjfpvtqr3oFSuvMBWtn4h9ayLhD4L65ACuXCDZ64,10445
|
77
77
|
airbyte_cdk/sources/declarative/decoders/__init__.py,sha256=JHb_0d3SE6kNY10mxA5YBEKPeSbsWYjByq1gUQxepoE,953
|
78
|
-
airbyte_cdk/sources/declarative/decoders/composite_raw_decoder.py,sha256=
|
78
|
+
airbyte_cdk/sources/declarative/decoders/composite_raw_decoder.py,sha256=54ag8xx_BcZcXuhVp1VkC2AE-KdY2cUgLVUORWaP-ig,5080
|
79
79
|
airbyte_cdk/sources/declarative/decoders/decoder.py,sha256=sl-Gt8lXi7yD2Q-sD8je5QS2PbgrgsYjxRLWsay7DMc,826
|
80
80
|
airbyte_cdk/sources/declarative/decoders/json_decoder.py,sha256=BdWpXXPhEGf_zknggJmhojLosmxuw51RBVTS0jvdCPc,2080
|
81
81
|
airbyte_cdk/sources/declarative/decoders/noop_decoder.py,sha256=iZh0yKY_JzgBnJWiubEusf5c0o6Khd-8EWFWT-8EgFo,542
|
@@ -89,10 +89,10 @@ airbyte_cdk/sources/declarative/extractors/http_selector.py,sha256=2zWZ4ewTqQC8V
|
|
89
89
|
airbyte_cdk/sources/declarative/extractors/record_extractor.py,sha256=XJELMjahAsaomlvQgN2zrNO0DJX0G0fr9r682gUz7Pg,691
|
90
90
|
airbyte_cdk/sources/declarative/extractors/record_filter.py,sha256=yTdEkyDUSW2KbFkEwJJMlS963C955LgCCOVfTmmScpQ,3367
|
91
91
|
airbyte_cdk/sources/declarative/extractors/record_selector.py,sha256=HCqx7IyENM_aRF4it2zJN26_vDu6WeP8XgCxQWHUvcY,6934
|
92
|
-
airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py,sha256=
|
92
|
+
airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py,sha256=L6hQV7bdwEp8y-TBMeQY-xmrNyRggL14lKXdWnzYFfA,6622
|
93
93
|
airbyte_cdk/sources/declarative/extractors/type_transformer.py,sha256=d6Y2Rfg8pMVEEnHllfVksWZdNVOU55yk34O03dP9muY,1626
|
94
94
|
airbyte_cdk/sources/declarative/incremental/__init__.py,sha256=U1oZKtBaEC6IACmvziY9Wzg7Z8EgF4ZuR7NwvjlB_Sk,1255
|
95
|
-
airbyte_cdk/sources/declarative/incremental/concurrent_partition_cursor.py,sha256=
|
95
|
+
airbyte_cdk/sources/declarative/incremental/concurrent_partition_cursor.py,sha256=MT5JbdEbnPzk3VWZGGvThe4opoX5dHhSXFrnTRYC6dg,22210
|
96
96
|
airbyte_cdk/sources/declarative/incremental/datetime_based_cursor.py,sha256=Rbe6lJLTtZ5en33MwZiB9-H9-AwDMNHgwBZs8EqhYqk,22172
|
97
97
|
airbyte_cdk/sources/declarative/incremental/declarative_cursor.py,sha256=5Bhw9VRPyIuCaD0wmmq_L3DZsa-rJgtKSEUzSd8YYD0,536
|
98
98
|
airbyte_cdk/sources/declarative/incremental/global_substream_cursor.py,sha256=2tsE6FgXzemf4fZZ4uGtd8QpRBl9GJ2CRqSNJE5p0EI,16077
|
@@ -119,7 +119,7 @@ airbyte_cdk/sources/declarative/parsers/custom_code_compiler.py,sha256=958MMX6_Z
|
|
119
119
|
airbyte_cdk/sources/declarative/parsers/custom_exceptions.py,sha256=Rir9_z3Kcd5Es0-LChrzk-0qubAsiK_RSEnLmK2OXm8,553
|
120
120
|
airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py,sha256=CXwTfD3wSQq3okcqwigpprbHhSURUokh4GK2OmOyKC8,9132
|
121
121
|
airbyte_cdk/sources/declarative/parsers/manifest_reference_resolver.py,sha256=IWUOdF03o-aQn0Occo1BJCxU0Pz-QILk5L67nzw2thw,6803
|
122
|
-
airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py,sha256=
|
122
|
+
airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py,sha256=dj7b3s8jaDV7Tb7EXVSzkvC8QY-mxyOf48rxkSMws6A,134851
|
123
123
|
airbyte_cdk/sources/declarative/partition_routers/__init__.py,sha256=HJ-Syp3p7RpyR_OK0X_a2kSyISfu3W-PKrRI16iY0a8,957
|
124
124
|
airbyte_cdk/sources/declarative/partition_routers/async_job_partition_router.py,sha256=VelO7zKqKtzMJ35jyFeg0ypJLQC0plqqIBNXoBW1G2E,3001
|
125
125
|
airbyte_cdk/sources/declarative/partition_routers/cartesian_product_stream_slicer.py,sha256=c5cuVFM6NFkuQqG8Z5IwkBuwDrvXZN1CunUOM_L0ezg,6892
|
@@ -142,7 +142,7 @@ airbyte_cdk/sources/declarative/requesters/error_handlers/default_error_handler.
|
|
142
142
|
airbyte_cdk/sources/declarative/requesters/error_handlers/default_http_response_filter.py,sha256=q0YkeYUUWO6iErUy0vjqiOkhg8_9d5YcCmtlpXAJJ9E,1314
|
143
143
|
airbyte_cdk/sources/declarative/requesters/error_handlers/error_handler.py,sha256=Tan66odx8VHzfdyyXMQkXz2pJYksllGqvxmpoajgcK4,669
|
144
144
|
airbyte_cdk/sources/declarative/requesters/error_handlers/http_response_filter.py,sha256=E-fQbt4ShfxZVoqfnmOx69C6FUPWZz8BIqI3DN9Kcjs,7935
|
145
|
-
airbyte_cdk/sources/declarative/requesters/http_job_repository.py,sha256=
|
145
|
+
airbyte_cdk/sources/declarative/requesters/http_job_repository.py,sha256=Wjh5ARW1kcuRfihfFEloQ_l1N6yY1NN_4qFZJiqlH4o,11612
|
146
146
|
airbyte_cdk/sources/declarative/requesters/http_requester.py,sha256=pR2uR5b9eGyvYIOYwus3mz3OaqRu1ozwja_ys1SE7hc,14952
|
147
147
|
airbyte_cdk/sources/declarative/requesters/paginators/__init__.py,sha256=uArbKs9JKNCt7t9tZoeWwjDpyI1HoPp29FNW0JzvaEM,644
|
148
148
|
airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py,sha256=ZW4lwWNAzb4zL0jKc-HjowP5-y0Zg9xi0YlK6tkx_XY,12057
|
@@ -169,7 +169,7 @@ airbyte_cdk/sources/declarative/resolvers/components_resolver.py,sha256=KPjKc0yb
|
|
169
169
|
airbyte_cdk/sources/declarative/resolvers/config_components_resolver.py,sha256=dz4iJV9liD_LzY_Mn4XmAStoUll60R3MIGWV4aN3pgg,5223
|
170
170
|
airbyte_cdk/sources/declarative/resolvers/http_components_resolver.py,sha256=AiojNs8wItJFrENZBFUaDvau3sgwudO6Wkra36upSPo,4639
|
171
171
|
airbyte_cdk/sources/declarative/retrievers/__init__.py,sha256=ix9m1dkR69DcXCXUKC5RK_ZZM7ojTLBQ4IkWQTfmfCk,456
|
172
|
-
airbyte_cdk/sources/declarative/retrievers/async_retriever.py,sha256=
|
172
|
+
airbyte_cdk/sources/declarative/retrievers/async_retriever.py,sha256=dwYZ70eg9DKHEqZydHhMFPkEILbNcXu7E-djOCikNgI,3530
|
173
173
|
airbyte_cdk/sources/declarative/retrievers/retriever.py,sha256=XPLs593Xv8c5cKMc37XzUAYmzlXd1a7eSsspM-CMuWA,1696
|
174
174
|
airbyte_cdk/sources/declarative/retrievers/simple_retriever.py,sha256=bOAKQLgMv1Vca-ozMPRVAg1V5nkyUoPwqC02lKpnLiM,24575
|
175
175
|
airbyte_cdk/sources/declarative/schema/__init__.py,sha256=xU45UvM5O4c1PSM13UHpCdh5hpW3HXy9vRRGEiAC1rg,795
|
@@ -250,7 +250,7 @@ airbyte_cdk/sources/file_based/stream/identities_stream.py,sha256=DwgNU-jDp5vZ_W
|
|
250
250
|
airbyte_cdk/sources/file_based/stream/permissions_file_based_stream.py,sha256=i0Jn0zuAPomLa4pHSu9TQ3gAN5xXhNzPTYVwUDiDEyE,3523
|
251
251
|
airbyte_cdk/sources/file_based/types.py,sha256=INxG7OPnkdUP69oYNKMAbwhvV1AGvLRHs1J6pIia2FI,218
|
252
252
|
airbyte_cdk/sources/http_config.py,sha256=OBZeuyFilm6NlDlBhFQvHhTWabEvZww6OHDIlZujIS0,730
|
253
|
-
airbyte_cdk/sources/http_logger.py,sha256=
|
253
|
+
airbyte_cdk/sources/http_logger.py,sha256=H93kPAujHhPmXNX0JSFG3D-SL6yEFA5PtKot9Hu3TYA,1690
|
254
254
|
airbyte_cdk/sources/message/__init__.py,sha256=y98fzHsQBwXwp2zEa4K5mxGFqjnx9lDn9O0pTk-VS4U,395
|
255
255
|
airbyte_cdk/sources/message/repository.py,sha256=SG7avgti_-dj8FcRHTTrhgLLGJbElv14_zIB0SH8AIc,4763
|
256
256
|
airbyte_cdk/sources/source.py,sha256=KIBBH5VLEb8BZ8B9aROlfaI6OLoJqKDPMJ10jkAR7nk,3611
|
@@ -303,7 +303,7 @@ airbyte_cdk/sources/streams/http/http.py,sha256=0uariNq8OFnlX7iqOHwBhecxA-Hfd5hS
|
|
303
303
|
airbyte_cdk/sources/streams/http/http_client.py,sha256=tDE0ROtxjGMVphvsw8INvGMtZ97hIF-v47pZ3jIyiwc,23011
|
304
304
|
airbyte_cdk/sources/streams/http/rate_limiting.py,sha256=IwdjrHKUnU97XO4qONgYRv4YYW51xQ8SJm4WLafXDB8,6351
|
305
305
|
airbyte_cdk/sources/streams/http/requests_native_auth/__init__.py,sha256=RN0D3nOX1xLgwEwKWu6pkGy3XqBFzKSNZ8Lf6umU2eY,413
|
306
|
-
airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py,sha256=
|
306
|
+
airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py,sha256=P9U8vtcrZ3m0InSG2W0H4gTYTxjQxkIe6mhF9xvO8Ug,18824
|
307
307
|
airbyte_cdk/sources/streams/http/requests_native_auth/abstract_token.py,sha256=Y3n7J-sk5yGjv_OxtY6Z6k0PEsFZmtIRi-x0KCbaHdA,1010
|
308
308
|
airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py,sha256=C2j2uVfi9d-3KgHO3NGxIiFdfASjHOtsd6g_LWPYOAs,20311
|
309
309
|
airbyte_cdk/sources/streams/http/requests_native_auth/token.py,sha256=h5PTzcdH-RQLeCg7xZ45w_484OPUDSwNWl_iMJQmZoI,2526
|
@@ -360,9 +360,9 @@ airbyte_cdk/utils/slice_hasher.py,sha256=EDxgROHDbfG-QKQb59m7h_7crN1tRiawdf5uU7G
|
|
360
360
|
airbyte_cdk/utils/spec_schema_transformations.py,sha256=-5HTuNsnDBAhj-oLeQXwpTGA0HdcjFOf2zTEMUTTg_Y,816
|
361
361
|
airbyte_cdk/utils/stream_status_utils.py,sha256=ZmBoiy5HVbUEHAMrUONxZvxnvfV9CesmQJLDTAIWnWw,1171
|
362
362
|
airbyte_cdk/utils/traced_exception.py,sha256=C8uIBuCL_E4WnBAOPSxBicD06JAldoN9fGsQDp463OY,6292
|
363
|
-
airbyte_cdk-6.36.
|
364
|
-
airbyte_cdk-6.36.
|
365
|
-
airbyte_cdk-6.36.
|
366
|
-
airbyte_cdk-6.36.
|
367
|
-
airbyte_cdk-6.36.
|
368
|
-
airbyte_cdk-6.36.
|
363
|
+
airbyte_cdk-6.36.3.dev0.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
|
364
|
+
airbyte_cdk-6.36.3.dev0.dist-info/LICENSE_SHORT,sha256=aqF6D1NcESmpn-cqsxBtszTEnHKnlsp8L4x9wAh3Nxg,55
|
365
|
+
airbyte_cdk-6.36.3.dev0.dist-info/METADATA,sha256=IvkvzmBsCDO9JoaNx-h-185jaiJA7VWpfnKB6cgw-W0,6015
|
366
|
+
airbyte_cdk-6.36.3.dev0.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
|
367
|
+
airbyte_cdk-6.36.3.dev0.dist-info/entry_points.txt,sha256=fj-e3PAQvsxsQzyyq8UkG1k8spunWnD4BAH2AwlR6NM,95
|
368
|
+
airbyte_cdk-6.36.3.dev0.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|