airbyte-cdk 0.61.2__py3-none-any.whl → 0.62.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airbyte_cdk/sources/abstract_source.py +14 -33
- airbyte_cdk/sources/connector_state_manager.py +16 -4
- airbyte_cdk/sources/file_based/file_based_source.py +87 -35
- airbyte_cdk/sources/file_based/stream/abstract_file_based_stream.py +3 -0
- airbyte_cdk/sources/file_based/stream/concurrent/adapters.py +15 -13
- airbyte_cdk/sources/file_based/stream/concurrent/cursor/__init__.py +5 -0
- airbyte_cdk/sources/file_based/stream/concurrent/{cursor.py → cursor/abstract_concurrent_file_based_cursor.py} +22 -44
- airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_concurrent_cursor.py +279 -0
- airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_noop_cursor.py +56 -0
- airbyte_cdk/sources/file_based/stream/default_file_based_stream.py +11 -2
- airbyte_cdk/test/mock_http/mocker.py +3 -1
- airbyte_cdk/test/mock_http/response.py +9 -1
- airbyte_cdk/utils/traced_exception.py +1 -16
- {airbyte_cdk-0.61.2.dist-info → airbyte_cdk-0.62.1.dist-info}/METADATA +1 -1
- {airbyte_cdk-0.61.2.dist-info → airbyte_cdk-0.62.1.dist-info}/RECORD +33 -26
- unit_tests/sources/file_based/helpers.py +5 -0
- unit_tests/sources/file_based/scenarios/concurrent_incremental_scenarios.py +2860 -0
- unit_tests/sources/file_based/scenarios/incremental_scenarios.py +11 -0
- unit_tests/sources/file_based/scenarios/scenario_builder.py +6 -2
- unit_tests/sources/file_based/stream/concurrent/__init__.py +0 -0
- unit_tests/sources/file_based/stream/concurrent/test_adapters.py +365 -0
- unit_tests/sources/file_based/stream/concurrent/test_file_based_concurrent_cursor.py +462 -0
- unit_tests/sources/file_based/test_file_based_scenarios.py +45 -0
- unit_tests/sources/file_based/test_scenarios.py +16 -8
- unit_tests/sources/streams/concurrent/scenarios/stream_facade_builder.py +13 -2
- unit_tests/sources/test_abstract_source.py +36 -170
- unit_tests/sources/test_connector_state_manager.py +20 -13
- unit_tests/sources/test_integration_source.py +8 -25
- unit_tests/sources/test_source_read.py +1 -1
- unit_tests/test/mock_http/test_mocker.py +3 -1
- {airbyte_cdk-0.61.2.dist-info → airbyte_cdk-0.62.1.dist-info}/LICENSE.txt +0 -0
- {airbyte_cdk-0.61.2.dist-info → airbyte_cdk-0.62.1.dist-info}/WHEEL +0 -0
- {airbyte_cdk-0.61.2.dist-info → airbyte_cdk-0.62.1.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,279 @@
|
|
1
|
+
#
|
2
|
+
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
3
|
+
#
|
4
|
+
|
5
|
+
import logging
|
6
|
+
from datetime import datetime, timedelta
|
7
|
+
from threading import RLock
|
8
|
+
from typing import TYPE_CHECKING, Any, Dict, Iterable, List, MutableMapping, Optional, Tuple
|
9
|
+
|
10
|
+
from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, Level, Type
|
11
|
+
from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager
|
12
|
+
from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileBasedStreamConfig
|
13
|
+
from airbyte_cdk.sources.file_based.remote_file import RemoteFile
|
14
|
+
from airbyte_cdk.sources.file_based.stream.concurrent.cursor.abstract_concurrent_file_based_cursor import AbstractConcurrentFileBasedCursor
|
15
|
+
from airbyte_cdk.sources.file_based.stream.cursor import DefaultFileBasedCursor
|
16
|
+
from airbyte_cdk.sources.file_based.types import StreamState
|
17
|
+
from airbyte_cdk.sources.message.repository import MessageRepository
|
18
|
+
from airbyte_cdk.sources.streams.concurrent.cursor import CursorField
|
19
|
+
from airbyte_cdk.sources.streams.concurrent.partitions.partition import Partition
|
20
|
+
from airbyte_cdk.sources.streams.concurrent.partitions.record import Record
|
21
|
+
|
22
|
+
if TYPE_CHECKING:
|
23
|
+
from airbyte_cdk.sources.file_based.stream.concurrent.adapters import FileBasedStreamPartition
|
24
|
+
|
25
|
+
_NULL_FILE = ""
|
26
|
+
|
27
|
+
|
28
|
+
class FileBasedConcurrentCursor(AbstractConcurrentFileBasedCursor):
|
29
|
+
CURSOR_FIELD = "_ab_source_file_last_modified"
|
30
|
+
DEFAULT_DAYS_TO_SYNC_IF_HISTORY_IS_FULL = DefaultFileBasedCursor.DEFAULT_DAYS_TO_SYNC_IF_HISTORY_IS_FULL
|
31
|
+
DEFAULT_MAX_HISTORY_SIZE = 10_000
|
32
|
+
DATE_TIME_FORMAT = DefaultFileBasedCursor.DATE_TIME_FORMAT
|
33
|
+
zero_value = datetime.min
|
34
|
+
zero_cursor_value = f"0001-01-01T00:00:00.000000Z_{_NULL_FILE}"
|
35
|
+
|
36
|
+
def __init__(
|
37
|
+
self,
|
38
|
+
stream_config: FileBasedStreamConfig,
|
39
|
+
stream_name: str,
|
40
|
+
stream_namespace: Optional[str],
|
41
|
+
stream_state: MutableMapping[str, Any],
|
42
|
+
message_repository: MessageRepository,
|
43
|
+
connector_state_manager: ConnectorStateManager,
|
44
|
+
cursor_field: CursorField,
|
45
|
+
) -> None:
|
46
|
+
super().__init__()
|
47
|
+
self._stream_name = stream_name
|
48
|
+
self._stream_namespace = stream_namespace
|
49
|
+
self._state = stream_state
|
50
|
+
self._message_repository = message_repository
|
51
|
+
self._connector_state_manager = connector_state_manager
|
52
|
+
self._cursor_field = cursor_field
|
53
|
+
self._time_window_if_history_is_full = timedelta(
|
54
|
+
days=stream_config.days_to_sync_if_history_is_full or self.DEFAULT_DAYS_TO_SYNC_IF_HISTORY_IS_FULL
|
55
|
+
)
|
56
|
+
self._state_lock = RLock()
|
57
|
+
self._pending_files_lock = RLock()
|
58
|
+
self._pending_files: Optional[Dict[str, RemoteFile]] = None
|
59
|
+
self._file_to_datetime_history = stream_state.get("history", {}) if stream_state else {}
|
60
|
+
self._prev_cursor_value = self._compute_prev_sync_cursor(stream_state)
|
61
|
+
self._sync_start = self._compute_start_time()
|
62
|
+
|
63
|
+
@property
|
64
|
+
def state(self) -> MutableMapping[str, Any]:
|
65
|
+
return self._state
|
66
|
+
|
67
|
+
def observe(self, record: Record) -> None:
|
68
|
+
pass
|
69
|
+
|
70
|
+
def close_partition(self, partition: Partition) -> None:
|
71
|
+
with self._pending_files_lock:
|
72
|
+
if self._pending_files is None:
|
73
|
+
raise RuntimeError("Expected pending partitions to be set but it was not. This is unexpected. Please contact Support.")
|
74
|
+
|
75
|
+
def set_pending_partitions(self, partitions: List["FileBasedStreamPartition"]) -> None:
|
76
|
+
with self._pending_files_lock:
|
77
|
+
self._pending_files = {}
|
78
|
+
for partition in partitions:
|
79
|
+
_slice = partition.to_slice()
|
80
|
+
if _slice is None:
|
81
|
+
continue
|
82
|
+
for file in _slice["files"]:
|
83
|
+
if file.uri in self._pending_files.keys():
|
84
|
+
raise RuntimeError(f"Already found file {_slice} in pending files. This is unexpected. Please contact Support.")
|
85
|
+
self._pending_files.update({file.uri: file})
|
86
|
+
|
87
|
+
def _compute_prev_sync_cursor(self, value: Optional[StreamState]) -> Tuple[datetime, str]:
|
88
|
+
if not value:
|
89
|
+
return self.zero_value, ""
|
90
|
+
prev_cursor_str = value.get(self._cursor_field.cursor_field_key) or self.zero_cursor_value
|
91
|
+
# So if we see a cursor greater than the earliest file, it means that we have likely synced all files.
|
92
|
+
# However, we take the earliest file as the cursor value for the purpose of checking which files to
|
93
|
+
# sync, in case new files have been uploaded in the meantime.
|
94
|
+
# This should be very rare, as it would indicate a race condition where a file with an earlier
|
95
|
+
# last_modified time was uploaded after a file with a later last_modified time. Since last_modified
|
96
|
+
# represents the start time that the file was uploaded, we can usually expect that all previous
|
97
|
+
# files have already been uploaded. If that's the case, they'll be in history and we'll skip
|
98
|
+
# re-uploading them.
|
99
|
+
earliest_file_cursor_value = self._get_cursor_key_from_file(self._compute_earliest_file_in_history())
|
100
|
+
cursor_str = min(prev_cursor_str, earliest_file_cursor_value)
|
101
|
+
cursor_dt, cursor_uri = cursor_str.split("_", 1)
|
102
|
+
return datetime.strptime(cursor_dt, self.DATE_TIME_FORMAT), cursor_uri
|
103
|
+
|
104
|
+
def _get_cursor_key_from_file(self, file: Optional[RemoteFile]) -> str:
|
105
|
+
if file:
|
106
|
+
return f"{datetime.strftime(file.last_modified, self.DATE_TIME_FORMAT)}_{file.uri}"
|
107
|
+
return self.zero_cursor_value
|
108
|
+
|
109
|
+
def _compute_earliest_file_in_history(self) -> Optional[RemoteFile]:
|
110
|
+
with self._state_lock:
|
111
|
+
if self._file_to_datetime_history:
|
112
|
+
filename, last_modified = min(self._file_to_datetime_history.items(), key=lambda f: (f[1], f[0]))
|
113
|
+
return RemoteFile(uri=filename, last_modified=datetime.strptime(last_modified, self.DATE_TIME_FORMAT))
|
114
|
+
else:
|
115
|
+
return None
|
116
|
+
|
117
|
+
def add_file(self, file: RemoteFile) -> None:
|
118
|
+
"""
|
119
|
+
Add a file to the cursor. This method is called when a file is processed by the stream.
|
120
|
+
:param file: The file to add
|
121
|
+
"""
|
122
|
+
if self._pending_files is None:
|
123
|
+
raise RuntimeError("Expected pending partitions to be set but it was not. This is unexpected. Please contact Support.")
|
124
|
+
with self._pending_files_lock:
|
125
|
+
with self._state_lock:
|
126
|
+
if file.uri not in self._pending_files:
|
127
|
+
self._message_repository.emit_message(
|
128
|
+
AirbyteMessage(
|
129
|
+
type=Type.LOG,
|
130
|
+
log=AirbyteLogMessage(
|
131
|
+
level=Level.WARN,
|
132
|
+
message=f"The file {file.uri} was not found in the list of pending files. This is unexpected. Please contact Support",
|
133
|
+
),
|
134
|
+
)
|
135
|
+
)
|
136
|
+
else:
|
137
|
+
self._pending_files.pop(file.uri)
|
138
|
+
self._file_to_datetime_history[file.uri] = file.last_modified.strftime(self.DATE_TIME_FORMAT)
|
139
|
+
if len(self._file_to_datetime_history) > self.DEFAULT_MAX_HISTORY_SIZE:
|
140
|
+
# Get the earliest file based on its last modified date and its uri
|
141
|
+
oldest_file = self._compute_earliest_file_in_history()
|
142
|
+
if oldest_file:
|
143
|
+
del self._file_to_datetime_history[oldest_file.uri]
|
144
|
+
else:
|
145
|
+
raise Exception(
|
146
|
+
"The history is full but there is no files in the history. This should never happen and might be indicative of a bug in the CDK."
|
147
|
+
)
|
148
|
+
self.emit_state_message()
|
149
|
+
|
150
|
+
def emit_state_message(self) -> None:
|
151
|
+
with self._state_lock:
|
152
|
+
new_state = self.get_state()
|
153
|
+
self._connector_state_manager.update_state_for_stream(
|
154
|
+
self._stream_name,
|
155
|
+
self._stream_namespace,
|
156
|
+
new_state,
|
157
|
+
)
|
158
|
+
state_message = self._connector_state_manager.create_state_message(
|
159
|
+
self._stream_name, self._stream_namespace, send_per_stream_state=True
|
160
|
+
)
|
161
|
+
self._message_repository.emit_message(state_message)
|
162
|
+
|
163
|
+
def _get_new_cursor_value(self) -> str:
|
164
|
+
with self._pending_files_lock:
|
165
|
+
with self._state_lock:
|
166
|
+
if self._pending_files:
|
167
|
+
# If there are partitions that haven't been synced, we don't know whether the files that have been synced
|
168
|
+
# represent a contiguous region.
|
169
|
+
# To avoid missing files, we only increment the cursor up to the oldest pending file, because we know
|
170
|
+
# that all older files have been synced.
|
171
|
+
return self._get_cursor_key_from_file(self._compute_earliest_pending_file())
|
172
|
+
elif self._file_to_datetime_history:
|
173
|
+
# If all partitions have been synced, we know that the sync is up-to-date and so can advance
|
174
|
+
# the cursor to the newest file in history.
|
175
|
+
return self._get_cursor_key_from_file(self._compute_latest_file_in_history())
|
176
|
+
else:
|
177
|
+
return f"{self.zero_value.strftime(self.DATE_TIME_FORMAT)}_"
|
178
|
+
|
179
|
+
def _compute_earliest_pending_file(self) -> Optional[RemoteFile]:
|
180
|
+
if self._pending_files:
|
181
|
+
return min(self._pending_files.values(), key=lambda x: x.last_modified)
|
182
|
+
else:
|
183
|
+
return None
|
184
|
+
|
185
|
+
def _compute_latest_file_in_history(self) -> Optional[RemoteFile]:
|
186
|
+
with self._state_lock:
|
187
|
+
if self._file_to_datetime_history:
|
188
|
+
filename, last_modified = max(self._file_to_datetime_history.items(), key=lambda f: (f[1], f[0]))
|
189
|
+
return RemoteFile(uri=filename, last_modified=datetime.strptime(last_modified, self.DATE_TIME_FORMAT))
|
190
|
+
else:
|
191
|
+
return None
|
192
|
+
|
193
|
+
def get_files_to_sync(self, all_files: Iterable[RemoteFile], logger: logging.Logger) -> Iterable[RemoteFile]:
|
194
|
+
"""
|
195
|
+
Given the list of files in the source, return the files that should be synced.
|
196
|
+
:param all_files: All files in the source
|
197
|
+
:param logger:
|
198
|
+
:return: The files that should be synced
|
199
|
+
"""
|
200
|
+
with self._state_lock:
|
201
|
+
if self._is_history_full():
|
202
|
+
logger.warning(
|
203
|
+
f"The state history is full. "
|
204
|
+
f"This sync and future syncs won't be able to use the history to filter out duplicate files. "
|
205
|
+
f"It will instead use the time window of {self._time_window_if_history_is_full} to filter out files."
|
206
|
+
)
|
207
|
+
for f in all_files:
|
208
|
+
if self._should_sync_file(f, logger):
|
209
|
+
yield f
|
210
|
+
|
211
|
+
def _should_sync_file(self, file: RemoteFile, logger: logging.Logger) -> bool:
|
212
|
+
with self._state_lock:
|
213
|
+
if file.uri in self._file_to_datetime_history:
|
214
|
+
# If the file's uri is in the history, we should sync the file if it has been modified since it was synced
|
215
|
+
updated_at_from_history = datetime.strptime(self._file_to_datetime_history[file.uri], self.DATE_TIME_FORMAT)
|
216
|
+
if file.last_modified < updated_at_from_history:
|
217
|
+
self._message_repository.emit_message(
|
218
|
+
AirbyteMessage(
|
219
|
+
type=Type.LOG,
|
220
|
+
log=AirbyteLogMessage(
|
221
|
+
level=Level.WARN,
|
222
|
+
message=f"The file {file.uri}'s last modified date is older than the last time it was synced. This is unexpected. Skipping the file.",
|
223
|
+
),
|
224
|
+
)
|
225
|
+
)
|
226
|
+
return False
|
227
|
+
else:
|
228
|
+
return file.last_modified > updated_at_from_history
|
229
|
+
|
230
|
+
prev_cursor_timestamp, prev_cursor_uri = self._prev_cursor_value
|
231
|
+
if self._is_history_full():
|
232
|
+
if file.last_modified > prev_cursor_timestamp:
|
233
|
+
# If the history is partial and the file's datetime is strictly greater than the cursor, we should sync it
|
234
|
+
return True
|
235
|
+
elif file.last_modified == prev_cursor_timestamp:
|
236
|
+
# If the history is partial and the file's datetime is equal to the earliest file in the history,
|
237
|
+
# we should sync it if its uri is greater than or equal to the cursor value.
|
238
|
+
return file.uri > prev_cursor_uri
|
239
|
+
else:
|
240
|
+
return file.last_modified >= self._sync_start
|
241
|
+
else:
|
242
|
+
# The file is not in the history and the history is complete. We know we need to sync the file
|
243
|
+
return True
|
244
|
+
|
245
|
+
def _is_history_full(self) -> bool:
|
246
|
+
"""
|
247
|
+
Returns true if the state's history is full, meaning new entries will start to replace old entries.
|
248
|
+
"""
|
249
|
+
with self._state_lock:
|
250
|
+
if self._file_to_datetime_history is None:
|
251
|
+
raise RuntimeError("The history object has not been set. This is unexpected. Please contact Support.")
|
252
|
+
return len(self._file_to_datetime_history) >= self.DEFAULT_MAX_HISTORY_SIZE
|
253
|
+
|
254
|
+
def _compute_start_time(self) -> datetime:
|
255
|
+
if not self._file_to_datetime_history:
|
256
|
+
return datetime.min
|
257
|
+
else:
|
258
|
+
earliest = min(self._file_to_datetime_history.values())
|
259
|
+
earliest_dt = datetime.strptime(earliest, self.DATE_TIME_FORMAT)
|
260
|
+
if self._is_history_full():
|
261
|
+
time_window = datetime.now() - self._time_window_if_history_is_full
|
262
|
+
earliest_dt = min(earliest_dt, time_window)
|
263
|
+
return earliest_dt
|
264
|
+
|
265
|
+
def get_start_time(self) -> datetime:
|
266
|
+
return self._sync_start
|
267
|
+
|
268
|
+
def get_state(self) -> MutableMapping[str, Any]:
|
269
|
+
"""
|
270
|
+
Get the state of the cursor.
|
271
|
+
"""
|
272
|
+
with self._state_lock:
|
273
|
+
return {"history": self._file_to_datetime_history, self._cursor_field.cursor_field_key: self._get_new_cursor_value()}
|
274
|
+
|
275
|
+
def set_initial_state(self, value: StreamState) -> None:
|
276
|
+
pass
|
277
|
+
|
278
|
+
def ensure_at_least_one_state_emitted(self) -> None:
|
279
|
+
self.emit_state_message()
|
@@ -0,0 +1,56 @@
|
|
1
|
+
#
|
2
|
+
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
3
|
+
#
|
4
|
+
|
5
|
+
import logging
|
6
|
+
from datetime import datetime
|
7
|
+
from typing import TYPE_CHECKING, Any, Iterable, List, MutableMapping
|
8
|
+
|
9
|
+
from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileBasedStreamConfig
|
10
|
+
from airbyte_cdk.sources.file_based.remote_file import RemoteFile
|
11
|
+
from airbyte_cdk.sources.file_based.stream.concurrent.cursor.abstract_concurrent_file_based_cursor import AbstractConcurrentFileBasedCursor
|
12
|
+
from airbyte_cdk.sources.file_based.types import StreamState
|
13
|
+
from airbyte_cdk.sources.streams.concurrent.partitions.partition import Partition
|
14
|
+
from airbyte_cdk.sources.streams.concurrent.partitions.record import Record
|
15
|
+
|
16
|
+
if TYPE_CHECKING:
|
17
|
+
from airbyte_cdk.sources.file_based.stream.concurrent.adapters import FileBasedStreamPartition
|
18
|
+
|
19
|
+
|
20
|
+
class FileBasedNoopCursor(AbstractConcurrentFileBasedCursor):
|
21
|
+
def __init__(self, stream_config: FileBasedStreamConfig, **kwargs: Any):
|
22
|
+
pass
|
23
|
+
|
24
|
+
@property
|
25
|
+
def state(self) -> MutableMapping[str, Any]:
|
26
|
+
return {}
|
27
|
+
|
28
|
+
def observe(self, record: Record) -> None:
|
29
|
+
pass
|
30
|
+
|
31
|
+
def close_partition(self, partition: Partition) -> None:
|
32
|
+
pass
|
33
|
+
|
34
|
+
def set_pending_partitions(self, partitions: List["FileBasedStreamPartition"]) -> None:
|
35
|
+
pass
|
36
|
+
|
37
|
+
def add_file(self, file: RemoteFile) -> None:
|
38
|
+
pass
|
39
|
+
|
40
|
+
def get_files_to_sync(self, all_files: Iterable[RemoteFile], logger: logging.Logger) -> Iterable[RemoteFile]:
|
41
|
+
return all_files
|
42
|
+
|
43
|
+
def get_state(self) -> MutableMapping[str, Any]:
|
44
|
+
return {}
|
45
|
+
|
46
|
+
def set_initial_state(self, value: StreamState) -> None:
|
47
|
+
return None
|
48
|
+
|
49
|
+
def get_start_time(self) -> datetime:
|
50
|
+
return datetime.min
|
51
|
+
|
52
|
+
def emit_state_message(self) -> None:
|
53
|
+
pass
|
54
|
+
|
55
|
+
def ensure_at_least_one_state_emitted(self) -> None:
|
56
|
+
pass
|
@@ -43,9 +43,8 @@ class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin):
|
|
43
43
|
ab_file_name_col = "_ab_source_file_url"
|
44
44
|
airbyte_columns = [ab_last_mod_col, ab_file_name_col]
|
45
45
|
|
46
|
-
def __init__(self,
|
46
|
+
def __init__(self, **kwargs: Any):
|
47
47
|
super().__init__(**kwargs)
|
48
|
-
self._cursor = cursor
|
49
48
|
|
50
49
|
@property
|
51
50
|
def state(self) -> MutableMapping[str, Any]:
|
@@ -56,6 +55,16 @@ class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin):
|
|
56
55
|
"""State setter, accept state serialized by state getter."""
|
57
56
|
self._cursor.set_initial_state(value)
|
58
57
|
|
58
|
+
@property
|
59
|
+
def cursor(self) -> Optional[AbstractFileBasedCursor]:
|
60
|
+
return self._cursor
|
61
|
+
|
62
|
+
@cursor.setter
|
63
|
+
def cursor(self, value: AbstractFileBasedCursor) -> None:
|
64
|
+
if self._cursor is not None:
|
65
|
+
raise RuntimeError(f"Cursor for stream {self.name} is already set. This is unexpected. Please contact Support.")
|
66
|
+
self._cursor = value
|
67
|
+
|
59
68
|
@property
|
60
69
|
def primary_key(self) -> PrimaryKeyType:
|
61
70
|
return self.config.primary_key or self.get_parser().get_parser_defined_primary_key(self.config)
|
@@ -60,7 +60,9 @@ class HttpMocker(contextlib.ContextDecorator):
|
|
60
60
|
getattr(self._mocker, method)(
|
61
61
|
requests_mock.ANY,
|
62
62
|
additional_matcher=self._matches_wrapper(matcher),
|
63
|
-
response_list=[
|
63
|
+
response_list=[
|
64
|
+
{"text": response.body, "status_code": response.status_code, "headers": response.headers} for response in responses
|
65
|
+
],
|
64
66
|
)
|
65
67
|
|
66
68
|
def get(self, request: HttpRequest, responses: Union[HttpResponse, List[HttpResponse]]) -> None:
|
@@ -1,10 +1,14 @@
|
|
1
1
|
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
2
2
|
|
3
|
+
from types import MappingProxyType
|
4
|
+
from typing import Mapping
|
5
|
+
|
3
6
|
|
4
7
|
class HttpResponse:
|
5
|
-
def __init__(self, body: str, status_code: int = 200):
|
8
|
+
def __init__(self, body: str, status_code: int = 200, headers: Mapping[str, str] = MappingProxyType({})):
|
6
9
|
self._body = body
|
7
10
|
self._status_code = status_code
|
11
|
+
self._headers = headers
|
8
12
|
|
9
13
|
@property
|
10
14
|
def body(self) -> str:
|
@@ -13,3 +17,7 @@ class HttpResponse:
|
|
13
17
|
@property
|
14
18
|
def status_code(self) -> int:
|
15
19
|
return self._status_code
|
20
|
+
|
21
|
+
@property
|
22
|
+
def headers(self) -> Mapping[str, str]:
|
23
|
+
return self._headers
|
@@ -13,7 +13,6 @@ from airbyte_cdk.models import (
|
|
13
13
|
AirbyteTraceMessage,
|
14
14
|
FailureType,
|
15
15
|
Status,
|
16
|
-
StreamDescriptor,
|
17
16
|
TraceType,
|
18
17
|
)
|
19
18
|
from airbyte_cdk.models import Type as MessageType
|
@@ -44,7 +43,7 @@ class AirbyteTracedException(Exception):
|
|
44
43
|
self._exception = exception
|
45
44
|
super().__init__(internal_message)
|
46
45
|
|
47
|
-
def as_airbyte_message(self
|
46
|
+
def as_airbyte_message(self) -> AirbyteMessage:
|
48
47
|
"""
|
49
48
|
Builds an AirbyteTraceMessage from the exception
|
50
49
|
"""
|
@@ -61,7 +60,6 @@ class AirbyteTracedException(Exception):
|
|
61
60
|
internal_message=self.internal_message,
|
62
61
|
failure_type=self.failure_type,
|
63
62
|
stack_trace=stack_trace_str,
|
64
|
-
stream_descriptor=stream_descriptor,
|
65
63
|
),
|
66
64
|
)
|
67
65
|
|
@@ -90,16 +88,3 @@ class AirbyteTracedException(Exception):
|
|
90
88
|
:param exc: the exception that caused the error
|
91
89
|
"""
|
92
90
|
return cls(internal_message=str(exc), exception=exc, *args, **kwargs) # type: ignore # ignoring because of args and kwargs
|
93
|
-
|
94
|
-
def as_sanitized_airbyte_message(self, stream_descriptor: StreamDescriptor = None) -> AirbyteMessage:
|
95
|
-
"""
|
96
|
-
Builds an AirbyteTraceMessage from the exception and sanitizes any secrets from the message body
|
97
|
-
"""
|
98
|
-
error_message = self.as_airbyte_message(stream_descriptor=stream_descriptor)
|
99
|
-
if error_message.trace.error.message:
|
100
|
-
error_message.trace.error.message = filter_secrets(error_message.trace.error.message)
|
101
|
-
if error_message.trace.error.internal_message:
|
102
|
-
error_message.trace.error.internal_message = filter_secrets(error_message.trace.error.internal_message)
|
103
|
-
if error_message.trace.error.stack_trace:
|
104
|
-
error_message.trace.error.stack_trace = filter_secrets(error_message.trace.error.stack_trace)
|
105
|
-
return error_message
|
@@ -24,9 +24,9 @@ airbyte_cdk/models/__init__.py,sha256=Kg8YHBqUsNWHlAw-u3ZGdG4dxLh7qBlHhqMRfamNCR
|
|
24
24
|
airbyte_cdk/models/airbyte_protocol.py,sha256=DoJvnmGM3xMAZFTwA6_RGMiKSFqfE3ib_Ru0KJ65Ag4,100
|
25
25
|
airbyte_cdk/models/well_known_types.py,sha256=KKfNbow2gdLoC1Z4hcXy_JR8m_acsB2ol7gQuEgjobw,117
|
26
26
|
airbyte_cdk/sources/__init__.py,sha256=Ov7Uf03KPSZUmMZqZfUAK3tQwsdKjDQUDvTb-H0JyfA,1141
|
27
|
-
airbyte_cdk/sources/abstract_source.py,sha256=
|
27
|
+
airbyte_cdk/sources/abstract_source.py,sha256=GSpNwbwJ0v-KvxWa0u_nWeC0r6G2fZNkpKUhXzf6YlI,14399
|
28
28
|
airbyte_cdk/sources/config.py,sha256=PYsY7y2u3EUwxLiEb96JnuKwH_E8CuxKggsRO2ZPSRc,856
|
29
|
-
airbyte_cdk/sources/connector_state_manager.py,sha256=
|
29
|
+
airbyte_cdk/sources/connector_state_manager.py,sha256=p9iwWbb5uqRbsrHsdZBMXKmyHgLVbsOcV3QQexBFnPE,11052
|
30
30
|
airbyte_cdk/sources/http_config.py,sha256=OBZeuyFilm6NlDlBhFQvHhTWabEvZww6OHDIlZujIS0,730
|
31
31
|
airbyte_cdk/sources/http_logger.py,sha256=v0kkpDtA0GUOgj6_3AayrYaBrSHBqG4t3MGbrtxaNmU,1437
|
32
32
|
airbyte_cdk/sources/source.py,sha256=dk50z8Roc28MJ8FxWe652B-GwItO__bTZqFm7WOtHnw,4412
|
@@ -153,7 +153,7 @@ airbyte_cdk/sources/embedded/runner.py,sha256=kZ0CcUANuMjdZ4fmvp_w9P2IcsS9WSHxNq
|
|
153
153
|
airbyte_cdk/sources/embedded/tools.py,sha256=-Z4tZ4AP1OTi_zrqFM3YV8Rt7c60wvsrv0Dc-rTZ2uw,744
|
154
154
|
airbyte_cdk/sources/file_based/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
155
155
|
airbyte_cdk/sources/file_based/exceptions.py,sha256=-SjdDk-mbkp5qQVUESkn788W8NmGtC2LROkZRKS_Dxc,5613
|
156
|
-
airbyte_cdk/sources/file_based/file_based_source.py,sha256=
|
156
|
+
airbyte_cdk/sources/file_based/file_based_source.py,sha256=WH-BCGu5vWB0WjAI54Q30uHSkX9J5QgIpBwWI5Gx7ag,13645
|
157
157
|
airbyte_cdk/sources/file_based/file_based_stream_reader.py,sha256=K9fFHcSL4E8v-X2l38wRAcZCjpyifr35orvby8vQt84,3749
|
158
158
|
airbyte_cdk/sources/file_based/remote_file.py,sha256=dtRX7X06Fug-XDz93a5lkwPQy5nQgxH0-ZcXW2HuMGI,312
|
159
159
|
airbyte_cdk/sources/file_based/schema_helpers.py,sha256=XBkOutIw_n6SNYU34qbyTbl0Ppt0i4k3sVFMSaX3wJo,9103
|
@@ -183,11 +183,14 @@ airbyte_cdk/sources/file_based/schema_validation_policies/__init__.py,sha256=sEV
|
|
183
183
|
airbyte_cdk/sources/file_based/schema_validation_policies/abstract_schema_validation_policy.py,sha256=uwk6Ugf23xKG4PRPVVRVwpcNjTwPgxejl03vLSEzK0s,604
|
184
184
|
airbyte_cdk/sources/file_based/schema_validation_policies/default_schema_validation_policies.py,sha256=ZeAa0z50ywMU2chNjQ7JpL4yePU1NajhBa8FS7rXLVo,1643
|
185
185
|
airbyte_cdk/sources/file_based/stream/__init__.py,sha256=QPDqdgjsabOQD93dSFqHGaFS_3pIwm-chEabZHiPJi0,265
|
186
|
-
airbyte_cdk/sources/file_based/stream/abstract_file_based_stream.py,sha256=
|
187
|
-
airbyte_cdk/sources/file_based/stream/default_file_based_stream.py,sha256=
|
186
|
+
airbyte_cdk/sources/file_based/stream/abstract_file_based_stream.py,sha256=cmO1SQt5PIQRNNoh2KBv6aeY8NEY9x2dlmiRwGwU1vg,6557
|
187
|
+
airbyte_cdk/sources/file_based/stream/default_file_based_stream.py,sha256=qS0DJzXlVew6armFDJ0eNcSxRCmkA7JWQYFl6gcv3dU,13113
|
188
188
|
airbyte_cdk/sources/file_based/stream/concurrent/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
189
|
-
airbyte_cdk/sources/file_based/stream/concurrent/adapters.py,sha256=
|
190
|
-
airbyte_cdk/sources/file_based/stream/concurrent/cursor.py,sha256=
|
189
|
+
airbyte_cdk/sources/file_based/stream/concurrent/adapters.py,sha256=AX_H5cIWkWOJkhXGuTSuZ56Jr5szoNfQ3NdabbWPTtI,13043
|
190
|
+
airbyte_cdk/sources/file_based/stream/concurrent/cursor/__init__.py,sha256=WKEYXZwSla6xwp7k1mnyG3kl9xCzEZ9B3eE-cxIuzIM,310
|
191
|
+
airbyte_cdk/sources/file_based/stream/concurrent/cursor/abstract_concurrent_file_based_cursor.py,sha256=UYLE2A2RdV-5FaQ70naZZWY34l5AEJkIRlTH05-e_-k,1961
|
192
|
+
airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_concurrent_cursor.py,sha256=jHiej28aKQJ3UmWXQxHRCK8xkzY5H0-zxQiVqFs5rAI,14389
|
193
|
+
airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_noop_cursor.py,sha256=wblXBgNw-QLVPNOAL8DihlQBXbvPC1zCdDWMsPdZPzQ,1852
|
191
194
|
airbyte_cdk/sources/file_based/stream/cursor/__init__.py,sha256=MhFB5hOo8sjwvCh8gangaymdg3EJWYt_72brFOZt068,191
|
192
195
|
airbyte_cdk/sources/file_based/stream/cursor/abstract_file_based_cursor.py,sha256=i-FPeK8lwCzX34GCcmvL5Yvdh8-uu7FeCVYDoFbD7IY,1920
|
193
196
|
airbyte_cdk/sources/file_based/stream/cursor/default_file_based_cursor.py,sha256=kuJRKgDYOGXRk0V0I8BpFxg0hGv7SfV_nBpmmn45F88,6815
|
@@ -250,9 +253,9 @@ airbyte_cdk/test/entrypoint_wrapper.py,sha256=uTOEYoWkYnbkooPJ4a4gZ-NEll5j1tTCAz
|
|
250
253
|
airbyte_cdk/test/state_builder.py,sha256=SlKadhKVi38ZSKMeceVAxjowxsDDT9vJoG6gU4zDrQE,705
|
251
254
|
airbyte_cdk/test/mock_http/__init__.py,sha256=uil6k-0NbUyDFZXtWw88HaS7r13i43VzA9H7hOHzZx8,322
|
252
255
|
airbyte_cdk/test/mock_http/matcher.py,sha256=J4C8g8PkdKo4OwHWMJGYJIyrLnQpXI5gXWUtyxsxHpM,1240
|
253
|
-
airbyte_cdk/test/mock_http/mocker.py,sha256=
|
256
|
+
airbyte_cdk/test/mock_http/mocker.py,sha256=Sb1Nnf3bVEJfiy5_IliRcyIiIPQL8esSWmm5j9u0E_E,6202
|
254
257
|
airbyte_cdk/test/mock_http/request.py,sha256=dd_i47FOGD5iRlU23daotv2gEn5NOVqTBAqykxdG6-0,3687
|
255
|
-
airbyte_cdk/test/mock_http/response.py,sha256=
|
258
|
+
airbyte_cdk/test/mock_http/response.py,sha256=F09QGG8N3Z8fL_b0rmSKTYoKgku5yZJQCpj0Fwwxu3s,588
|
256
259
|
airbyte_cdk/test/mock_http/response_builder.py,sha256=sc0lU_LN3wjBc4mFFV-3Y5IhYeapRdtB_-EDdHfyArA,7804
|
257
260
|
airbyte_cdk/utils/__init__.py,sha256=qZoNqzEKhIXdN_ZfvXlIGnmiDDjCFy6BVCzzWjUZcuU,294
|
258
261
|
airbyte_cdk/utils/airbyte_secrets_utils.py,sha256=q3aDl8T10ufGbeqnUPqbZLxQcHdkf2kDfQK_upWzBbI,2894
|
@@ -266,7 +269,7 @@ airbyte_cdk/utils/oneof_option_config.py,sha256=N8EmWdYdwt0FM7fuShh6H8nj_r4KEL9t
|
|
266
269
|
airbyte_cdk/utils/schema_inferrer.py,sha256=D8vFVgeK6VLcAug4YVAHfa3D29On0A_nMlwq9SPlfPI,3799
|
267
270
|
airbyte_cdk/utils/spec_schema_transformations.py,sha256=LGjSSk8lmBiC0GiHqxDwu_iMN6bCe05UMpz9e7nCw5E,741
|
268
271
|
airbyte_cdk/utils/stream_status_utils.py,sha256=k7OY6AkJW8ifyh7ZYetC5Yy1nxM6Mx3apOAviCjJh80,971
|
269
|
-
airbyte_cdk/utils/traced_exception.py,sha256=
|
272
|
+
airbyte_cdk/utils/traced_exception.py,sha256=ChtuhSV_fkmMv8QjPBR1dV1US8uxlmVt_Myt-C2OIqQ,3396
|
270
273
|
source_declarative_manifest/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
271
274
|
source_declarative_manifest/main.py,sha256=HXzuRsRyhHwPrGU-hc4S7RrgoOoHImqkdfbmO2geBeE,1027
|
272
275
|
unit_tests/connector_builder/__init__.py,sha256=4Hw-PX1-VgESLF16cDdvuYCzGJtHntThLF4qIiULWeo,61
|
@@ -279,14 +282,14 @@ unit_tests/singer/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU
|
|
279
282
|
unit_tests/singer/test_singer_helpers.py,sha256=pZV6VxJuK-3-FICNGmoGbokrA_zkaFZEd4rYZCVpSRU,1762
|
280
283
|
unit_tests/singer/test_singer_source.py,sha256=edN_kv7dnYAdBveWdUYOs74ak0dK6p8uaX225h_ZILA,4442
|
281
284
|
unit_tests/sources/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
282
|
-
unit_tests/sources/test_abstract_source.py,sha256=
|
285
|
+
unit_tests/sources/test_abstract_source.py,sha256=m-YcMK1DgIhJLKUHoANFPx_d6yh-zgLrU1wLNlNCuTg,52802
|
283
286
|
unit_tests/sources/test_concurrent_source.py,sha256=3i7pSRetKSoP6LBpXyuXpWi2_VOwta_aTm_kgnDaLqk,3704
|
284
287
|
unit_tests/sources/test_config.py,sha256=lxjeaf48pOMF4Pf3-Z1ux_tHTyjRFCdG_hpnxw3e7uQ,2839
|
285
|
-
unit_tests/sources/test_connector_state_manager.py,sha256=
|
288
|
+
unit_tests/sources/test_connector_state_manager.py,sha256=KAvYmuaWwg2kSnPNKri6Ne8TmLpsSimotsnDLLKkDD0,24369
|
286
289
|
unit_tests/sources/test_http_logger.py,sha256=VT6DqgspI3DcRnoBQkkQX0z4dF_AOiYZ5P_zxmMW8oU,9004
|
287
|
-
unit_tests/sources/test_integration_source.py,sha256=
|
290
|
+
unit_tests/sources/test_integration_source.py,sha256=u_w5NS9n8GkTsoTjJvBE3-g8x0NG2054hL3PtW7IfAM,3458
|
288
291
|
unit_tests/sources/test_source.py,sha256=W0I4umL_d_OToLYYiRkjkJR6e-cCYjdV8zKc3uLvF0k,27999
|
289
|
-
unit_tests/sources/test_source_read.py,sha256=
|
292
|
+
unit_tests/sources/test_source_read.py,sha256=AEFoJfzM0_5QQIJyKwGLK_kq_Vz_CBivImnUnXJQJ0I,17176
|
290
293
|
unit_tests/sources/concurrent_source/__init__.py,sha256=4Hw-PX1-VgESLF16cDdvuYCzGJtHntThLF4qIiULWeo,61
|
291
294
|
unit_tests/sources/concurrent_source/test_concurrent_source_adapter.py,sha256=zsGnMcEsBedjW8wahil6LNqniil-3NXhyZd5W-80Km0,3665
|
292
295
|
unit_tests/sources/declarative/__init__.py,sha256=ZnqYNxHsKCgO38IwB34RQyRMXTs4GTvlRi3ImKnIioo,61
|
@@ -366,11 +369,11 @@ unit_tests/sources/declarative/states/__init__.py,sha256=ZnqYNxHsKCgO38IwB34RQyR
|
|
366
369
|
unit_tests/sources/declarative/stream_slicers/__init__.py,sha256=ZnqYNxHsKCgO38IwB34RQyRMXTs4GTvlRi3ImKnIioo,61
|
367
370
|
unit_tests/sources/declarative/stream_slicers/test_cartesian_product_stream_slicer.py,sha256=FADH2_qI8mD0K1NEl8--MJwSYiRtOqesv3msR-xnAlM,7825
|
368
371
|
unit_tests/sources/file_based/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
369
|
-
unit_tests/sources/file_based/helpers.py,sha256=
|
372
|
+
unit_tests/sources/file_based/helpers.py,sha256=JSKXrPL7iSBSH7nkKde-jcylVuDohJid5Oi4YtVRPwg,2854
|
370
373
|
unit_tests/sources/file_based/in_memory_files_source.py,sha256=1UCfRMgaovPdhkORT5k5Izj6e0ldPp802iiaffG2ghk,8550
|
371
|
-
unit_tests/sources/file_based/test_file_based_scenarios.py,sha256=
|
374
|
+
unit_tests/sources/file_based/test_file_based_scenarios.py,sha256=r2qP3WbDV48YyeBt5o18Ztt8r1v0-zZPjgrITOiJSVM,15356
|
372
375
|
unit_tests/sources/file_based/test_file_based_stream_reader.py,sha256=P6yTp7tbPfREzi5SXg4SSSql5nxiRV571YdOmwb_SzY,9219
|
373
|
-
unit_tests/sources/file_based/test_scenarios.py,sha256=
|
376
|
+
unit_tests/sources/file_based/test_scenarios.py,sha256=w7ROHOY6dcbd5-tqcf1OtdBfA-QCC7n7Y0FP9zcH4bY,9317
|
374
377
|
unit_tests/sources/file_based/test_schema_helpers.py,sha256=IYIDdLRK41RkSG_ZW2cagAt9krV4QLbkzu6r7vPx9Js,12047
|
375
378
|
unit_tests/sources/file_based/availability_strategy/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
376
379
|
unit_tests/sources/file_based/availability_strategy/test_default_file_based_availability_strategy.py,sha256=14ffoRWC4RHPrmBFZpplnAd1Uezn8neuQrIyZqvjTK0,4964
|
@@ -389,18 +392,22 @@ unit_tests/sources/file_based/file_types/test_unstructured_parser.py,sha256=kmVl
|
|
389
392
|
unit_tests/sources/file_based/scenarios/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
390
393
|
unit_tests/sources/file_based/scenarios/avro_scenarios.py,sha256=oeQUmCV7d2aTShreYc-PvVb4cWqLSsVwHfg-lcKjzPs,30554
|
391
394
|
unit_tests/sources/file_based/scenarios/check_scenarios.py,sha256=0xkt21ASTnTAMP0RYJEsF3yMGsNN7wWOoG_tmzL9PYw,6750
|
395
|
+
unit_tests/sources/file_based/scenarios/concurrent_incremental_scenarios.py,sha256=h1k8ZZ8UODX0i6Squz5bI59ldQGqAPpxoKFkdfBjvAQ,103709
|
392
396
|
unit_tests/sources/file_based/scenarios/csv_scenarios.py,sha256=w4yPhLQ_7flHTMiKIxRNTmv4ru6J-t2HnYxJsd_B1fg,123227
|
393
397
|
unit_tests/sources/file_based/scenarios/file_based_source_builder.py,sha256=3gAFkguYH87v_WpV0lUttTKu7LG8a-viokDW232ecUw,4123
|
394
|
-
unit_tests/sources/file_based/scenarios/incremental_scenarios.py,sha256=
|
398
|
+
unit_tests/sources/file_based/scenarios/incremental_scenarios.py,sha256=68lmzAdan9ohO7kTuKJ9eafpJhJJSehf9GzpV-lryKI,67817
|
395
399
|
unit_tests/sources/file_based/scenarios/jsonl_scenarios.py,sha256=vE-j-8lszkbqU_7t_1OK2EBvpHKA9dJDrmKGxesVMbY,31733
|
396
400
|
unit_tests/sources/file_based/scenarios/parquet_scenarios.py,sha256=0DZbrb2wbaGSQ3OjD8gCH673dPqtVcLCR_LVkA_qVpA,26658
|
397
|
-
unit_tests/sources/file_based/scenarios/scenario_builder.py,sha256=
|
401
|
+
unit_tests/sources/file_based/scenarios/scenario_builder.py,sha256=ynywaMWNvPnJ8Mg2h3vYZPLfaOzHcSFYj7e8bmY_0gY,9894
|
398
402
|
unit_tests/sources/file_based/scenarios/unstructured_scenarios.py,sha256=2_p15Phk2xiBgZ0OdGYrCU9eAlTT8h_SU5nk1ehUcLk,67894
|
399
403
|
unit_tests/sources/file_based/scenarios/user_input_schema_scenarios.py,sha256=FVYbRfdj2RCLFVwUNqQKiBFMm78y6FvmTO447i3SXqY,28697
|
400
404
|
unit_tests/sources/file_based/scenarios/validation_policy_scenarios.py,sha256=pXcLV4MTtT55qHDjuf9aYHi5K3dVX0YSpltuAQApISs,28511
|
401
405
|
unit_tests/sources/file_based/stream/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
402
406
|
unit_tests/sources/file_based/stream/test_default_file_based_cursor.py,sha256=XhtCGvgSBFyeQwgqGciPsIB1HIlWqTcXROwnxrjutHc,13109
|
403
407
|
unit_tests/sources/file_based/stream/test_default_file_based_stream.py,sha256=1GZPMIL00KGMIaYcPPBhQ0gpkYAJ48xtxXOgEwxkg84,10263
|
408
|
+
unit_tests/sources/file_based/stream/concurrent/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
409
|
+
unit_tests/sources/file_based/stream/concurrent/test_adapters.py,sha256=aDtXineIKv1VfJ5xQWszgBfeomSKytrYh_FEOt9DnUc,15202
|
410
|
+
unit_tests/sources/file_based/stream/concurrent/test_file_based_concurrent_cursor.py,sha256=ovST4Hle8dEb-wtjx-x6DJ0VxCjvGlKWKKT9XL29nzs,19960
|
404
411
|
unit_tests/sources/fixtures/__init__.py,sha256=ZnqYNxHsKCgO38IwB34RQyRMXTs4GTvlRi3ImKnIioo,61
|
405
412
|
unit_tests/sources/fixtures/source_test_fixture.py,sha256=dvpISgio2sOp-U3bXudH_49vY4c68sO_PMs1JZTMaj0,5502
|
406
413
|
unit_tests/sources/message/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
@@ -421,7 +428,7 @@ unit_tests/sources/streams/concurrent/test_partition_reader.py,sha256=bBX2mP62t4
|
|
421
428
|
unit_tests/sources/streams/concurrent/test_thread_pool_manager.py,sha256=l0rwdDX79MRip0IKTXKGIqEZy2NptMTUTPYYQQU5yjQ,4203
|
422
429
|
unit_tests/sources/streams/concurrent/scenarios/__init__.py,sha256=4Hw-PX1-VgESLF16cDdvuYCzGJtHntThLF4qIiULWeo,61
|
423
430
|
unit_tests/sources/streams/concurrent/scenarios/incremental_scenarios.py,sha256=JlsJrYFgs2tg5hb1q47_YEPJ2_i7K_IMaHMNt9VWVdI,9910
|
424
|
-
unit_tests/sources/streams/concurrent/scenarios/stream_facade_builder.py,sha256=
|
431
|
+
unit_tests/sources/streams/concurrent/scenarios/stream_facade_builder.py,sha256=HKtWlCbx81CdS8hqCs-d43JndiLL6Tp4K0Yf8VdycDg,6239
|
425
432
|
unit_tests/sources/streams/concurrent/scenarios/stream_facade_scenarios.py,sha256=eA5Kug7AAKoNtW3gBSJWDz-Y-Udi3ulwKgAxxs8skAc,13992
|
426
433
|
unit_tests/sources/streams/concurrent/scenarios/test_concurrent_scenarios.py,sha256=Z_4-ClsxBupmN7Pbl8lF9bkSA9wnjLtrgA9WR_8VRi8,3757
|
427
434
|
unit_tests/sources/streams/concurrent/scenarios/thread_based_concurrent_stream_scenarios.py,sha256=Qa1z48QLKy8xOViyiqpkIEhREF4rZHqJh8FwJ8fzqiQ,13435
|
@@ -438,7 +445,7 @@ unit_tests/test/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
438
445
|
unit_tests/test/test_entrypoint_wrapper.py,sha256=m4csYvjO2PzvZZma7K322SBBiL5D33xuv8eUMjitDXE,10839
|
439
446
|
unit_tests/test/mock_http/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
440
447
|
unit_tests/test/mock_http/test_matcher.py,sha256=dBndYzqvo3AdHRilLrqruXdPviwi91gWt-ubDsGb-yg,2327
|
441
|
-
unit_tests/test/mock_http/test_mocker.py,sha256=
|
448
|
+
unit_tests/test/mock_http/test_mocker.py,sha256=sOoWutnrPDKB99Y3bkEyr3HFELGivwuklVJ_ii8C-ew,8523
|
442
449
|
unit_tests/test/mock_http/test_request.py,sha256=O9ihefGNiZKpHqsGtis6BjF8VoaOULNR8zOblVqmsL4,7602
|
443
450
|
unit_tests/test/mock_http/test_response_builder.py,sha256=IxAww4gaOxG-9MW8kEZkRzYL2mO6xe4jIsxhi40i2ow,7878
|
444
451
|
unit_tests/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
@@ -449,8 +456,8 @@ unit_tests/utils/test_schema_inferrer.py,sha256=Z2jHBZ540wnYkylIdV_2xr75Vtwlxuyg
|
|
449
456
|
unit_tests/utils/test_secret_utils.py,sha256=XKe0f1RHYii8iwE6ATmBr5JGDI1pzzrnZUGdUSMJQP4,4886
|
450
457
|
unit_tests/utils/test_stream_status_utils.py,sha256=Xr8MZ2HWgTVIyMbywDvuYkRaUF4RZLQOT8-JjvcfR24,2970
|
451
458
|
unit_tests/utils/test_traced_exception.py,sha256=bDFP5zMBizFenz6V2WvEZTRCKGB5ijh3DBezjbfoYIs,4198
|
452
|
-
airbyte_cdk-0.
|
453
|
-
airbyte_cdk-0.
|
454
|
-
airbyte_cdk-0.
|
455
|
-
airbyte_cdk-0.
|
456
|
-
airbyte_cdk-0.
|
459
|
+
airbyte_cdk-0.62.1.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
|
460
|
+
airbyte_cdk-0.62.1.dist-info/METADATA,sha256=9m_kiUnGlScq9ANU1dhvx00D_p5inrTOtJ-09MkF1vo,11073
|
461
|
+
airbyte_cdk-0.62.1.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
|
462
|
+
airbyte_cdk-0.62.1.dist-info/top_level.txt,sha256=edvsDKTnE6sD2wfCUaeTfKf5gQIL6CPVMwVL2sWZzqo,51
|
463
|
+
airbyte_cdk-0.62.1.dist-info/RECORD,,
|
@@ -15,6 +15,7 @@ from airbyte_cdk.sources.file_based.file_types.file_type_parser import FileTypeP
|
|
15
15
|
from airbyte_cdk.sources.file_based.file_types.jsonl_parser import JsonlParser
|
16
16
|
from airbyte_cdk.sources.file_based.remote_file import RemoteFile
|
17
17
|
from airbyte_cdk.sources.file_based.schema_validation_policies import AbstractSchemaValidationPolicy
|
18
|
+
from airbyte_cdk.sources.file_based.stream.concurrent.cursor import FileBasedConcurrentCursor
|
18
19
|
from airbyte_cdk.sources.file_based.stream.cursor import DefaultFileBasedCursor
|
19
20
|
from unit_tests.sources.file_based.in_memory_files_source import InMemoryFilesStreamReader
|
20
21
|
|
@@ -61,5 +62,9 @@ class LowHistoryLimitCursor(DefaultFileBasedCursor):
|
|
61
62
|
DEFAULT_MAX_HISTORY_SIZE = 3
|
62
63
|
|
63
64
|
|
65
|
+
class LowHistoryLimitConcurrentCursor(FileBasedConcurrentCursor):
|
66
|
+
DEFAULT_MAX_HISTORY_SIZE = 3
|
67
|
+
|
68
|
+
|
64
69
|
def make_remote_files(files: List[str]) -> List[RemoteFile]:
|
65
70
|
return [RemoteFile(uri=f, last_modified=datetime.strptime("2023-06-05T03:54:07.000Z", "%Y-%m-%dT%H:%M:%S.%fZ")) for f in files]
|