airbyte-cdk 0.61.2__py3-none-any.whl → 0.62.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (33) hide show
  1. airbyte_cdk/sources/abstract_source.py +14 -33
  2. airbyte_cdk/sources/connector_state_manager.py +16 -4
  3. airbyte_cdk/sources/file_based/file_based_source.py +87 -35
  4. airbyte_cdk/sources/file_based/stream/abstract_file_based_stream.py +3 -0
  5. airbyte_cdk/sources/file_based/stream/concurrent/adapters.py +15 -13
  6. airbyte_cdk/sources/file_based/stream/concurrent/cursor/__init__.py +5 -0
  7. airbyte_cdk/sources/file_based/stream/concurrent/{cursor.py → cursor/abstract_concurrent_file_based_cursor.py} +22 -44
  8. airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_concurrent_cursor.py +279 -0
  9. airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_noop_cursor.py +56 -0
  10. airbyte_cdk/sources/file_based/stream/default_file_based_stream.py +11 -2
  11. airbyte_cdk/test/mock_http/mocker.py +3 -1
  12. airbyte_cdk/test/mock_http/response.py +9 -1
  13. airbyte_cdk/utils/traced_exception.py +1 -16
  14. {airbyte_cdk-0.61.2.dist-info → airbyte_cdk-0.62.1.dist-info}/METADATA +1 -1
  15. {airbyte_cdk-0.61.2.dist-info → airbyte_cdk-0.62.1.dist-info}/RECORD +33 -26
  16. unit_tests/sources/file_based/helpers.py +5 -0
  17. unit_tests/sources/file_based/scenarios/concurrent_incremental_scenarios.py +2860 -0
  18. unit_tests/sources/file_based/scenarios/incremental_scenarios.py +11 -0
  19. unit_tests/sources/file_based/scenarios/scenario_builder.py +6 -2
  20. unit_tests/sources/file_based/stream/concurrent/__init__.py +0 -0
  21. unit_tests/sources/file_based/stream/concurrent/test_adapters.py +365 -0
  22. unit_tests/sources/file_based/stream/concurrent/test_file_based_concurrent_cursor.py +462 -0
  23. unit_tests/sources/file_based/test_file_based_scenarios.py +45 -0
  24. unit_tests/sources/file_based/test_scenarios.py +16 -8
  25. unit_tests/sources/streams/concurrent/scenarios/stream_facade_builder.py +13 -2
  26. unit_tests/sources/test_abstract_source.py +36 -170
  27. unit_tests/sources/test_connector_state_manager.py +20 -13
  28. unit_tests/sources/test_integration_source.py +8 -25
  29. unit_tests/sources/test_source_read.py +1 -1
  30. unit_tests/test/mock_http/test_mocker.py +3 -1
  31. {airbyte_cdk-0.61.2.dist-info → airbyte_cdk-0.62.1.dist-info}/LICENSE.txt +0 -0
  32. {airbyte_cdk-0.61.2.dist-info → airbyte_cdk-0.62.1.dist-info}/WHEEL +0 -0
  33. {airbyte_cdk-0.61.2.dist-info → airbyte_cdk-0.62.1.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,279 @@
1
+ #
2
+ # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
3
+ #
4
+
5
+ import logging
6
+ from datetime import datetime, timedelta
7
+ from threading import RLock
8
+ from typing import TYPE_CHECKING, Any, Dict, Iterable, List, MutableMapping, Optional, Tuple
9
+
10
+ from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, Level, Type
11
+ from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager
12
+ from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileBasedStreamConfig
13
+ from airbyte_cdk.sources.file_based.remote_file import RemoteFile
14
+ from airbyte_cdk.sources.file_based.stream.concurrent.cursor.abstract_concurrent_file_based_cursor import AbstractConcurrentFileBasedCursor
15
+ from airbyte_cdk.sources.file_based.stream.cursor import DefaultFileBasedCursor
16
+ from airbyte_cdk.sources.file_based.types import StreamState
17
+ from airbyte_cdk.sources.message.repository import MessageRepository
18
+ from airbyte_cdk.sources.streams.concurrent.cursor import CursorField
19
+ from airbyte_cdk.sources.streams.concurrent.partitions.partition import Partition
20
+ from airbyte_cdk.sources.streams.concurrent.partitions.record import Record
21
+
22
+ if TYPE_CHECKING:
23
+ from airbyte_cdk.sources.file_based.stream.concurrent.adapters import FileBasedStreamPartition
24
+
25
+ _NULL_FILE = ""
26
+
27
+
28
+ class FileBasedConcurrentCursor(AbstractConcurrentFileBasedCursor):
29
+ CURSOR_FIELD = "_ab_source_file_last_modified"
30
+ DEFAULT_DAYS_TO_SYNC_IF_HISTORY_IS_FULL = DefaultFileBasedCursor.DEFAULT_DAYS_TO_SYNC_IF_HISTORY_IS_FULL
31
+ DEFAULT_MAX_HISTORY_SIZE = 10_000
32
+ DATE_TIME_FORMAT = DefaultFileBasedCursor.DATE_TIME_FORMAT
33
+ zero_value = datetime.min
34
+ zero_cursor_value = f"0001-01-01T00:00:00.000000Z_{_NULL_FILE}"
35
+
36
+ def __init__(
37
+ self,
38
+ stream_config: FileBasedStreamConfig,
39
+ stream_name: str,
40
+ stream_namespace: Optional[str],
41
+ stream_state: MutableMapping[str, Any],
42
+ message_repository: MessageRepository,
43
+ connector_state_manager: ConnectorStateManager,
44
+ cursor_field: CursorField,
45
+ ) -> None:
46
+ super().__init__()
47
+ self._stream_name = stream_name
48
+ self._stream_namespace = stream_namespace
49
+ self._state = stream_state
50
+ self._message_repository = message_repository
51
+ self._connector_state_manager = connector_state_manager
52
+ self._cursor_field = cursor_field
53
+ self._time_window_if_history_is_full = timedelta(
54
+ days=stream_config.days_to_sync_if_history_is_full or self.DEFAULT_DAYS_TO_SYNC_IF_HISTORY_IS_FULL
55
+ )
56
+ self._state_lock = RLock()
57
+ self._pending_files_lock = RLock()
58
+ self._pending_files: Optional[Dict[str, RemoteFile]] = None
59
+ self._file_to_datetime_history = stream_state.get("history", {}) if stream_state else {}
60
+ self._prev_cursor_value = self._compute_prev_sync_cursor(stream_state)
61
+ self._sync_start = self._compute_start_time()
62
+
63
+ @property
64
+ def state(self) -> MutableMapping[str, Any]:
65
+ return self._state
66
+
67
+ def observe(self, record: Record) -> None:
68
+ pass
69
+
70
+ def close_partition(self, partition: Partition) -> None:
71
+ with self._pending_files_lock:
72
+ if self._pending_files is None:
73
+ raise RuntimeError("Expected pending partitions to be set but it was not. This is unexpected. Please contact Support.")
74
+
75
+ def set_pending_partitions(self, partitions: List["FileBasedStreamPartition"]) -> None:
76
+ with self._pending_files_lock:
77
+ self._pending_files = {}
78
+ for partition in partitions:
79
+ _slice = partition.to_slice()
80
+ if _slice is None:
81
+ continue
82
+ for file in _slice["files"]:
83
+ if file.uri in self._pending_files.keys():
84
+ raise RuntimeError(f"Already found file {_slice} in pending files. This is unexpected. Please contact Support.")
85
+ self._pending_files.update({file.uri: file})
86
+
87
+ def _compute_prev_sync_cursor(self, value: Optional[StreamState]) -> Tuple[datetime, str]:
88
+ if not value:
89
+ return self.zero_value, ""
90
+ prev_cursor_str = value.get(self._cursor_field.cursor_field_key) or self.zero_cursor_value
91
+ # So if we see a cursor greater than the earliest file, it means that we have likely synced all files.
92
+ # However, we take the earliest file as the cursor value for the purpose of checking which files to
93
+ # sync, in case new files have been uploaded in the meantime.
94
+ # This should be very rare, as it would indicate a race condition where a file with an earlier
95
+ # last_modified time was uploaded after a file with a later last_modified time. Since last_modified
96
+ # represents the start time that the file was uploaded, we can usually expect that all previous
97
+ # files have already been uploaded. If that's the case, they'll be in history and we'll skip
98
+ # re-uploading them.
99
+ earliest_file_cursor_value = self._get_cursor_key_from_file(self._compute_earliest_file_in_history())
100
+ cursor_str = min(prev_cursor_str, earliest_file_cursor_value)
101
+ cursor_dt, cursor_uri = cursor_str.split("_", 1)
102
+ return datetime.strptime(cursor_dt, self.DATE_TIME_FORMAT), cursor_uri
103
+
104
+ def _get_cursor_key_from_file(self, file: Optional[RemoteFile]) -> str:
105
+ if file:
106
+ return f"{datetime.strftime(file.last_modified, self.DATE_TIME_FORMAT)}_{file.uri}"
107
+ return self.zero_cursor_value
108
+
109
+ def _compute_earliest_file_in_history(self) -> Optional[RemoteFile]:
110
+ with self._state_lock:
111
+ if self._file_to_datetime_history:
112
+ filename, last_modified = min(self._file_to_datetime_history.items(), key=lambda f: (f[1], f[0]))
113
+ return RemoteFile(uri=filename, last_modified=datetime.strptime(last_modified, self.DATE_TIME_FORMAT))
114
+ else:
115
+ return None
116
+
117
+ def add_file(self, file: RemoteFile) -> None:
118
+ """
119
+ Add a file to the cursor. This method is called when a file is processed by the stream.
120
+ :param file: The file to add
121
+ """
122
+ if self._pending_files is None:
123
+ raise RuntimeError("Expected pending partitions to be set but it was not. This is unexpected. Please contact Support.")
124
+ with self._pending_files_lock:
125
+ with self._state_lock:
126
+ if file.uri not in self._pending_files:
127
+ self._message_repository.emit_message(
128
+ AirbyteMessage(
129
+ type=Type.LOG,
130
+ log=AirbyteLogMessage(
131
+ level=Level.WARN,
132
+ message=f"The file {file.uri} was not found in the list of pending files. This is unexpected. Please contact Support",
133
+ ),
134
+ )
135
+ )
136
+ else:
137
+ self._pending_files.pop(file.uri)
138
+ self._file_to_datetime_history[file.uri] = file.last_modified.strftime(self.DATE_TIME_FORMAT)
139
+ if len(self._file_to_datetime_history) > self.DEFAULT_MAX_HISTORY_SIZE:
140
+ # Get the earliest file based on its last modified date and its uri
141
+ oldest_file = self._compute_earliest_file_in_history()
142
+ if oldest_file:
143
+ del self._file_to_datetime_history[oldest_file.uri]
144
+ else:
145
+ raise Exception(
146
+ "The history is full but there is no files in the history. This should never happen and might be indicative of a bug in the CDK."
147
+ )
148
+ self.emit_state_message()
149
+
150
+ def emit_state_message(self) -> None:
151
+ with self._state_lock:
152
+ new_state = self.get_state()
153
+ self._connector_state_manager.update_state_for_stream(
154
+ self._stream_name,
155
+ self._stream_namespace,
156
+ new_state,
157
+ )
158
+ state_message = self._connector_state_manager.create_state_message(
159
+ self._stream_name, self._stream_namespace, send_per_stream_state=True
160
+ )
161
+ self._message_repository.emit_message(state_message)
162
+
163
+ def _get_new_cursor_value(self) -> str:
164
+ with self._pending_files_lock:
165
+ with self._state_lock:
166
+ if self._pending_files:
167
+ # If there are partitions that haven't been synced, we don't know whether the files that have been synced
168
+ # represent a contiguous region.
169
+ # To avoid missing files, we only increment the cursor up to the oldest pending file, because we know
170
+ # that all older files have been synced.
171
+ return self._get_cursor_key_from_file(self._compute_earliest_pending_file())
172
+ elif self._file_to_datetime_history:
173
+ # If all partitions have been synced, we know that the sync is up-to-date and so can advance
174
+ # the cursor to the newest file in history.
175
+ return self._get_cursor_key_from_file(self._compute_latest_file_in_history())
176
+ else:
177
+ return f"{self.zero_value.strftime(self.DATE_TIME_FORMAT)}_"
178
+
179
+ def _compute_earliest_pending_file(self) -> Optional[RemoteFile]:
180
+ if self._pending_files:
181
+ return min(self._pending_files.values(), key=lambda x: x.last_modified)
182
+ else:
183
+ return None
184
+
185
+ def _compute_latest_file_in_history(self) -> Optional[RemoteFile]:
186
+ with self._state_lock:
187
+ if self._file_to_datetime_history:
188
+ filename, last_modified = max(self._file_to_datetime_history.items(), key=lambda f: (f[1], f[0]))
189
+ return RemoteFile(uri=filename, last_modified=datetime.strptime(last_modified, self.DATE_TIME_FORMAT))
190
+ else:
191
+ return None
192
+
193
+ def get_files_to_sync(self, all_files: Iterable[RemoteFile], logger: logging.Logger) -> Iterable[RemoteFile]:
194
+ """
195
+ Given the list of files in the source, return the files that should be synced.
196
+ :param all_files: All files in the source
197
+ :param logger:
198
+ :return: The files that should be synced
199
+ """
200
+ with self._state_lock:
201
+ if self._is_history_full():
202
+ logger.warning(
203
+ f"The state history is full. "
204
+ f"This sync and future syncs won't be able to use the history to filter out duplicate files. "
205
+ f"It will instead use the time window of {self._time_window_if_history_is_full} to filter out files."
206
+ )
207
+ for f in all_files:
208
+ if self._should_sync_file(f, logger):
209
+ yield f
210
+
211
+ def _should_sync_file(self, file: RemoteFile, logger: logging.Logger) -> bool:
212
+ with self._state_lock:
213
+ if file.uri in self._file_to_datetime_history:
214
+ # If the file's uri is in the history, we should sync the file if it has been modified since it was synced
215
+ updated_at_from_history = datetime.strptime(self._file_to_datetime_history[file.uri], self.DATE_TIME_FORMAT)
216
+ if file.last_modified < updated_at_from_history:
217
+ self._message_repository.emit_message(
218
+ AirbyteMessage(
219
+ type=Type.LOG,
220
+ log=AirbyteLogMessage(
221
+ level=Level.WARN,
222
+ message=f"The file {file.uri}'s last modified date is older than the last time it was synced. This is unexpected. Skipping the file.",
223
+ ),
224
+ )
225
+ )
226
+ return False
227
+ else:
228
+ return file.last_modified > updated_at_from_history
229
+
230
+ prev_cursor_timestamp, prev_cursor_uri = self._prev_cursor_value
231
+ if self._is_history_full():
232
+ if file.last_modified > prev_cursor_timestamp:
233
+ # If the history is partial and the file's datetime is strictly greater than the cursor, we should sync it
234
+ return True
235
+ elif file.last_modified == prev_cursor_timestamp:
236
+ # If the history is partial and the file's datetime is equal to the earliest file in the history,
237
+ # we should sync it if its uri is greater than or equal to the cursor value.
238
+ return file.uri > prev_cursor_uri
239
+ else:
240
+ return file.last_modified >= self._sync_start
241
+ else:
242
+ # The file is not in the history and the history is complete. We know we need to sync the file
243
+ return True
244
+
245
+ def _is_history_full(self) -> bool:
246
+ """
247
+ Returns true if the state's history is full, meaning new entries will start to replace old entries.
248
+ """
249
+ with self._state_lock:
250
+ if self._file_to_datetime_history is None:
251
+ raise RuntimeError("The history object has not been set. This is unexpected. Please contact Support.")
252
+ return len(self._file_to_datetime_history) >= self.DEFAULT_MAX_HISTORY_SIZE
253
+
254
+ def _compute_start_time(self) -> datetime:
255
+ if not self._file_to_datetime_history:
256
+ return datetime.min
257
+ else:
258
+ earliest = min(self._file_to_datetime_history.values())
259
+ earliest_dt = datetime.strptime(earliest, self.DATE_TIME_FORMAT)
260
+ if self._is_history_full():
261
+ time_window = datetime.now() - self._time_window_if_history_is_full
262
+ earliest_dt = min(earliest_dt, time_window)
263
+ return earliest_dt
264
+
265
+ def get_start_time(self) -> datetime:
266
+ return self._sync_start
267
+
268
+ def get_state(self) -> MutableMapping[str, Any]:
269
+ """
270
+ Get the state of the cursor.
271
+ """
272
+ with self._state_lock:
273
+ return {"history": self._file_to_datetime_history, self._cursor_field.cursor_field_key: self._get_new_cursor_value()}
274
+
275
+ def set_initial_state(self, value: StreamState) -> None:
276
+ pass
277
+
278
+ def ensure_at_least_one_state_emitted(self) -> None:
279
+ self.emit_state_message()
@@ -0,0 +1,56 @@
1
+ #
2
+ # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
3
+ #
4
+
5
+ import logging
6
+ from datetime import datetime
7
+ from typing import TYPE_CHECKING, Any, Iterable, List, MutableMapping
8
+
9
+ from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileBasedStreamConfig
10
+ from airbyte_cdk.sources.file_based.remote_file import RemoteFile
11
+ from airbyte_cdk.sources.file_based.stream.concurrent.cursor.abstract_concurrent_file_based_cursor import AbstractConcurrentFileBasedCursor
12
+ from airbyte_cdk.sources.file_based.types import StreamState
13
+ from airbyte_cdk.sources.streams.concurrent.partitions.partition import Partition
14
+ from airbyte_cdk.sources.streams.concurrent.partitions.record import Record
15
+
16
+ if TYPE_CHECKING:
17
+ from airbyte_cdk.sources.file_based.stream.concurrent.adapters import FileBasedStreamPartition
18
+
19
+
20
+ class FileBasedNoopCursor(AbstractConcurrentFileBasedCursor):
21
+ def __init__(self, stream_config: FileBasedStreamConfig, **kwargs: Any):
22
+ pass
23
+
24
+ @property
25
+ def state(self) -> MutableMapping[str, Any]:
26
+ return {}
27
+
28
+ def observe(self, record: Record) -> None:
29
+ pass
30
+
31
+ def close_partition(self, partition: Partition) -> None:
32
+ pass
33
+
34
+ def set_pending_partitions(self, partitions: List["FileBasedStreamPartition"]) -> None:
35
+ pass
36
+
37
+ def add_file(self, file: RemoteFile) -> None:
38
+ pass
39
+
40
+ def get_files_to_sync(self, all_files: Iterable[RemoteFile], logger: logging.Logger) -> Iterable[RemoteFile]:
41
+ return all_files
42
+
43
+ def get_state(self) -> MutableMapping[str, Any]:
44
+ return {}
45
+
46
+ def set_initial_state(self, value: StreamState) -> None:
47
+ return None
48
+
49
+ def get_start_time(self) -> datetime:
50
+ return datetime.min
51
+
52
+ def emit_state_message(self) -> None:
53
+ pass
54
+
55
+ def ensure_at_least_one_state_emitted(self) -> None:
56
+ pass
@@ -43,9 +43,8 @@ class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin):
43
43
  ab_file_name_col = "_ab_source_file_url"
44
44
  airbyte_columns = [ab_last_mod_col, ab_file_name_col]
45
45
 
46
- def __init__(self, cursor: AbstractFileBasedCursor, **kwargs: Any):
46
+ def __init__(self, **kwargs: Any):
47
47
  super().__init__(**kwargs)
48
- self._cursor = cursor
49
48
 
50
49
  @property
51
50
  def state(self) -> MutableMapping[str, Any]:
@@ -56,6 +55,16 @@ class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin):
56
55
  """State setter, accept state serialized by state getter."""
57
56
  self._cursor.set_initial_state(value)
58
57
 
58
+ @property
59
+ def cursor(self) -> Optional[AbstractFileBasedCursor]:
60
+ return self._cursor
61
+
62
+ @cursor.setter
63
+ def cursor(self, value: AbstractFileBasedCursor) -> None:
64
+ if self._cursor is not None:
65
+ raise RuntimeError(f"Cursor for stream {self.name} is already set. This is unexpected. Please contact Support.")
66
+ self._cursor = value
67
+
59
68
  @property
60
69
  def primary_key(self) -> PrimaryKeyType:
61
70
  return self.config.primary_key or self.get_parser().get_parser_defined_primary_key(self.config)
@@ -60,7 +60,9 @@ class HttpMocker(contextlib.ContextDecorator):
60
60
  getattr(self._mocker, method)(
61
61
  requests_mock.ANY,
62
62
  additional_matcher=self._matches_wrapper(matcher),
63
- response_list=[{"text": response.body, "status_code": response.status_code} for response in responses],
63
+ response_list=[
64
+ {"text": response.body, "status_code": response.status_code, "headers": response.headers} for response in responses
65
+ ],
64
66
  )
65
67
 
66
68
  def get(self, request: HttpRequest, responses: Union[HttpResponse, List[HttpResponse]]) -> None:
@@ -1,10 +1,14 @@
1
1
  # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
2
2
 
3
+ from types import MappingProxyType
4
+ from typing import Mapping
5
+
3
6
 
4
7
  class HttpResponse:
5
- def __init__(self, body: str, status_code: int = 200):
8
+ def __init__(self, body: str, status_code: int = 200, headers: Mapping[str, str] = MappingProxyType({})):
6
9
  self._body = body
7
10
  self._status_code = status_code
11
+ self._headers = headers
8
12
 
9
13
  @property
10
14
  def body(self) -> str:
@@ -13,3 +17,7 @@ class HttpResponse:
13
17
  @property
14
18
  def status_code(self) -> int:
15
19
  return self._status_code
20
+
21
+ @property
22
+ def headers(self) -> Mapping[str, str]:
23
+ return self._headers
@@ -13,7 +13,6 @@ from airbyte_cdk.models import (
13
13
  AirbyteTraceMessage,
14
14
  FailureType,
15
15
  Status,
16
- StreamDescriptor,
17
16
  TraceType,
18
17
  )
19
18
  from airbyte_cdk.models import Type as MessageType
@@ -44,7 +43,7 @@ class AirbyteTracedException(Exception):
44
43
  self._exception = exception
45
44
  super().__init__(internal_message)
46
45
 
47
- def as_airbyte_message(self, stream_descriptor: StreamDescriptor = None) -> AirbyteMessage:
46
+ def as_airbyte_message(self) -> AirbyteMessage:
48
47
  """
49
48
  Builds an AirbyteTraceMessage from the exception
50
49
  """
@@ -61,7 +60,6 @@ class AirbyteTracedException(Exception):
61
60
  internal_message=self.internal_message,
62
61
  failure_type=self.failure_type,
63
62
  stack_trace=stack_trace_str,
64
- stream_descriptor=stream_descriptor,
65
63
  ),
66
64
  )
67
65
 
@@ -90,16 +88,3 @@ class AirbyteTracedException(Exception):
90
88
  :param exc: the exception that caused the error
91
89
  """
92
90
  return cls(internal_message=str(exc), exception=exc, *args, **kwargs) # type: ignore # ignoring because of args and kwargs
93
-
94
- def as_sanitized_airbyte_message(self, stream_descriptor: StreamDescriptor = None) -> AirbyteMessage:
95
- """
96
- Builds an AirbyteTraceMessage from the exception and sanitizes any secrets from the message body
97
- """
98
- error_message = self.as_airbyte_message(stream_descriptor=stream_descriptor)
99
- if error_message.trace.error.message:
100
- error_message.trace.error.message = filter_secrets(error_message.trace.error.message)
101
- if error_message.trace.error.internal_message:
102
- error_message.trace.error.internal_message = filter_secrets(error_message.trace.error.internal_message)
103
- if error_message.trace.error.stack_trace:
104
- error_message.trace.error.stack_trace = filter_secrets(error_message.trace.error.stack_trace)
105
- return error_message
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: airbyte-cdk
3
- Version: 0.61.2
3
+ Version: 0.62.1
4
4
  Summary: A framework for writing Airbyte Connectors.
5
5
  Home-page: https://github.com/airbytehq/airbyte
6
6
  Author: Airbyte
@@ -24,9 +24,9 @@ airbyte_cdk/models/__init__.py,sha256=Kg8YHBqUsNWHlAw-u3ZGdG4dxLh7qBlHhqMRfamNCR
24
24
  airbyte_cdk/models/airbyte_protocol.py,sha256=DoJvnmGM3xMAZFTwA6_RGMiKSFqfE3ib_Ru0KJ65Ag4,100
25
25
  airbyte_cdk/models/well_known_types.py,sha256=KKfNbow2gdLoC1Z4hcXy_JR8m_acsB2ol7gQuEgjobw,117
26
26
  airbyte_cdk/sources/__init__.py,sha256=Ov7Uf03KPSZUmMZqZfUAK3tQwsdKjDQUDvTb-H0JyfA,1141
27
- airbyte_cdk/sources/abstract_source.py,sha256=Ehdh0AFBPBS8sLM7Ez7DANFsytv0tRvoNkEgr4gAV1s,15872
27
+ airbyte_cdk/sources/abstract_source.py,sha256=GSpNwbwJ0v-KvxWa0u_nWeC0r6G2fZNkpKUhXzf6YlI,14399
28
28
  airbyte_cdk/sources/config.py,sha256=PYsY7y2u3EUwxLiEb96JnuKwH_E8CuxKggsRO2ZPSRc,856
29
- airbyte_cdk/sources/connector_state_manager.py,sha256=wsmUgII398MazCTKxwLBLzeiU6Z-tMTrKX882EEy-YE,10904
29
+ airbyte_cdk/sources/connector_state_manager.py,sha256=p9iwWbb5uqRbsrHsdZBMXKmyHgLVbsOcV3QQexBFnPE,11052
30
30
  airbyte_cdk/sources/http_config.py,sha256=OBZeuyFilm6NlDlBhFQvHhTWabEvZww6OHDIlZujIS0,730
31
31
  airbyte_cdk/sources/http_logger.py,sha256=v0kkpDtA0GUOgj6_3AayrYaBrSHBqG4t3MGbrtxaNmU,1437
32
32
  airbyte_cdk/sources/source.py,sha256=dk50z8Roc28MJ8FxWe652B-GwItO__bTZqFm7WOtHnw,4412
@@ -153,7 +153,7 @@ airbyte_cdk/sources/embedded/runner.py,sha256=kZ0CcUANuMjdZ4fmvp_w9P2IcsS9WSHxNq
153
153
  airbyte_cdk/sources/embedded/tools.py,sha256=-Z4tZ4AP1OTi_zrqFM3YV8Rt7c60wvsrv0Dc-rTZ2uw,744
154
154
  airbyte_cdk/sources/file_based/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
155
155
  airbyte_cdk/sources/file_based/exceptions.py,sha256=-SjdDk-mbkp5qQVUESkn788W8NmGtC2LROkZRKS_Dxc,5613
156
- airbyte_cdk/sources/file_based/file_based_source.py,sha256=Hs9q27db2x5wnvmo7uN8NVM0EGp-7KK_R4IjR2ig3tU,11316
156
+ airbyte_cdk/sources/file_based/file_based_source.py,sha256=WH-BCGu5vWB0WjAI54Q30uHSkX9J5QgIpBwWI5Gx7ag,13645
157
157
  airbyte_cdk/sources/file_based/file_based_stream_reader.py,sha256=K9fFHcSL4E8v-X2l38wRAcZCjpyifr35orvby8vQt84,3749
158
158
  airbyte_cdk/sources/file_based/remote_file.py,sha256=dtRX7X06Fug-XDz93a5lkwPQy5nQgxH0-ZcXW2HuMGI,312
159
159
  airbyte_cdk/sources/file_based/schema_helpers.py,sha256=XBkOutIw_n6SNYU34qbyTbl0Ppt0i4k3sVFMSaX3wJo,9103
@@ -183,11 +183,14 @@ airbyte_cdk/sources/file_based/schema_validation_policies/__init__.py,sha256=sEV
183
183
  airbyte_cdk/sources/file_based/schema_validation_policies/abstract_schema_validation_policy.py,sha256=uwk6Ugf23xKG4PRPVVRVwpcNjTwPgxejl03vLSEzK0s,604
184
184
  airbyte_cdk/sources/file_based/schema_validation_policies/default_schema_validation_policies.py,sha256=ZeAa0z50ywMU2chNjQ7JpL4yePU1NajhBa8FS7rXLVo,1643
185
185
  airbyte_cdk/sources/file_based/stream/__init__.py,sha256=QPDqdgjsabOQD93dSFqHGaFS_3pIwm-chEabZHiPJi0,265
186
- airbyte_cdk/sources/file_based/stream/abstract_file_based_stream.py,sha256=GJu02B2_fcfaOnSBvhpRyXIEEtu4v8ubFR_vQpe-YAU,6405
187
- airbyte_cdk/sources/file_based/stream/default_file_based_stream.py,sha256=x9VsiWhr1LBW2AEYIJYHdMZ4V_Jv2Pimll6OcTve3S8,12801
186
+ airbyte_cdk/sources/file_based/stream/abstract_file_based_stream.py,sha256=cmO1SQt5PIQRNNoh2KBv6aeY8NEY9x2dlmiRwGwU1vg,6557
187
+ airbyte_cdk/sources/file_based/stream/default_file_based_stream.py,sha256=qS0DJzXlVew6armFDJ0eNcSxRCmkA7JWQYFl6gcv3dU,13113
188
188
  airbyte_cdk/sources/file_based/stream/concurrent/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
189
- airbyte_cdk/sources/file_based/stream/concurrent/adapters.py,sha256=YUiyt7oMOt5P6rQalS1vJcpLnBRmgqyRgbN0HJ1ppIk,12871
190
- airbyte_cdk/sources/file_based/stream/concurrent/cursor.py,sha256=X8pLsNQHpmQlInpgNpwPJAYHtcgu_Xi6u0gWN-OxMK0,2650
189
+ airbyte_cdk/sources/file_based/stream/concurrent/adapters.py,sha256=AX_H5cIWkWOJkhXGuTSuZ56Jr5szoNfQ3NdabbWPTtI,13043
190
+ airbyte_cdk/sources/file_based/stream/concurrent/cursor/__init__.py,sha256=WKEYXZwSla6xwp7k1mnyG3kl9xCzEZ9B3eE-cxIuzIM,310
191
+ airbyte_cdk/sources/file_based/stream/concurrent/cursor/abstract_concurrent_file_based_cursor.py,sha256=UYLE2A2RdV-5FaQ70naZZWY34l5AEJkIRlTH05-e_-k,1961
192
+ airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_concurrent_cursor.py,sha256=jHiej28aKQJ3UmWXQxHRCK8xkzY5H0-zxQiVqFs5rAI,14389
193
+ airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_noop_cursor.py,sha256=wblXBgNw-QLVPNOAL8DihlQBXbvPC1zCdDWMsPdZPzQ,1852
191
194
  airbyte_cdk/sources/file_based/stream/cursor/__init__.py,sha256=MhFB5hOo8sjwvCh8gangaymdg3EJWYt_72brFOZt068,191
192
195
  airbyte_cdk/sources/file_based/stream/cursor/abstract_file_based_cursor.py,sha256=i-FPeK8lwCzX34GCcmvL5Yvdh8-uu7FeCVYDoFbD7IY,1920
193
196
  airbyte_cdk/sources/file_based/stream/cursor/default_file_based_cursor.py,sha256=kuJRKgDYOGXRk0V0I8BpFxg0hGv7SfV_nBpmmn45F88,6815
@@ -250,9 +253,9 @@ airbyte_cdk/test/entrypoint_wrapper.py,sha256=uTOEYoWkYnbkooPJ4a4gZ-NEll5j1tTCAz
250
253
  airbyte_cdk/test/state_builder.py,sha256=SlKadhKVi38ZSKMeceVAxjowxsDDT9vJoG6gU4zDrQE,705
251
254
  airbyte_cdk/test/mock_http/__init__.py,sha256=uil6k-0NbUyDFZXtWw88HaS7r13i43VzA9H7hOHzZx8,322
252
255
  airbyte_cdk/test/mock_http/matcher.py,sha256=J4C8g8PkdKo4OwHWMJGYJIyrLnQpXI5gXWUtyxsxHpM,1240
253
- airbyte_cdk/test/mock_http/mocker.py,sha256=R-RvMufoI22TtZno8WpSIy0BVhtwoI9OvEGDL6HJU5w,6143
256
+ airbyte_cdk/test/mock_http/mocker.py,sha256=Sb1Nnf3bVEJfiy5_IliRcyIiIPQL8esSWmm5j9u0E_E,6202
254
257
  airbyte_cdk/test/mock_http/request.py,sha256=dd_i47FOGD5iRlU23daotv2gEn5NOVqTBAqykxdG6-0,3687
255
- airbyte_cdk/test/mock_http/response.py,sha256=XjkouKy3eCsylZRROzl1qZs6LfjnO4C3L1hcJoqTHOs,354
258
+ airbyte_cdk/test/mock_http/response.py,sha256=F09QGG8N3Z8fL_b0rmSKTYoKgku5yZJQCpj0Fwwxu3s,588
256
259
  airbyte_cdk/test/mock_http/response_builder.py,sha256=sc0lU_LN3wjBc4mFFV-3Y5IhYeapRdtB_-EDdHfyArA,7804
257
260
  airbyte_cdk/utils/__init__.py,sha256=qZoNqzEKhIXdN_ZfvXlIGnmiDDjCFy6BVCzzWjUZcuU,294
258
261
  airbyte_cdk/utils/airbyte_secrets_utils.py,sha256=q3aDl8T10ufGbeqnUPqbZLxQcHdkf2kDfQK_upWzBbI,2894
@@ -266,7 +269,7 @@ airbyte_cdk/utils/oneof_option_config.py,sha256=N8EmWdYdwt0FM7fuShh6H8nj_r4KEL9t
266
269
  airbyte_cdk/utils/schema_inferrer.py,sha256=D8vFVgeK6VLcAug4YVAHfa3D29On0A_nMlwq9SPlfPI,3799
267
270
  airbyte_cdk/utils/spec_schema_transformations.py,sha256=LGjSSk8lmBiC0GiHqxDwu_iMN6bCe05UMpz9e7nCw5E,741
268
271
  airbyte_cdk/utils/stream_status_utils.py,sha256=k7OY6AkJW8ifyh7ZYetC5Yy1nxM6Mx3apOAviCjJh80,971
269
- airbyte_cdk/utils/traced_exception.py,sha256=IDYvUkbgkOMjusiuP0xU65mHzl5nLDkhA3o-FvNDfjI,4336
272
+ airbyte_cdk/utils/traced_exception.py,sha256=ChtuhSV_fkmMv8QjPBR1dV1US8uxlmVt_Myt-C2OIqQ,3396
270
273
  source_declarative_manifest/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
271
274
  source_declarative_manifest/main.py,sha256=HXzuRsRyhHwPrGU-hc4S7RrgoOoHImqkdfbmO2geBeE,1027
272
275
  unit_tests/connector_builder/__init__.py,sha256=4Hw-PX1-VgESLF16cDdvuYCzGJtHntThLF4qIiULWeo,61
@@ -279,14 +282,14 @@ unit_tests/singer/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU
279
282
  unit_tests/singer/test_singer_helpers.py,sha256=pZV6VxJuK-3-FICNGmoGbokrA_zkaFZEd4rYZCVpSRU,1762
280
283
  unit_tests/singer/test_singer_source.py,sha256=edN_kv7dnYAdBveWdUYOs74ak0dK6p8uaX225h_ZILA,4442
281
284
  unit_tests/sources/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
282
- unit_tests/sources/test_abstract_source.py,sha256=MM8-37yidqqbz87PVllWpnCl9leDaarM71siOIKj33A,59013
285
+ unit_tests/sources/test_abstract_source.py,sha256=m-YcMK1DgIhJLKUHoANFPx_d6yh-zgLrU1wLNlNCuTg,52802
283
286
  unit_tests/sources/test_concurrent_source.py,sha256=3i7pSRetKSoP6LBpXyuXpWi2_VOwta_aTm_kgnDaLqk,3704
284
287
  unit_tests/sources/test_config.py,sha256=lxjeaf48pOMF4Pf3-Z1ux_tHTyjRFCdG_hpnxw3e7uQ,2839
285
- unit_tests/sources/test_connector_state_manager.py,sha256=ynFxA63Cxe6t-wMMh9C6ByTlMAuk8W7H2FikDhnUEQ0,24264
288
+ unit_tests/sources/test_connector_state_manager.py,sha256=KAvYmuaWwg2kSnPNKri6Ne8TmLpsSimotsnDLLKkDD0,24369
286
289
  unit_tests/sources/test_http_logger.py,sha256=VT6DqgspI3DcRnoBQkkQX0z4dF_AOiYZ5P_zxmMW8oU,9004
287
- unit_tests/sources/test_integration_source.py,sha256=qcWld9evB1rAjALWX8SDshGz7seYkN3HCamQ6KQ2Idw,4269
290
+ unit_tests/sources/test_integration_source.py,sha256=u_w5NS9n8GkTsoTjJvBE3-g8x0NG2054hL3PtW7IfAM,3458
288
291
  unit_tests/sources/test_source.py,sha256=W0I4umL_d_OToLYYiRkjkJR6e-cCYjdV8zKc3uLvF0k,27999
289
- unit_tests/sources/test_source_read.py,sha256=n9XpVQLfsQH8eh6D99MDiNVBBKcf6UtouThDJcGH6SU,17186
292
+ unit_tests/sources/test_source_read.py,sha256=AEFoJfzM0_5QQIJyKwGLK_kq_Vz_CBivImnUnXJQJ0I,17176
290
293
  unit_tests/sources/concurrent_source/__init__.py,sha256=4Hw-PX1-VgESLF16cDdvuYCzGJtHntThLF4qIiULWeo,61
291
294
  unit_tests/sources/concurrent_source/test_concurrent_source_adapter.py,sha256=zsGnMcEsBedjW8wahil6LNqniil-3NXhyZd5W-80Km0,3665
292
295
  unit_tests/sources/declarative/__init__.py,sha256=ZnqYNxHsKCgO38IwB34RQyRMXTs4GTvlRi3ImKnIioo,61
@@ -366,11 +369,11 @@ unit_tests/sources/declarative/states/__init__.py,sha256=ZnqYNxHsKCgO38IwB34RQyR
366
369
  unit_tests/sources/declarative/stream_slicers/__init__.py,sha256=ZnqYNxHsKCgO38IwB34RQyRMXTs4GTvlRi3ImKnIioo,61
367
370
  unit_tests/sources/declarative/stream_slicers/test_cartesian_product_stream_slicer.py,sha256=FADH2_qI8mD0K1NEl8--MJwSYiRtOqesv3msR-xnAlM,7825
368
371
  unit_tests/sources/file_based/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
369
- unit_tests/sources/file_based/helpers.py,sha256=MZTwaWtX0a6TPbFcUMP-EgqBunK2wpoElgApCEE1bN4,2659
372
+ unit_tests/sources/file_based/helpers.py,sha256=JSKXrPL7iSBSH7nkKde-jcylVuDohJid5Oi4YtVRPwg,2854
370
373
  unit_tests/sources/file_based/in_memory_files_source.py,sha256=1UCfRMgaovPdhkORT5k5Izj6e0ldPp802iiaffG2ghk,8550
371
- unit_tests/sources/file_based/test_file_based_scenarios.py,sha256=rQaORUdsRdWxTMMshJxAxnp3x6Bsnuirit4yjrT0Oao,11680
374
+ unit_tests/sources/file_based/test_file_based_scenarios.py,sha256=r2qP3WbDV48YyeBt5o18Ztt8r1v0-zZPjgrITOiJSVM,15356
372
375
  unit_tests/sources/file_based/test_file_based_stream_reader.py,sha256=P6yTp7tbPfREzi5SXg4SSSql5nxiRV571YdOmwb_SzY,9219
373
- unit_tests/sources/file_based/test_scenarios.py,sha256=xE5OoCcyTVMP0QFUgTiT0-qtigjlbQvqeIwFRNEWmgE,8752
376
+ unit_tests/sources/file_based/test_scenarios.py,sha256=w7ROHOY6dcbd5-tqcf1OtdBfA-QCC7n7Y0FP9zcH4bY,9317
374
377
  unit_tests/sources/file_based/test_schema_helpers.py,sha256=IYIDdLRK41RkSG_ZW2cagAt9krV4QLbkzu6r7vPx9Js,12047
375
378
  unit_tests/sources/file_based/availability_strategy/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
376
379
  unit_tests/sources/file_based/availability_strategy/test_default_file_based_availability_strategy.py,sha256=14ffoRWC4RHPrmBFZpplnAd1Uezn8neuQrIyZqvjTK0,4964
@@ -389,18 +392,22 @@ unit_tests/sources/file_based/file_types/test_unstructured_parser.py,sha256=kmVl
389
392
  unit_tests/sources/file_based/scenarios/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
390
393
  unit_tests/sources/file_based/scenarios/avro_scenarios.py,sha256=oeQUmCV7d2aTShreYc-PvVb4cWqLSsVwHfg-lcKjzPs,30554
391
394
  unit_tests/sources/file_based/scenarios/check_scenarios.py,sha256=0xkt21ASTnTAMP0RYJEsF3yMGsNN7wWOoG_tmzL9PYw,6750
395
+ unit_tests/sources/file_based/scenarios/concurrent_incremental_scenarios.py,sha256=h1k8ZZ8UODX0i6Squz5bI59ldQGqAPpxoKFkdfBjvAQ,103709
392
396
  unit_tests/sources/file_based/scenarios/csv_scenarios.py,sha256=w4yPhLQ_7flHTMiKIxRNTmv4ru6J-t2HnYxJsd_B1fg,123227
393
397
  unit_tests/sources/file_based/scenarios/file_based_source_builder.py,sha256=3gAFkguYH87v_WpV0lUttTKu7LG8a-viokDW232ecUw,4123
394
- unit_tests/sources/file_based/scenarios/incremental_scenarios.py,sha256=L5NTFW7S-NAHgCyP0QczgiZtgGtdzbyQO92iu45Z3Kc,67257
398
+ unit_tests/sources/file_based/scenarios/incremental_scenarios.py,sha256=68lmzAdan9ohO7kTuKJ9eafpJhJJSehf9GzpV-lryKI,67817
395
399
  unit_tests/sources/file_based/scenarios/jsonl_scenarios.py,sha256=vE-j-8lszkbqU_7t_1OK2EBvpHKA9dJDrmKGxesVMbY,31733
396
400
  unit_tests/sources/file_based/scenarios/parquet_scenarios.py,sha256=0DZbrb2wbaGSQ3OjD8gCH673dPqtVcLCR_LVkA_qVpA,26658
397
- unit_tests/sources/file_based/scenarios/scenario_builder.py,sha256=mqs9q-MSwAGT6m7n5hChEygLlckGoR3lpj5ACk9UTIs,9715
401
+ unit_tests/sources/file_based/scenarios/scenario_builder.py,sha256=ynywaMWNvPnJ8Mg2h3vYZPLfaOzHcSFYj7e8bmY_0gY,9894
398
402
  unit_tests/sources/file_based/scenarios/unstructured_scenarios.py,sha256=2_p15Phk2xiBgZ0OdGYrCU9eAlTT8h_SU5nk1ehUcLk,67894
399
403
  unit_tests/sources/file_based/scenarios/user_input_schema_scenarios.py,sha256=FVYbRfdj2RCLFVwUNqQKiBFMm78y6FvmTO447i3SXqY,28697
400
404
  unit_tests/sources/file_based/scenarios/validation_policy_scenarios.py,sha256=pXcLV4MTtT55qHDjuf9aYHi5K3dVX0YSpltuAQApISs,28511
401
405
  unit_tests/sources/file_based/stream/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
402
406
  unit_tests/sources/file_based/stream/test_default_file_based_cursor.py,sha256=XhtCGvgSBFyeQwgqGciPsIB1HIlWqTcXROwnxrjutHc,13109
403
407
  unit_tests/sources/file_based/stream/test_default_file_based_stream.py,sha256=1GZPMIL00KGMIaYcPPBhQ0gpkYAJ48xtxXOgEwxkg84,10263
408
+ unit_tests/sources/file_based/stream/concurrent/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
409
+ unit_tests/sources/file_based/stream/concurrent/test_adapters.py,sha256=aDtXineIKv1VfJ5xQWszgBfeomSKytrYh_FEOt9DnUc,15202
410
+ unit_tests/sources/file_based/stream/concurrent/test_file_based_concurrent_cursor.py,sha256=ovST4Hle8dEb-wtjx-x6DJ0VxCjvGlKWKKT9XL29nzs,19960
404
411
  unit_tests/sources/fixtures/__init__.py,sha256=ZnqYNxHsKCgO38IwB34RQyRMXTs4GTvlRi3ImKnIioo,61
405
412
  unit_tests/sources/fixtures/source_test_fixture.py,sha256=dvpISgio2sOp-U3bXudH_49vY4c68sO_PMs1JZTMaj0,5502
406
413
  unit_tests/sources/message/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -421,7 +428,7 @@ unit_tests/sources/streams/concurrent/test_partition_reader.py,sha256=bBX2mP62t4
421
428
  unit_tests/sources/streams/concurrent/test_thread_pool_manager.py,sha256=l0rwdDX79MRip0IKTXKGIqEZy2NptMTUTPYYQQU5yjQ,4203
422
429
  unit_tests/sources/streams/concurrent/scenarios/__init__.py,sha256=4Hw-PX1-VgESLF16cDdvuYCzGJtHntThLF4qIiULWeo,61
423
430
  unit_tests/sources/streams/concurrent/scenarios/incremental_scenarios.py,sha256=JlsJrYFgs2tg5hb1q47_YEPJ2_i7K_IMaHMNt9VWVdI,9910
424
- unit_tests/sources/streams/concurrent/scenarios/stream_facade_builder.py,sha256=dY3dpAru55V8iZ5z74AE8qxfLBwimR9av2UfTJMPsGg,5912
431
+ unit_tests/sources/streams/concurrent/scenarios/stream_facade_builder.py,sha256=HKtWlCbx81CdS8hqCs-d43JndiLL6Tp4K0Yf8VdycDg,6239
425
432
  unit_tests/sources/streams/concurrent/scenarios/stream_facade_scenarios.py,sha256=eA5Kug7AAKoNtW3gBSJWDz-Y-Udi3ulwKgAxxs8skAc,13992
426
433
  unit_tests/sources/streams/concurrent/scenarios/test_concurrent_scenarios.py,sha256=Z_4-ClsxBupmN7Pbl8lF9bkSA9wnjLtrgA9WR_8VRi8,3757
427
434
  unit_tests/sources/streams/concurrent/scenarios/thread_based_concurrent_stream_scenarios.py,sha256=Qa1z48QLKy8xOViyiqpkIEhREF4rZHqJh8FwJ8fzqiQ,13435
@@ -438,7 +445,7 @@ unit_tests/test/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
438
445
  unit_tests/test/test_entrypoint_wrapper.py,sha256=m4csYvjO2PzvZZma7K322SBBiL5D33xuv8eUMjitDXE,10839
439
446
  unit_tests/test/mock_http/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
440
447
  unit_tests/test/mock_http/test_matcher.py,sha256=dBndYzqvo3AdHRilLrqruXdPviwi91gWt-ubDsGb-yg,2327
441
- unit_tests/test/mock_http/test_mocker.py,sha256=Zp7nhuEGWLsysNFMIcNhu5GhWrNd_JQMIXOjZHYRdY4,8409
448
+ unit_tests/test/mock_http/test_mocker.py,sha256=sOoWutnrPDKB99Y3bkEyr3HFELGivwuklVJ_ii8C-ew,8523
442
449
  unit_tests/test/mock_http/test_request.py,sha256=O9ihefGNiZKpHqsGtis6BjF8VoaOULNR8zOblVqmsL4,7602
443
450
  unit_tests/test/mock_http/test_response_builder.py,sha256=IxAww4gaOxG-9MW8kEZkRzYL2mO6xe4jIsxhi40i2ow,7878
444
451
  unit_tests/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -449,8 +456,8 @@ unit_tests/utils/test_schema_inferrer.py,sha256=Z2jHBZ540wnYkylIdV_2xr75Vtwlxuyg
449
456
  unit_tests/utils/test_secret_utils.py,sha256=XKe0f1RHYii8iwE6ATmBr5JGDI1pzzrnZUGdUSMJQP4,4886
450
457
  unit_tests/utils/test_stream_status_utils.py,sha256=Xr8MZ2HWgTVIyMbywDvuYkRaUF4RZLQOT8-JjvcfR24,2970
451
458
  unit_tests/utils/test_traced_exception.py,sha256=bDFP5zMBizFenz6V2WvEZTRCKGB5ijh3DBezjbfoYIs,4198
452
- airbyte_cdk-0.61.2.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
453
- airbyte_cdk-0.61.2.dist-info/METADATA,sha256=rRpxOblfkKNJTkiGdS1S3AJSVoNm10I4DX1nZG3J_gw,11073
454
- airbyte_cdk-0.61.2.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
455
- airbyte_cdk-0.61.2.dist-info/top_level.txt,sha256=edvsDKTnE6sD2wfCUaeTfKf5gQIL6CPVMwVL2sWZzqo,51
456
- airbyte_cdk-0.61.2.dist-info/RECORD,,
459
+ airbyte_cdk-0.62.1.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
460
+ airbyte_cdk-0.62.1.dist-info/METADATA,sha256=9m_kiUnGlScq9ANU1dhvx00D_p5inrTOtJ-09MkF1vo,11073
461
+ airbyte_cdk-0.62.1.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
462
+ airbyte_cdk-0.62.1.dist-info/top_level.txt,sha256=edvsDKTnE6sD2wfCUaeTfKf5gQIL6CPVMwVL2sWZzqo,51
463
+ airbyte_cdk-0.62.1.dist-info/RECORD,,
@@ -15,6 +15,7 @@ from airbyte_cdk.sources.file_based.file_types.file_type_parser import FileTypeP
15
15
  from airbyte_cdk.sources.file_based.file_types.jsonl_parser import JsonlParser
16
16
  from airbyte_cdk.sources.file_based.remote_file import RemoteFile
17
17
  from airbyte_cdk.sources.file_based.schema_validation_policies import AbstractSchemaValidationPolicy
18
+ from airbyte_cdk.sources.file_based.stream.concurrent.cursor import FileBasedConcurrentCursor
18
19
  from airbyte_cdk.sources.file_based.stream.cursor import DefaultFileBasedCursor
19
20
  from unit_tests.sources.file_based.in_memory_files_source import InMemoryFilesStreamReader
20
21
 
@@ -61,5 +62,9 @@ class LowHistoryLimitCursor(DefaultFileBasedCursor):
61
62
  DEFAULT_MAX_HISTORY_SIZE = 3
62
63
 
63
64
 
65
+ class LowHistoryLimitConcurrentCursor(FileBasedConcurrentCursor):
66
+ DEFAULT_MAX_HISTORY_SIZE = 3
67
+
68
+
64
69
  def make_remote_files(files: List[str]) -> List[RemoteFile]:
65
70
  return [RemoteFile(uri=f, last_modified=datetime.strptime("2023-06-05T03:54:07.000Z", "%Y-%m-%dT%H:%M:%S.%fZ")) for f in files]