airbyte-cdk 6.50.0__py3-none-any.whl → 6.51.0.post4.dev15422454178__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airbyte_cdk/cli/source_declarative_manifest/_run.py +9 -1
- airbyte_cdk/sources/declarative/concurrent_declarative_source.py +2 -0
- airbyte_cdk/sources/declarative/incremental/concurrent_partition_cursor.py +63 -54
- airbyte_cdk/sources/declarative/manifest_declarative_source.py +45 -12
- airbyte_cdk/sources/declarative/spec/spec.py +5 -32
- airbyte_cdk/sources/declarative/yaml_declarative_source.py +2 -0
- {airbyte_cdk-6.50.0.dist-info → airbyte_cdk-6.51.0.post4.dev15422454178.dist-info}/METADATA +1 -1
- {airbyte_cdk-6.50.0.dist-info → airbyte_cdk-6.51.0.post4.dev15422454178.dist-info}/RECORD +12 -12
- {airbyte_cdk-6.50.0.dist-info → airbyte_cdk-6.51.0.post4.dev15422454178.dist-info}/LICENSE.txt +0 -0
- {airbyte_cdk-6.50.0.dist-info → airbyte_cdk-6.51.0.post4.dev15422454178.dist-info}/LICENSE_SHORT +0 -0
- {airbyte_cdk-6.50.0.dist-info → airbyte_cdk-6.51.0.post4.dev15422454178.dist-info}/WHEEL +0 -0
- {airbyte_cdk-6.50.0.dist-info → airbyte_cdk-6.51.0.post4.dev15422454178.dist-info}/entry_points.txt +0 -0
@@ -58,6 +58,7 @@ class SourceLocalYaml(YamlDeclarativeSource):
|
|
58
58
|
catalog: ConfiguredAirbyteCatalog | None,
|
59
59
|
config: MutableMapping[str, Any] | None,
|
60
60
|
state: TState,
|
61
|
+
config_path: str | None = None,
|
61
62
|
**kwargs: Any,
|
62
63
|
) -> None:
|
63
64
|
"""
|
@@ -76,6 +77,7 @@ class SourceLocalYaml(YamlDeclarativeSource):
|
|
76
77
|
config=config,
|
77
78
|
state=state, # type: ignore [arg-type]
|
78
79
|
path_to_yaml="manifest.yaml",
|
80
|
+
config_path=config_path,
|
79
81
|
)
|
80
82
|
|
81
83
|
|
@@ -95,7 +97,12 @@ def _get_local_yaml_source(args: list[str]) -> SourceLocalYaml:
|
|
95
97
|
try:
|
96
98
|
parsed_args = AirbyteEntrypoint.parse_args(args)
|
97
99
|
config, catalog, state = _parse_inputs_into_config_catalog_state(parsed_args)
|
98
|
-
return SourceLocalYaml(
|
100
|
+
return SourceLocalYaml(
|
101
|
+
config=config,
|
102
|
+
catalog=catalog,
|
103
|
+
state=state,
|
104
|
+
config_path=parsed_args.config if hasattr(parsed_args, "config") else None,
|
105
|
+
)
|
99
106
|
except Exception as error:
|
100
107
|
print(
|
101
108
|
orjson.dumps(
|
@@ -204,6 +211,7 @@ def create_declarative_source(
|
|
204
211
|
catalog=catalog,
|
205
212
|
state=state,
|
206
213
|
source_config=cast(dict[str, Any], config["__injected_declarative_manifest"]),
|
214
|
+
config_path=parsed_args.config if hasattr(parsed_args, "config") else None,
|
207
215
|
)
|
208
216
|
except Exception as error:
|
209
217
|
print(
|
@@ -74,6 +74,7 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
|
|
74
74
|
debug: bool = False,
|
75
75
|
emit_connector_builder_messages: bool = False,
|
76
76
|
component_factory: Optional[ModelToComponentFactory] = None,
|
77
|
+
config_path: Optional[str] = None,
|
77
78
|
**kwargs: Any,
|
78
79
|
) -> None:
|
79
80
|
# todo: We could remove state from initialization. Now that streams are grouped during the read(), a source
|
@@ -96,6 +97,7 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
|
|
96
97
|
debug=debug,
|
97
98
|
emit_connector_builder_messages=emit_connector_builder_messages,
|
98
99
|
component_factory=component_factory,
|
100
|
+
config_path=config_path,
|
99
101
|
)
|
100
102
|
|
101
103
|
concurrency_level_from_manifest = self._source_config.get("concurrency_level")
|
@@ -9,7 +9,7 @@ import time
|
|
9
9
|
from collections import OrderedDict
|
10
10
|
from copy import deepcopy
|
11
11
|
from datetime import timedelta
|
12
|
-
from typing import Any, Callable, Iterable, Mapping, MutableMapping, Optional
|
12
|
+
from typing import Any, Callable, Iterable, List, Mapping, MutableMapping, Optional
|
13
13
|
|
14
14
|
from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager
|
15
15
|
from airbyte_cdk.sources.declarative.incremental.global_substream_cursor import (
|
@@ -66,8 +66,8 @@ class ConcurrentPerPartitionCursor(Cursor):
|
|
66
66
|
_GLOBAL_STATE_KEY = "state"
|
67
67
|
_PERPARTITION_STATE_KEY = "states"
|
68
68
|
_IS_PARTITION_DUPLICATION_LOGGED = False
|
69
|
-
|
70
|
-
|
69
|
+
_PARENT_STATE = 0
|
70
|
+
_GENERATION_SEQUENCE = 1
|
71
71
|
|
72
72
|
def __init__(
|
73
73
|
self,
|
@@ -99,19 +99,29 @@ class ConcurrentPerPartitionCursor(Cursor):
|
|
99
99
|
self._semaphore_per_partition: OrderedDict[str, threading.Semaphore] = OrderedDict()
|
100
100
|
|
101
101
|
# Parent-state tracking: store each partition’s parent state in creation order
|
102
|
-
self._partition_parent_state_map: OrderedDict[str, Mapping[str, Any]] =
|
102
|
+
self._partition_parent_state_map: OrderedDict[str, tuple[Mapping[str, Any], int]] = (
|
103
|
+
OrderedDict()
|
104
|
+
)
|
105
|
+
self._parent_state: Optional[StreamState] = None
|
106
|
+
|
107
|
+
# Tracks when the last slice for partition is emitted
|
108
|
+
self._partitions_done_generating_stream_slices: set[str] = set()
|
109
|
+
# Used to track the index of partitions that are not closed yet
|
110
|
+
self._processing_partitions_indexes: List[int] = list()
|
111
|
+
self._generated_partitions_count: int = 0
|
112
|
+
# Dictionary to map partition keys to their index
|
113
|
+
self._partition_key_to_index: dict[str, int] = {}
|
103
114
|
|
104
|
-
self._finished_partitions: set[str] = set()
|
105
115
|
self._lock = threading.Lock()
|
106
|
-
self._timer = Timer()
|
107
|
-
self._new_global_cursor: Optional[StreamState] = None
|
108
116
|
self._lookback_window: int = 0
|
109
|
-
self.
|
117
|
+
self._new_global_cursor: Optional[StreamState] = None
|
110
118
|
self._number_of_partitions: int = 0
|
111
119
|
self._use_global_cursor: bool = use_global_cursor
|
112
120
|
self._partition_serializer = PerPartitionKeySerializer()
|
121
|
+
|
113
122
|
# Track the last time a state message was emitted
|
114
123
|
self._last_emission_time: float = 0.0
|
124
|
+
self._timer = Timer()
|
115
125
|
|
116
126
|
self._set_initial_state(stream_state)
|
117
127
|
|
@@ -157,60 +167,37 @@ class ConcurrentPerPartitionCursor(Cursor):
|
|
157
167
|
self._cursor_per_partition[partition_key].close_partition(partition=partition)
|
158
168
|
cursor = self._cursor_per_partition[partition_key]
|
159
169
|
if (
|
160
|
-
partition_key in self.
|
170
|
+
partition_key in self._partitions_done_generating_stream_slices
|
161
171
|
and self._semaphore_per_partition[partition_key]._value == 0
|
162
172
|
):
|
163
173
|
self._update_global_cursor(cursor.state[self.cursor_field.cursor_field_key])
|
164
174
|
|
175
|
+
# Clean up the partition if it is fully processed
|
176
|
+
self._cleanup_if_done(partition_key)
|
177
|
+
|
165
178
|
self._check_and_update_parent_state()
|
166
179
|
|
167
180
|
self._emit_state_message()
|
168
181
|
|
169
182
|
def _check_and_update_parent_state(self) -> None:
|
170
|
-
"""
|
171
|
-
Pop the leftmost partition state from _partition_parent_state_map only if
|
172
|
-
*all partitions* up to (and including) that partition key in _semaphore_per_partition
|
173
|
-
are fully finished (i.e. in _finished_partitions and semaphore._value == 0).
|
174
|
-
Additionally, delete finished semaphores with a value of 0 to free up memory,
|
175
|
-
as they are only needed to track errors and completion status.
|
176
|
-
"""
|
177
183
|
last_closed_state = None
|
178
184
|
|
179
185
|
while self._partition_parent_state_map:
|
180
|
-
|
181
|
-
|
182
|
-
|
183
|
-
# Verify ALL partitions from the left up to earliest_key are finished
|
184
|
-
all_left_finished = True
|
185
|
-
for p_key, sem in list(
|
186
|
-
self._semaphore_per_partition.items()
|
187
|
-
): # Use list to allow modification during iteration
|
188
|
-
# If any earlier partition is still not finished, we must stop
|
189
|
-
if p_key not in self._finished_partitions or sem._value != 0:
|
190
|
-
all_left_finished = False
|
191
|
-
break
|
192
|
-
# Once we've reached earliest_key in the semaphore order, we can stop checking
|
193
|
-
if p_key == earliest_key:
|
194
|
-
break
|
195
|
-
|
196
|
-
# If the partitions up to earliest_key are not all finished, break the while-loop
|
197
|
-
if not all_left_finished:
|
198
|
-
break
|
186
|
+
earliest_key, (candidate_state, candidate_seq) = next(
|
187
|
+
iter(self._partition_parent_state_map.items())
|
188
|
+
)
|
199
189
|
|
200
|
-
#
|
201
|
-
|
202
|
-
|
190
|
+
# if any partition that started <= candidate_seq is still open, we must wait
|
191
|
+
if (
|
192
|
+
self._processing_partitions_indexes
|
193
|
+
and self._processing_partitions_indexes[0] <= candidate_seq
|
194
|
+
):
|
195
|
+
break
|
203
196
|
|
204
|
-
#
|
205
|
-
|
206
|
-
|
207
|
-
if p_key in self._finished_partitions and sem._value == 0:
|
208
|
-
del self._semaphore_per_partition[p_key]
|
209
|
-
logger.debug(f"Deleted finished semaphore for partition {p_key} with value 0")
|
210
|
-
if p_key == earliest_key:
|
211
|
-
break
|
197
|
+
# safe to pop
|
198
|
+
self._partition_parent_state_map.popitem(last=False)
|
199
|
+
last_closed_state = candidate_state
|
212
200
|
|
213
|
-
# Update _parent_state if we popped at least one partition
|
214
201
|
if last_closed_state is not None:
|
215
202
|
self._parent_state = last_closed_state
|
216
203
|
|
@@ -289,18 +276,24 @@ class ConcurrentPerPartitionCursor(Cursor):
|
|
289
276
|
if not self._IS_PARTITION_DUPLICATION_LOGGED:
|
290
277
|
logger.warning(f"Partition duplication detected for stream {self._stream_name}")
|
291
278
|
self._IS_PARTITION_DUPLICATION_LOGGED = True
|
279
|
+
return
|
292
280
|
else:
|
293
281
|
self._semaphore_per_partition[partition_key] = threading.Semaphore(0)
|
294
282
|
|
295
283
|
with self._lock:
|
284
|
+
seq = self._generated_partitions_count
|
285
|
+
self._generated_partitions_count += 1
|
286
|
+
self._processing_partitions_indexes.append(seq)
|
287
|
+
self._partition_key_to_index[partition_key] = seq
|
288
|
+
|
296
289
|
if (
|
297
290
|
len(self._partition_parent_state_map) == 0
|
298
291
|
or self._partition_parent_state_map[
|
299
292
|
next(reversed(self._partition_parent_state_map))
|
300
|
-
]
|
293
|
+
][self._PARENT_STATE]
|
301
294
|
!= parent_state
|
302
295
|
):
|
303
|
-
self._partition_parent_state_map[partition_key] = deepcopy(parent_state)
|
296
|
+
self._partition_parent_state_map[partition_key] = (deepcopy(parent_state), seq)
|
304
297
|
|
305
298
|
for cursor_slice, is_last_slice, _ in iterate_with_last_flag_and_state(
|
306
299
|
cursor.stream_slices(),
|
@@ -308,7 +301,7 @@ class ConcurrentPerPartitionCursor(Cursor):
|
|
308
301
|
):
|
309
302
|
self._semaphore_per_partition[partition_key].release()
|
310
303
|
if is_last_slice:
|
311
|
-
self.
|
304
|
+
self._partitions_done_generating_stream_slices.add(partition_key)
|
312
305
|
yield StreamSlice(
|
313
306
|
partition=partition, cursor_slice=cursor_slice, extra_fields=partition.extra_fields
|
314
307
|
)
|
@@ -338,10 +331,7 @@ class ConcurrentPerPartitionCursor(Cursor):
|
|
338
331
|
while len(self._cursor_per_partition) > self.DEFAULT_MAX_PARTITIONS_NUMBER - 1:
|
339
332
|
# Try removing finished partitions first
|
340
333
|
for partition_key in list(self._cursor_per_partition.keys()):
|
341
|
-
if partition_key in self.
|
342
|
-
partition_key not in self._semaphore_per_partition
|
343
|
-
or self._semaphore_per_partition[partition_key]._value == 0
|
344
|
-
):
|
334
|
+
if partition_key not in self._partition_key_to_index:
|
345
335
|
oldest_partition = self._cursor_per_partition.pop(
|
346
336
|
partition_key
|
347
337
|
) # Remove the oldest partition
|
@@ -474,6 +464,25 @@ class ConcurrentPerPartitionCursor(Cursor):
|
|
474
464
|
):
|
475
465
|
self._new_global_cursor = {self.cursor_field.cursor_field_key: copy.deepcopy(value)}
|
476
466
|
|
467
|
+
def _cleanup_if_done(self, partition_key: str) -> None:
|
468
|
+
"""
|
469
|
+
Free every in-memory structure that belonged to a completed partition:
|
470
|
+
cursor, semaphore, flag inside `_finished_partitions`
|
471
|
+
"""
|
472
|
+
if not (
|
473
|
+
partition_key in self._partitions_done_generating_stream_slices
|
474
|
+
and self._semaphore_per_partition[partition_key]._value == 0
|
475
|
+
):
|
476
|
+
return
|
477
|
+
|
478
|
+
self._semaphore_per_partition.pop(partition_key, None)
|
479
|
+
self._partitions_done_generating_stream_slices.discard(partition_key)
|
480
|
+
|
481
|
+
seq = self._partition_key_to_index.pop(partition_key)
|
482
|
+
self._processing_partitions_indexes.remove(seq)
|
483
|
+
|
484
|
+
logger.debug(f"Partition {partition_key} fully processed and cleaned up.")
|
485
|
+
|
477
486
|
def _to_partition_key(self, partition: Mapping[str, Any]) -> str:
|
478
487
|
return self._partition_serializer.to_partition_key(partition)
|
479
488
|
|
@@ -1,5 +1,5 @@
|
|
1
1
|
#
|
2
|
-
# Copyright (c)
|
2
|
+
# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
|
3
3
|
#
|
4
4
|
|
5
5
|
import json
|
@@ -8,13 +8,15 @@ import pkgutil
|
|
8
8
|
from copy import deepcopy
|
9
9
|
from importlib import metadata
|
10
10
|
from types import ModuleType
|
11
|
-
from typing import Any, Dict, Iterator, List, Mapping, Optional, Set
|
11
|
+
from typing import Any, Dict, Iterator, List, Mapping, MutableMapping, Optional, Set
|
12
12
|
|
13
|
+
import orjson
|
13
14
|
import yaml
|
14
15
|
from jsonschema.exceptions import ValidationError
|
15
16
|
from jsonschema.validators import validate
|
16
17
|
from packaging.version import InvalidVersion, Version
|
17
18
|
|
19
|
+
from airbyte_cdk.config_observation import create_connector_config_control_message
|
18
20
|
from airbyte_cdk.connector_builder.models import (
|
19
21
|
LogMessage as ConnectorBuilderLogMessage,
|
20
22
|
)
|
@@ -29,6 +31,7 @@ from airbyte_cdk.models import (
|
|
29
31
|
ConnectorSpecification,
|
30
32
|
FailureType,
|
31
33
|
)
|
34
|
+
from airbyte_cdk.models.airbyte_protocol_serializers import AirbyteMessageSerializer
|
32
35
|
from airbyte_cdk.sources.declarative.checks import COMPONENTS_CHECKER_TYPE_MAPPING
|
33
36
|
from airbyte_cdk.sources.declarative.checks.connection_checker import ConnectionChecker
|
34
37
|
from airbyte_cdk.sources.declarative.declarative_source import DeclarativeSource
|
@@ -57,9 +60,10 @@ from airbyte_cdk.sources.declarative.parsers.model_to_component_factory import (
|
|
57
60
|
ModelToComponentFactory,
|
58
61
|
)
|
59
62
|
from airbyte_cdk.sources.declarative.resolvers import COMPONENTS_RESOLVER_TYPE_MAPPING
|
63
|
+
from airbyte_cdk.sources.declarative.spec.spec import Spec
|
60
64
|
from airbyte_cdk.sources.message import MessageRepository
|
61
65
|
from airbyte_cdk.sources.streams.core import Stream
|
62
|
-
from airbyte_cdk.sources.types import ConnectionDefinition
|
66
|
+
from airbyte_cdk.sources.types import Config, ConnectionDefinition
|
63
67
|
from airbyte_cdk.sources.utils.slice_logger import (
|
64
68
|
AlwaysLogSliceLogger,
|
65
69
|
DebugSliceLogger,
|
@@ -99,6 +103,7 @@ class ManifestDeclarativeSource(DeclarativeSource):
|
|
99
103
|
component_factory: Optional[ModelToComponentFactory] = None,
|
100
104
|
migrate_manifest: Optional[bool] = False,
|
101
105
|
normalize_manifest: Optional[bool] = False,
|
106
|
+
config_path: Optional[str] = None,
|
102
107
|
) -> None:
|
103
108
|
"""
|
104
109
|
Args:
|
@@ -108,6 +113,7 @@ class ManifestDeclarativeSource(DeclarativeSource):
|
|
108
113
|
emit_connector_builder_messages: True if messages should be emitted to the connector builder.
|
109
114
|
component_factory: optional factory if ModelToComponentFactory's default behavior needs to be tweaked.
|
110
115
|
normalize_manifest: Optional flag to indicate if the manifest should be normalized.
|
116
|
+
config_path: Optional path to the config file.
|
111
117
|
"""
|
112
118
|
self.logger = logging.getLogger(f"airbyte.{self.name}")
|
113
119
|
self._should_normalize = normalize_manifest
|
@@ -130,7 +136,6 @@ class ManifestDeclarativeSource(DeclarativeSource):
|
|
130
136
|
self._slice_logger: SliceLogger = (
|
131
137
|
AlwaysLogSliceLogger() if emit_connector_builder_messages else DebugSliceLogger()
|
132
138
|
)
|
133
|
-
self._config = config or {}
|
134
139
|
|
135
140
|
# resolve all components in the manifest
|
136
141
|
self._source_config = self._pre_process_manifest(dict(source_config))
|
@@ -139,6 +144,12 @@ class ManifestDeclarativeSource(DeclarativeSource):
|
|
139
144
|
# apply additional post-processing to the manifest
|
140
145
|
self._post_process_manifest()
|
141
146
|
|
147
|
+
spec: Optional[Mapping[str, Any]] = self._source_config.get("spec")
|
148
|
+
self._spec_component: Optional[Spec] = (
|
149
|
+
self._constructor.create_component(SpecModel, spec, dict()) if spec else None
|
150
|
+
)
|
151
|
+
self._config = self._migrate_and_transform_config(config_path, config) or {}
|
152
|
+
|
142
153
|
@property
|
143
154
|
def resolved_manifest(self) -> Mapping[str, Any]:
|
144
155
|
"""
|
@@ -199,6 +210,30 @@ class ManifestDeclarativeSource(DeclarativeSource):
|
|
199
210
|
normalizer = ManifestNormalizer(self._source_config, self._declarative_component_schema)
|
200
211
|
self._source_config = normalizer.normalize()
|
201
212
|
|
213
|
+
def _migrate_and_transform_config(
|
214
|
+
self,
|
215
|
+
config_path: Optional[str],
|
216
|
+
config: Optional[Config],
|
217
|
+
) -> Optional[Config]:
|
218
|
+
if not config:
|
219
|
+
return None
|
220
|
+
if not self._spec_component:
|
221
|
+
return config
|
222
|
+
mutable_config = dict(config)
|
223
|
+
self._spec_component.migrate_config(mutable_config)
|
224
|
+
if mutable_config != config:
|
225
|
+
if config_path:
|
226
|
+
with open(config_path, "w") as f:
|
227
|
+
json.dump(mutable_config, f)
|
228
|
+
self.message_repository.emit_message(
|
229
|
+
create_connector_config_control_message(mutable_config)
|
230
|
+
)
|
231
|
+
# We have no mechanism for consuming the queue, so we print the messages to stdout
|
232
|
+
for message in self.message_repository.consume_queue():
|
233
|
+
print(orjson.dumps(AirbyteMessageSerializer.dump(message)).decode())
|
234
|
+
self._spec_component.transform_config(mutable_config)
|
235
|
+
return mutable_config
|
236
|
+
|
202
237
|
def _migrate_manifest(self) -> None:
|
203
238
|
"""
|
204
239
|
This method is used to migrate the manifest. It should be called after the manifest has been validated.
|
@@ -255,6 +290,9 @@ class ManifestDeclarativeSource(DeclarativeSource):
|
|
255
290
|
)
|
256
291
|
|
257
292
|
def streams(self, config: Mapping[str, Any]) -> List[Stream]:
|
293
|
+
if self._spec_component:
|
294
|
+
self._spec_component.validate_config(config)
|
295
|
+
|
258
296
|
self._emit_manifest_debug_message(
|
259
297
|
extra_args={
|
260
298
|
"source_name": self.name,
|
@@ -355,14 +393,9 @@ class ManifestDeclarativeSource(DeclarativeSource):
|
|
355
393
|
}
|
356
394
|
)
|
357
395
|
|
358
|
-
|
359
|
-
|
360
|
-
|
361
|
-
spec["type"] = "Spec"
|
362
|
-
spec_component = self._constructor.create_component(SpecModel, spec, dict())
|
363
|
-
return spec_component.generate_spec()
|
364
|
-
else:
|
365
|
-
return super().spec(logger)
|
396
|
+
return (
|
397
|
+
self._spec_component.generate_spec() if self._spec_component else super().spec(logger)
|
398
|
+
)
|
366
399
|
|
367
400
|
def check(self, logger: logging.Logger, config: Mapping[str, Any]) -> AirbyteConnectionStatus:
|
368
401
|
self._configure_logger_level(logger)
|
@@ -1,28 +1,21 @@
|
|
1
1
|
#
|
2
|
-
# Copyright (c)
|
2
|
+
# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
|
3
3
|
#
|
4
4
|
|
5
|
-
import json
|
6
5
|
from dataclasses import InitVar, dataclass, field
|
7
6
|
from typing import Any, List, Mapping, MutableMapping, Optional
|
8
7
|
|
9
|
-
import orjson
|
10
|
-
|
11
|
-
from airbyte_cdk.config_observation import create_connector_config_control_message
|
12
|
-
from airbyte_cdk.entrypoint import AirbyteEntrypoint
|
13
8
|
from airbyte_cdk.models import (
|
14
9
|
AdvancedAuth,
|
15
10
|
ConnectorSpecification,
|
16
11
|
ConnectorSpecificationSerializer,
|
17
12
|
)
|
18
|
-
from airbyte_cdk.models.airbyte_protocol_serializers import AirbyteMessageSerializer
|
19
13
|
from airbyte_cdk.sources.declarative.models.declarative_component_schema import AuthFlow
|
20
14
|
from airbyte_cdk.sources.declarative.transformations.config_transformations.config_transformation import (
|
21
15
|
ConfigTransformation,
|
22
16
|
)
|
23
17
|
from airbyte_cdk.sources.declarative.validators.validator import Validator
|
24
18
|
from airbyte_cdk.sources.message.repository import InMemoryMessageRepository, MessageRepository
|
25
|
-
from airbyte_cdk.sources.source import Source
|
26
19
|
|
27
20
|
|
28
21
|
@dataclass
|
@@ -48,7 +41,6 @@ class Spec:
|
|
48
41
|
config_migrations: List[ConfigMigration] = field(default_factory=list)
|
49
42
|
config_transformations: List[ConfigTransformation] = field(default_factory=list)
|
50
43
|
config_validations: List[Validator] = field(default_factory=list)
|
51
|
-
message_repository: MessageRepository = InMemoryMessageRepository()
|
52
44
|
|
53
45
|
def generate_spec(self) -> ConnectorSpecification:
|
54
46
|
"""
|
@@ -69,34 +61,15 @@ class Spec:
|
|
69
61
|
# We remap these keys to camel case because that's the existing format expected by the rest of the platform
|
70
62
|
return ConnectorSpecificationSerializer.load(obj)
|
71
63
|
|
72
|
-
def migrate_config(
|
73
|
-
self, args: List[str], source: Source, config: MutableMapping[str, Any]
|
74
|
-
) -> None:
|
64
|
+
def migrate_config(self, config: MutableMapping[str, Any]) -> None:
|
75
65
|
"""
|
76
|
-
Apply all specified config transformations to the provided config and
|
66
|
+
Apply all specified config transformations to the provided config and emit a control message.
|
77
67
|
|
78
|
-
:param args: Command line arguments
|
79
|
-
:param source: Source instance
|
80
68
|
:param config: The user-provided config to migrate
|
81
69
|
"""
|
82
|
-
config_path = AirbyteEntrypoint(source).extract_config(args)
|
83
|
-
|
84
|
-
if not config_path:
|
85
|
-
return
|
86
|
-
|
87
|
-
mutable_config = dict(config)
|
88
70
|
for migration in self.config_migrations:
|
89
71
|
for transformation in migration.transformations:
|
90
|
-
transformation.transform(
|
91
|
-
|
92
|
-
if mutable_config != config:
|
93
|
-
with open(config_path, "w") as f:
|
94
|
-
json.dump(mutable_config, f)
|
95
|
-
self.message_repository.emit_message(
|
96
|
-
create_connector_config_control_message(mutable_config)
|
97
|
-
)
|
98
|
-
for message in self.message_repository.consume_queue():
|
99
|
-
print(orjson.dumps(AirbyteMessageSerializer.dump(message)).decode())
|
72
|
+
transformation.transform(config)
|
100
73
|
|
101
74
|
def transform_config(self, config: MutableMapping[str, Any]) -> None:
|
102
75
|
"""
|
@@ -107,7 +80,7 @@ class Spec:
|
|
107
80
|
for transformation in self.config_transformations:
|
108
81
|
transformation.transform(config)
|
109
82
|
|
110
|
-
def validate_config(self, config:
|
83
|
+
def validate_config(self, config: Mapping[str, Any]) -> None:
|
111
84
|
"""
|
112
85
|
Apply all config validations to the provided config.
|
113
86
|
|
@@ -24,6 +24,7 @@ class YamlDeclarativeSource(ConcurrentDeclarativeSource[List[AirbyteStateMessage
|
|
24
24
|
catalog: Optional[ConfiguredAirbyteCatalog] = None,
|
25
25
|
config: Optional[Mapping[str, Any]] = None,
|
26
26
|
state: Optional[List[AirbyteStateMessage]] = None,
|
27
|
+
config_path: Optional[str] = None,
|
27
28
|
) -> None:
|
28
29
|
"""
|
29
30
|
:param path_to_yaml: Path to the yaml file describing the source
|
@@ -36,6 +37,7 @@ class YamlDeclarativeSource(ConcurrentDeclarativeSource[List[AirbyteStateMessage
|
|
36
37
|
config=config or {},
|
37
38
|
state=state or [],
|
38
39
|
source_config=source_config,
|
40
|
+
config_path=config_path,
|
39
41
|
)
|
40
42
|
|
41
43
|
def _read_and_parse_yaml_file(self, path_to_yaml_file: str) -> ConnectionDefinition:
|
@@ -8,7 +8,7 @@ airbyte_cdk/cli/airbyte_cdk/_secrets.py,sha256=jLtpkhFJHavABN3UuqddeDRGS8v1iEj0e
|
|
8
8
|
airbyte_cdk/cli/airbyte_cdk/_version.py,sha256=ohZNIktLFk91sdzqFW5idaNrZAPX2dIRnz---_fcKOE,352
|
9
9
|
airbyte_cdk/cli/airbyte_cdk/exceptions.py,sha256=bsGmlWN6cXL2jCD1WYAZMqFmK1OLg2xLrcC_60KHSeA,803
|
10
10
|
airbyte_cdk/cli/source_declarative_manifest/__init__.py,sha256=-0ST722Nj65bgRokzpzPkD1NBBW5CytEHFUe38cB86Q,91
|
11
|
-
airbyte_cdk/cli/source_declarative_manifest/_run.py,sha256=
|
11
|
+
airbyte_cdk/cli/source_declarative_manifest/_run.py,sha256=jo4yydSDrB157GtTSZPWRfhZHYvFUD_btI5LW7wOrmY,11129
|
12
12
|
airbyte_cdk/cli/source_declarative_manifest/spec.json,sha256=Earc1L6ngcdIr514oFQlUoOxdF4RHqtUyStSIAquXdY,554
|
13
13
|
airbyte_cdk/config_observation.py,sha256=7SSPxtN0nXPkm4euGNcTTr1iLbwUL01jy-24V1Hzde0,3986
|
14
14
|
airbyte_cdk/connector.py,sha256=N6TUlrZOMjLAI85JrNAKkfyTqnO5xfBCw4oEfgjJd9o,4254
|
@@ -85,7 +85,7 @@ airbyte_cdk/sources/declarative/checks/check_stream.py,sha256=QeExVmpSYjr_CnghHu
|
|
85
85
|
airbyte_cdk/sources/declarative/checks/connection_checker.py,sha256=MBRJo6WJlZQHpIfOGaNOkkHUmgUl_4wDM6VPo41z5Ss,1383
|
86
86
|
airbyte_cdk/sources/declarative/concurrency_level/__init__.py,sha256=5XUqrmlstYlMM0j6crktlKQwALek0uiz2D3WdM46MyA,191
|
87
87
|
airbyte_cdk/sources/declarative/concurrency_level/concurrency_level.py,sha256=YIwCTCpOr_QSNW4ltQK0yUGWInI8PKNY216HOOegYLk,2101
|
88
|
-
airbyte_cdk/sources/declarative/concurrent_declarative_source.py,sha256=
|
88
|
+
airbyte_cdk/sources/declarative/concurrent_declarative_source.py,sha256=OKor1mDDdJZz8kgiR0KI-_pWDjCF1nuDcto_fSYerW4,28442
|
89
89
|
airbyte_cdk/sources/declarative/datetime/__init__.py,sha256=4Hw-PX1-VgESLF16cDdvuYCzGJtHntThLF4qIiULWeo,61
|
90
90
|
airbyte_cdk/sources/declarative/datetime/datetime_parser.py,sha256=_zGNGq31RNy_0QBLt_EcTvgPyhj7urPdx6oA3M5-r3o,3150
|
91
91
|
airbyte_cdk/sources/declarative/datetime/min_max_datetime.py,sha256=0BHBtDNQZfvwM45-tY5pNlTcKAFSGGNxemoi0Jic-0E,5785
|
@@ -111,7 +111,7 @@ airbyte_cdk/sources/declarative/extractors/record_selector.py,sha256=vCpwX1PVRFP
|
|
111
111
|
airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py,sha256=WJyA2OYIEgFpVP5Y3o0tIj69AV6IKkn9B16MeXaEItI,6513
|
112
112
|
airbyte_cdk/sources/declarative/extractors/type_transformer.py,sha256=d6Y2Rfg8pMVEEnHllfVksWZdNVOU55yk34O03dP9muY,1626
|
113
113
|
airbyte_cdk/sources/declarative/incremental/__init__.py,sha256=U1oZKtBaEC6IACmvziY9Wzg7Z8EgF4ZuR7NwvjlB_Sk,1255
|
114
|
-
airbyte_cdk/sources/declarative/incremental/concurrent_partition_cursor.py,sha256=
|
114
|
+
airbyte_cdk/sources/declarative/incremental/concurrent_partition_cursor.py,sha256=vFMFp9OdYMie3tFpK_lfLsQtHhH8sQD5l6NJqj0DCX4,22593
|
115
115
|
airbyte_cdk/sources/declarative/incremental/datetime_based_cursor.py,sha256=Rbe6lJLTtZ5en33MwZiB9-H9-AwDMNHgwBZs8EqhYqk,22172
|
116
116
|
airbyte_cdk/sources/declarative/incremental/declarative_cursor.py,sha256=5Bhw9VRPyIuCaD0wmmq_L3DZsa-rJgtKSEUzSd8YYD0,536
|
117
117
|
airbyte_cdk/sources/declarative/incremental/global_substream_cursor.py,sha256=2tsE6FgXzemf4fZZ4uGtd8QpRBl9GJ2CRqSNJE5p0EI,16077
|
@@ -127,7 +127,7 @@ airbyte_cdk/sources/declarative/interpolation/interpolated_string.py,sha256=CQkH
|
|
127
127
|
airbyte_cdk/sources/declarative/interpolation/interpolation.py,sha256=9IoeuWam3L6GyN10L6U8xNWXmkt9cnahSDNkez1OmFY,982
|
128
128
|
airbyte_cdk/sources/declarative/interpolation/jinja.py,sha256=UQeuS4Vpyp4hlOn-R3tRyeBX0e9IoV6jQ6gH-Jz8lY0,7182
|
129
129
|
airbyte_cdk/sources/declarative/interpolation/macros.py,sha256=UYSJ5gW7TkHALYnNvUnRP3RlyGwGuRMObF3BHuNzjJM,5320
|
130
|
-
airbyte_cdk/sources/declarative/manifest_declarative_source.py,sha256=
|
130
|
+
airbyte_cdk/sources/declarative/manifest_declarative_source.py,sha256=AVb86oWIAxAOrE3KAyNr81bF5Dd2QjKdQ6E4Jl4yrQw,24719
|
131
131
|
airbyte_cdk/sources/declarative/migrations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
132
132
|
airbyte_cdk/sources/declarative/migrations/legacy_to_per_partition_state_migration.py,sha256=V2lpYE9LJKvz6BUViHk4vaRGndxNABmPbDCtyYdkqaE,4013
|
133
133
|
airbyte_cdk/sources/declarative/migrations/state_migration.py,sha256=KWPjealMLKSMtajXgkdGgKg7EmTLR-CqqD7UIh0-eDU,794
|
@@ -216,7 +216,7 @@ airbyte_cdk/sources/declarative/schema/inline_schema_loader.py,sha256=bVETE10hRs
|
|
216
216
|
airbyte_cdk/sources/declarative/schema/json_file_schema_loader.py,sha256=5Wl-fqW-pVf_dxJ4yGHMAFfC4JjKHYJhqFJT1xA57F4,4177
|
217
217
|
airbyte_cdk/sources/declarative/schema/schema_loader.py,sha256=kjt8v0N5wWKA5zyLnrDLxf1PJKdUqvQq2RVnAOAzNSY,379
|
218
218
|
airbyte_cdk/sources/declarative/spec/__init__.py,sha256=9FYO-fVOclrwjAW4qwRTbZRVopTc9rOaauAJfThdNCQ,177
|
219
|
-
airbyte_cdk/sources/declarative/spec/spec.py,sha256=
|
219
|
+
airbyte_cdk/sources/declarative/spec/spec.py,sha256=SwL_pfXZgcLYLJY-MAeFMHug9oYh2tOWjgG0C3DoLOY,3602
|
220
220
|
airbyte_cdk/sources/declarative/stream_slicers/__init__.py,sha256=sI9vhc95RwJYOnA0VKjcbtKgFcmAbWjhdWBXFbAijOs,176
|
221
221
|
airbyte_cdk/sources/declarative/stream_slicers/declarative_partition_generator.py,sha256=cjKGm4r438dd1GxrFHJ4aYrdzG2bkncnwaWxAwlXR3M,3585
|
222
222
|
airbyte_cdk/sources/declarative/stream_slicers/stream_slicer.py,sha256=SOkIPBi2Wu7yxIvA15yFzUAB95a3IzA8LPq5DEqHQQc,725
|
@@ -241,7 +241,7 @@ airbyte_cdk/sources/declarative/validators/predicate_validator.py,sha256=Q4eVncl
|
|
241
241
|
airbyte_cdk/sources/declarative/validators/validate_adheres_to_schema.py,sha256=kjcuKxWMJEzpF4GiESITGMxBAXw6YZCAsgOQMgeBo4g,1085
|
242
242
|
airbyte_cdk/sources/declarative/validators/validation_strategy.py,sha256=LwqUX89cFdHTM1-h6c8vebBA9WC38HYoGBvJfCZHr0g,467
|
243
243
|
airbyte_cdk/sources/declarative/validators/validator.py,sha256=MAwo8OievUsuzBuPxI9pbPu87yq0tJZkGbydcrHZyQc,382
|
244
|
-
airbyte_cdk/sources/declarative/yaml_declarative_source.py,sha256
|
244
|
+
airbyte_cdk/sources/declarative/yaml_declarative_source.py,sha256=-pHwGO7ZW-x8lmsqSpbrN0pOgIyjJhFDGUNwB3kQWWc,2794
|
245
245
|
airbyte_cdk/sources/file_based/README.md,sha256=iMqww4VZ882jfNQIdljjDgqreKs-mkdtSrRKA94iX6A,11085
|
246
246
|
airbyte_cdk/sources/file_based/__init__.py,sha256=EaxHv_9ot-eRlUCR47ZMZ0IOtB-n0HH24om7Bfn-uuQ,868
|
247
247
|
airbyte_cdk/sources/file_based/availability_strategy/__init__.py,sha256=ddKQfUmk-Ls7LJaG8gtrqDybG3d8S7KXOAEjLeYLrTg,399
|
@@ -419,9 +419,9 @@ airbyte_cdk/utils/slice_hasher.py,sha256=EDxgROHDbfG-QKQb59m7h_7crN1tRiawdf5uU7G
|
|
419
419
|
airbyte_cdk/utils/spec_schema_transformations.py,sha256=-5HTuNsnDBAhj-oLeQXwpTGA0HdcjFOf2zTEMUTTg_Y,816
|
420
420
|
airbyte_cdk/utils/stream_status_utils.py,sha256=ZmBoiy5HVbUEHAMrUONxZvxnvfV9CesmQJLDTAIWnWw,1171
|
421
421
|
airbyte_cdk/utils/traced_exception.py,sha256=C8uIBuCL_E4WnBAOPSxBicD06JAldoN9fGsQDp463OY,6292
|
422
|
-
airbyte_cdk-6.
|
423
|
-
airbyte_cdk-6.
|
424
|
-
airbyte_cdk-6.
|
425
|
-
airbyte_cdk-6.
|
426
|
-
airbyte_cdk-6.
|
427
|
-
airbyte_cdk-6.
|
422
|
+
airbyte_cdk-6.51.0.post4.dev15422454178.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
|
423
|
+
airbyte_cdk-6.51.0.post4.dev15422454178.dist-info/LICENSE_SHORT,sha256=aqF6D1NcESmpn-cqsxBtszTEnHKnlsp8L4x9wAh3Nxg,55
|
424
|
+
airbyte_cdk-6.51.0.post4.dev15422454178.dist-info/METADATA,sha256=NYMHVBAElSCbpd0xpAoFVofyWEX43t_Uwxave4R6G4E,6364
|
425
|
+
airbyte_cdk-6.51.0.post4.dev15422454178.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
|
426
|
+
airbyte_cdk-6.51.0.post4.dev15422454178.dist-info/entry_points.txt,sha256=AKWbEkHfpzzk9nF9tqBUaw1MbvTM4mGtEzmZQm0ZWvM,139
|
427
|
+
airbyte_cdk-6.51.0.post4.dev15422454178.dist-info/RECORD,,
|
{airbyte_cdk-6.50.0.dist-info → airbyte_cdk-6.51.0.post4.dev15422454178.dist-info}/LICENSE.txt
RENAMED
File without changes
|
{airbyte_cdk-6.50.0.dist-info → airbyte_cdk-6.51.0.post4.dev15422454178.dist-info}/LICENSE_SHORT
RENAMED
File without changes
|
File without changes
|
{airbyte_cdk-6.50.0.dist-info → airbyte_cdk-6.51.0.post4.dev15422454178.dist-info}/entry_points.txt
RENAMED
File without changes
|