airbyte-cdk 6.50.0__py3-none-any.whl → 6.52.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airbyte_cdk/cli/source_declarative_manifest/_run.py +9 -1
- airbyte_cdk/sources/declarative/concurrent_declarative_source.py +2 -0
- airbyte_cdk/sources/declarative/declarative_component_schema.yaml +16 -1
- airbyte_cdk/sources/declarative/manifest_declarative_source.py +46 -15
- airbyte_cdk/sources/declarative/models/declarative_component_schema.py +11 -1
- airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +12 -3
- airbyte_cdk/sources/declarative/resolvers/components_resolver.py +2 -0
- airbyte_cdk/sources/declarative/resolvers/config_components_resolver.py +65 -13
- airbyte_cdk/sources/declarative/spec/spec.py +5 -32
- airbyte_cdk/sources/declarative/yaml_declarative_source.py +2 -0
- {airbyte_cdk-6.50.0.dist-info → airbyte_cdk-6.52.0.dist-info}/METADATA +1 -1
- {airbyte_cdk-6.50.0.dist-info → airbyte_cdk-6.52.0.dist-info}/RECORD +16 -16
- {airbyte_cdk-6.50.0.dist-info → airbyte_cdk-6.52.0.dist-info}/LICENSE.txt +0 -0
- {airbyte_cdk-6.50.0.dist-info → airbyte_cdk-6.52.0.dist-info}/LICENSE_SHORT +0 -0
- {airbyte_cdk-6.50.0.dist-info → airbyte_cdk-6.52.0.dist-info}/WHEEL +0 -0
- {airbyte_cdk-6.50.0.dist-info → airbyte_cdk-6.52.0.dist-info}/entry_points.txt +0 -0
@@ -58,6 +58,7 @@ class SourceLocalYaml(YamlDeclarativeSource):
|
|
58
58
|
catalog: ConfiguredAirbyteCatalog | None,
|
59
59
|
config: MutableMapping[str, Any] | None,
|
60
60
|
state: TState,
|
61
|
+
config_path: str | None = None,
|
61
62
|
**kwargs: Any,
|
62
63
|
) -> None:
|
63
64
|
"""
|
@@ -76,6 +77,7 @@ class SourceLocalYaml(YamlDeclarativeSource):
|
|
76
77
|
config=config,
|
77
78
|
state=state, # type: ignore [arg-type]
|
78
79
|
path_to_yaml="manifest.yaml",
|
80
|
+
config_path=config_path,
|
79
81
|
)
|
80
82
|
|
81
83
|
|
@@ -95,7 +97,12 @@ def _get_local_yaml_source(args: list[str]) -> SourceLocalYaml:
|
|
95
97
|
try:
|
96
98
|
parsed_args = AirbyteEntrypoint.parse_args(args)
|
97
99
|
config, catalog, state = _parse_inputs_into_config_catalog_state(parsed_args)
|
98
|
-
return SourceLocalYaml(
|
100
|
+
return SourceLocalYaml(
|
101
|
+
config=config,
|
102
|
+
catalog=catalog,
|
103
|
+
state=state,
|
104
|
+
config_path=parsed_args.config if hasattr(parsed_args, "config") else None,
|
105
|
+
)
|
99
106
|
except Exception as error:
|
100
107
|
print(
|
101
108
|
orjson.dumps(
|
@@ -204,6 +211,7 @@ def create_declarative_source(
|
|
204
211
|
catalog=catalog,
|
205
212
|
state=state,
|
206
213
|
source_config=cast(dict[str, Any], config["__injected_declarative_manifest"]),
|
214
|
+
config_path=parsed_args.config if hasattr(parsed_args, "config") else None,
|
207
215
|
)
|
208
216
|
except Exception as error:
|
209
217
|
print(
|
@@ -74,6 +74,7 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
|
|
74
74
|
debug: bool = False,
|
75
75
|
emit_connector_builder_messages: bool = False,
|
76
76
|
component_factory: Optional[ModelToComponentFactory] = None,
|
77
|
+
config_path: Optional[str] = None,
|
77
78
|
**kwargs: Any,
|
78
79
|
) -> None:
|
79
80
|
# todo: We could remove state from initialization. Now that streams are grouped during the read(), a source
|
@@ -96,6 +97,7 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
|
|
96
97
|
debug=debug,
|
97
98
|
emit_connector_builder_messages=emit_connector_builder_messages,
|
98
99
|
component_factory=component_factory,
|
100
|
+
config_path=config_path,
|
99
101
|
)
|
100
102
|
|
101
103
|
concurrency_level_from_manifest = self._source_config.get("concurrency_level")
|
@@ -4055,6 +4055,11 @@ definitions:
|
|
4055
4055
|
title: Value Type
|
4056
4056
|
description: The expected data type of the value. If omitted, the type will be inferred from the value provided.
|
4057
4057
|
"$ref": "#/definitions/ValueType"
|
4058
|
+
create_or_update:
|
4059
|
+
title: Create or Update
|
4060
|
+
description: Determines whether to create a new path if it doesn't exist (true) or only update existing paths (false). When set to true, the resolver will create new paths in the stream template if they don't exist. When false (default), it will only update existing paths.
|
4061
|
+
type: boolean
|
4062
|
+
default: false
|
4058
4063
|
$parameters:
|
4059
4064
|
type: object
|
4060
4065
|
additionalProperties: true
|
@@ -4106,6 +4111,12 @@ definitions:
|
|
4106
4111
|
- ["data"]
|
4107
4112
|
- ["data", "streams"]
|
4108
4113
|
- ["data", "{{ parameters.name }}"]
|
4114
|
+
default_values:
|
4115
|
+
title: Default Values
|
4116
|
+
description: A list of default values, each matching the structure expected from the parsed component value.
|
4117
|
+
type: array
|
4118
|
+
items:
|
4119
|
+
type: object
|
4109
4120
|
$parameters:
|
4110
4121
|
type: object
|
4111
4122
|
additionalProperties: true
|
@@ -4117,7 +4128,11 @@ definitions:
|
|
4117
4128
|
type: string
|
4118
4129
|
enum: [ConfigComponentsResolver]
|
4119
4130
|
stream_config:
|
4120
|
-
|
4131
|
+
anyOf:
|
4132
|
+
- type: array
|
4133
|
+
items:
|
4134
|
+
"$ref": "#/definitions/StreamConfig"
|
4135
|
+
- "$ref": "#/definitions/StreamConfig"
|
4121
4136
|
components_mapping:
|
4122
4137
|
type: array
|
4123
4138
|
items:
|
@@ -1,5 +1,5 @@
|
|
1
1
|
#
|
2
|
-
# Copyright (c)
|
2
|
+
# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
|
3
3
|
#
|
4
4
|
|
5
5
|
import json
|
@@ -8,13 +8,15 @@ import pkgutil
|
|
8
8
|
from copy import deepcopy
|
9
9
|
from importlib import metadata
|
10
10
|
from types import ModuleType
|
11
|
-
from typing import Any, Dict, Iterator, List, Mapping, Optional, Set
|
11
|
+
from typing import Any, Dict, Iterator, List, Mapping, MutableMapping, Optional, Set
|
12
12
|
|
13
|
+
import orjson
|
13
14
|
import yaml
|
14
15
|
from jsonschema.exceptions import ValidationError
|
15
16
|
from jsonschema.validators import validate
|
16
17
|
from packaging.version import InvalidVersion, Version
|
17
18
|
|
19
|
+
from airbyte_cdk.config_observation import create_connector_config_control_message
|
18
20
|
from airbyte_cdk.connector_builder.models import (
|
19
21
|
LogMessage as ConnectorBuilderLogMessage,
|
20
22
|
)
|
@@ -29,6 +31,7 @@ from airbyte_cdk.models import (
|
|
29
31
|
ConnectorSpecification,
|
30
32
|
FailureType,
|
31
33
|
)
|
34
|
+
from airbyte_cdk.models.airbyte_protocol_serializers import AirbyteMessageSerializer
|
32
35
|
from airbyte_cdk.sources.declarative.checks import COMPONENTS_CHECKER_TYPE_MAPPING
|
33
36
|
from airbyte_cdk.sources.declarative.checks.connection_checker import ConnectionChecker
|
34
37
|
from airbyte_cdk.sources.declarative.declarative_source import DeclarativeSource
|
@@ -57,9 +60,10 @@ from airbyte_cdk.sources.declarative.parsers.model_to_component_factory import (
|
|
57
60
|
ModelToComponentFactory,
|
58
61
|
)
|
59
62
|
from airbyte_cdk.sources.declarative.resolvers import COMPONENTS_RESOLVER_TYPE_MAPPING
|
63
|
+
from airbyte_cdk.sources.declarative.spec.spec import Spec
|
60
64
|
from airbyte_cdk.sources.message import MessageRepository
|
61
65
|
from airbyte_cdk.sources.streams.core import Stream
|
62
|
-
from airbyte_cdk.sources.types import ConnectionDefinition
|
66
|
+
from airbyte_cdk.sources.types import Config, ConnectionDefinition
|
63
67
|
from airbyte_cdk.sources.utils.slice_logger import (
|
64
68
|
AlwaysLogSliceLogger,
|
65
69
|
DebugSliceLogger,
|
@@ -99,6 +103,7 @@ class ManifestDeclarativeSource(DeclarativeSource):
|
|
99
103
|
component_factory: Optional[ModelToComponentFactory] = None,
|
100
104
|
migrate_manifest: Optional[bool] = False,
|
101
105
|
normalize_manifest: Optional[bool] = False,
|
106
|
+
config_path: Optional[str] = None,
|
102
107
|
) -> None:
|
103
108
|
"""
|
104
109
|
Args:
|
@@ -108,6 +113,7 @@ class ManifestDeclarativeSource(DeclarativeSource):
|
|
108
113
|
emit_connector_builder_messages: True if messages should be emitted to the connector builder.
|
109
114
|
component_factory: optional factory if ModelToComponentFactory's default behavior needs to be tweaked.
|
110
115
|
normalize_manifest: Optional flag to indicate if the manifest should be normalized.
|
116
|
+
config_path: Optional path to the config file.
|
111
117
|
"""
|
112
118
|
self.logger = logging.getLogger(f"airbyte.{self.name}")
|
113
119
|
self._should_normalize = normalize_manifest
|
@@ -130,7 +136,6 @@ class ManifestDeclarativeSource(DeclarativeSource):
|
|
130
136
|
self._slice_logger: SliceLogger = (
|
131
137
|
AlwaysLogSliceLogger() if emit_connector_builder_messages else DebugSliceLogger()
|
132
138
|
)
|
133
|
-
self._config = config or {}
|
134
139
|
|
135
140
|
# resolve all components in the manifest
|
136
141
|
self._source_config = self._pre_process_manifest(dict(source_config))
|
@@ -139,6 +144,12 @@ class ManifestDeclarativeSource(DeclarativeSource):
|
|
139
144
|
# apply additional post-processing to the manifest
|
140
145
|
self._post_process_manifest()
|
141
146
|
|
147
|
+
spec: Optional[Mapping[str, Any]] = self._source_config.get("spec")
|
148
|
+
self._spec_component: Optional[Spec] = (
|
149
|
+
self._constructor.create_component(SpecModel, spec, dict()) if spec else None
|
150
|
+
)
|
151
|
+
self._config = self._migrate_and_transform_config(config_path, config) or {}
|
152
|
+
|
142
153
|
@property
|
143
154
|
def resolved_manifest(self) -> Mapping[str, Any]:
|
144
155
|
"""
|
@@ -199,6 +210,30 @@ class ManifestDeclarativeSource(DeclarativeSource):
|
|
199
210
|
normalizer = ManifestNormalizer(self._source_config, self._declarative_component_schema)
|
200
211
|
self._source_config = normalizer.normalize()
|
201
212
|
|
213
|
+
def _migrate_and_transform_config(
|
214
|
+
self,
|
215
|
+
config_path: Optional[str],
|
216
|
+
config: Optional[Config],
|
217
|
+
) -> Optional[Config]:
|
218
|
+
if not config:
|
219
|
+
return None
|
220
|
+
if not self._spec_component:
|
221
|
+
return config
|
222
|
+
mutable_config = dict(config)
|
223
|
+
self._spec_component.migrate_config(mutable_config)
|
224
|
+
if mutable_config != config:
|
225
|
+
if config_path:
|
226
|
+
with open(config_path, "w") as f:
|
227
|
+
json.dump(mutable_config, f)
|
228
|
+
self.message_repository.emit_message(
|
229
|
+
create_connector_config_control_message(mutable_config)
|
230
|
+
)
|
231
|
+
# We have no mechanism for consuming the queue, so we print the messages to stdout
|
232
|
+
for message in self.message_repository.consume_queue():
|
233
|
+
print(orjson.dumps(AirbyteMessageSerializer.dump(message)).decode())
|
234
|
+
self._spec_component.transform_config(mutable_config)
|
235
|
+
return mutable_config
|
236
|
+
|
202
237
|
def _migrate_manifest(self) -> None:
|
203
238
|
"""
|
204
239
|
This method is used to migrate the manifest. It should be called after the manifest has been validated.
|
@@ -255,6 +290,9 @@ class ManifestDeclarativeSource(DeclarativeSource):
|
|
255
290
|
)
|
256
291
|
|
257
292
|
def streams(self, config: Mapping[str, Any]) -> List[Stream]:
|
293
|
+
if self._spec_component:
|
294
|
+
self._spec_component.validate_config(config)
|
295
|
+
|
258
296
|
self._emit_manifest_debug_message(
|
259
297
|
extra_args={
|
260
298
|
"source_name": self.name,
|
@@ -262,9 +300,7 @@ class ManifestDeclarativeSource(DeclarativeSource):
|
|
262
300
|
}
|
263
301
|
)
|
264
302
|
|
265
|
-
stream_configs = self._stream_configs(self._source_config) + self.
|
266
|
-
self._source_config, config
|
267
|
-
)
|
303
|
+
stream_configs = self._stream_configs(self._source_config) + self.dynamic_streams
|
268
304
|
|
269
305
|
api_budget_model = self._source_config.get("api_budget")
|
270
306
|
if api_budget_model:
|
@@ -355,14 +391,9 @@ class ManifestDeclarativeSource(DeclarativeSource):
|
|
355
391
|
}
|
356
392
|
)
|
357
393
|
|
358
|
-
|
359
|
-
|
360
|
-
|
361
|
-
spec["type"] = "Spec"
|
362
|
-
spec_component = self._constructor.create_component(SpecModel, spec, dict())
|
363
|
-
return spec_component.generate_spec()
|
364
|
-
else:
|
365
|
-
return super().spec(logger)
|
394
|
+
return (
|
395
|
+
self._spec_component.generate_spec() if self._spec_component else super().spec(logger)
|
396
|
+
)
|
366
397
|
|
367
398
|
def check(self, logger: logging.Logger, config: Mapping[str, Any]) -> AirbyteConnectionStatus:
|
368
399
|
self._configure_logger_level(logger)
|
@@ -1478,6 +1478,11 @@ class ComponentMappingDefinition(BaseModel):
|
|
1478
1478
|
description="The expected data type of the value. If omitted, the type will be inferred from the value provided.",
|
1479
1479
|
title="Value Type",
|
1480
1480
|
)
|
1481
|
+
create_or_update: Optional[bool] = Field(
|
1482
|
+
False,
|
1483
|
+
description="Determines whether to create a new path if it doesn't exist (true) or only update existing paths (false). When set to true, the resolver will create new paths in the stream template if they don't exist. When false (default), it will only update existing paths.",
|
1484
|
+
title="Create or Update",
|
1485
|
+
)
|
1481
1486
|
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
1482
1487
|
|
1483
1488
|
|
@@ -1489,12 +1494,17 @@ class StreamConfig(BaseModel):
|
|
1489
1494
|
examples=[["data"], ["data", "streams"], ["data", "{{ parameters.name }}"]],
|
1490
1495
|
title="Configs Pointer",
|
1491
1496
|
)
|
1497
|
+
default_values: Optional[List[Dict[str, Any]]] = Field(
|
1498
|
+
None,
|
1499
|
+
description="A list of default values, each matching the structure expected from the parsed component value.",
|
1500
|
+
title="Default Values",
|
1501
|
+
)
|
1492
1502
|
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
1493
1503
|
|
1494
1504
|
|
1495
1505
|
class ConfigComponentsResolver(BaseModel):
|
1496
1506
|
type: Literal["ConfigComponentsResolver"]
|
1497
|
-
stream_config: StreamConfig
|
1507
|
+
stream_config: Union[List[StreamConfig], StreamConfig]
|
1498
1508
|
components_mapping: List[ComponentMappingDefinition]
|
1499
1509
|
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
1500
1510
|
|
@@ -3754,6 +3754,7 @@ class ModelToComponentFactory:
|
|
3754
3754
|
field_path=field_path, # type: ignore[arg-type] # field_path can be str and InterpolatedString
|
3755
3755
|
value=interpolated_value,
|
3756
3756
|
value_type=ModelToComponentFactory._json_schema_type_name_to_type(model.value_type),
|
3757
|
+
create_or_update=model.create_or_update,
|
3757
3758
|
parameters=model.parameters or {},
|
3758
3759
|
)
|
3759
3760
|
|
@@ -3800,16 +3801,24 @@ class ModelToComponentFactory:
|
|
3800
3801
|
|
3801
3802
|
return StreamConfig(
|
3802
3803
|
configs_pointer=model_configs_pointer,
|
3804
|
+
default_values=model.default_values,
|
3803
3805
|
parameters=model.parameters or {},
|
3804
3806
|
)
|
3805
3807
|
|
3806
3808
|
def create_config_components_resolver(
|
3807
3809
|
self, model: ConfigComponentsResolverModel, config: Config
|
3808
3810
|
) -> Any:
|
3809
|
-
|
3810
|
-
model.stream_config
|
3811
|
+
model_stream_configs = (
|
3812
|
+
model.stream_config if isinstance(model.stream_config, list) else [model.stream_config]
|
3811
3813
|
)
|
3812
3814
|
|
3815
|
+
stream_configs = [
|
3816
|
+
self._create_component_from_model(
|
3817
|
+
stream_config, config=config, parameters=model.parameters or {}
|
3818
|
+
)
|
3819
|
+
for stream_config in model_stream_configs
|
3820
|
+
]
|
3821
|
+
|
3813
3822
|
components_mapping = [
|
3814
3823
|
self._create_component_from_model(
|
3815
3824
|
model=components_mapping_definition_model,
|
@@ -3822,7 +3831,7 @@ class ModelToComponentFactory:
|
|
3822
3831
|
]
|
3823
3832
|
|
3824
3833
|
return ConfigComponentsResolver(
|
3825
|
-
|
3834
|
+
stream_configs=stream_configs,
|
3826
3835
|
config=config,
|
3827
3836
|
components_mapping=components_mapping,
|
3828
3837
|
parameters=model.parameters or {},
|
@@ -22,6 +22,7 @@ class ComponentMappingDefinition:
|
|
22
22
|
value: Union["InterpolatedString", str]
|
23
23
|
value_type: Optional[Type[Any]]
|
24
24
|
parameters: InitVar[Mapping[str, Any]]
|
25
|
+
create_or_update: Optional[bool] = False
|
25
26
|
|
26
27
|
|
27
28
|
@dataclass(frozen=True)
|
@@ -34,6 +35,7 @@ class ResolvedComponentMappingDefinition:
|
|
34
35
|
value: "InterpolatedString"
|
35
36
|
value_type: Optional[Type[Any]]
|
36
37
|
parameters: InitVar[Mapping[str, Any]]
|
38
|
+
create_or_update: Optional[bool] = False
|
37
39
|
|
38
40
|
|
39
41
|
@deprecated("This class is experimental. Use at your own risk.", category=ExperimentalClassWarning)
|
@@ -4,10 +4,13 @@
|
|
4
4
|
|
5
5
|
from copy import deepcopy
|
6
6
|
from dataclasses import InitVar, dataclass, field
|
7
|
-
from
|
7
|
+
from itertools import product
|
8
|
+
from typing import Any, Dict, Iterable, List, Mapping, Optional, Tuple, Union
|
8
9
|
|
9
10
|
import dpath
|
11
|
+
import yaml
|
10
12
|
from typing_extensions import deprecated
|
13
|
+
from yaml.parser import ParserError
|
11
14
|
|
12
15
|
from airbyte_cdk.sources.declarative.interpolation import InterpolatedString
|
13
16
|
from airbyte_cdk.sources.declarative.resolvers.components_resolver import (
|
@@ -28,6 +31,7 @@ class StreamConfig:
|
|
28
31
|
|
29
32
|
configs_pointer: List[Union[InterpolatedString, str]]
|
30
33
|
parameters: InitVar[Mapping[str, Any]]
|
34
|
+
default_values: Optional[List[Any]] = None
|
31
35
|
|
32
36
|
def __post_init__(self, parameters: Mapping[str, Any]) -> None:
|
33
37
|
self.configs_pointer = [
|
@@ -48,7 +52,7 @@ class ConfigComponentsResolver(ComponentsResolver):
|
|
48
52
|
parameters (InitVar[Mapping[str, Any]]): Additional parameters for interpolation.
|
49
53
|
"""
|
50
54
|
|
51
|
-
|
55
|
+
stream_configs: List[StreamConfig]
|
52
56
|
config: Config
|
53
57
|
components_mapping: List[ComponentMappingDefinition]
|
54
58
|
parameters: InitVar[Mapping[str, Any]]
|
@@ -82,6 +86,7 @@ class ConfigComponentsResolver(ComponentsResolver):
|
|
82
86
|
field_path=field_path,
|
83
87
|
value=interpolated_value,
|
84
88
|
value_type=component_mapping.value_type,
|
89
|
+
create_or_update=component_mapping.create_or_update,
|
85
90
|
parameters=parameters,
|
86
91
|
)
|
87
92
|
)
|
@@ -90,18 +95,45 @@ class ConfigComponentsResolver(ComponentsResolver):
|
|
90
95
|
f"Expected a string or InterpolatedString for value in mapping: {component_mapping}"
|
91
96
|
)
|
92
97
|
|
98
|
+
@staticmethod
|
99
|
+
def _merge_combination(combo: Iterable[Tuple[int, Any]]) -> Dict[str, Any]:
|
100
|
+
"""Collapse a combination of ``(idx, elem)`` into one config dict."""
|
101
|
+
result: Dict[str, Any] = {}
|
102
|
+
for config_index, (elem_index, elem) in enumerate(combo):
|
103
|
+
if isinstance(elem, dict):
|
104
|
+
result.update(elem)
|
105
|
+
else:
|
106
|
+
# keep non-dict values under an artificial name
|
107
|
+
result.setdefault(f"source_config_{config_index}", (elem_index, elem))
|
108
|
+
return result
|
109
|
+
|
93
110
|
@property
|
94
|
-
def _stream_config(self) ->
|
95
|
-
|
96
|
-
|
97
|
-
|
111
|
+
def _stream_config(self) -> List[Dict[str, Any]]:
|
112
|
+
"""
|
113
|
+
Build every unique stream-configuration combination defined by
|
114
|
+
each ``StreamConfig`` and any ``default_values``.
|
115
|
+
"""
|
116
|
+
all_indexed_streams = []
|
117
|
+
for stream_config in self.stream_configs:
|
118
|
+
path = [
|
119
|
+
node.eval(self.config) if not isinstance(node, str) else node
|
120
|
+
for node in stream_config.configs_pointer
|
121
|
+
]
|
122
|
+
stream_configs_raw = dpath.get(dict(self.config), path, default=[])
|
123
|
+
stream_configs = (
|
124
|
+
list(stream_configs_raw)
|
125
|
+
if isinstance(stream_configs_raw, list)
|
126
|
+
else [stream_configs_raw]
|
127
|
+
)
|
128
|
+
|
129
|
+
if stream_config.default_values:
|
130
|
+
stream_configs.extend(stream_config.default_values)
|
131
|
+
|
132
|
+
all_indexed_streams.append([(i, item) for i, item in enumerate(stream_configs)])
|
133
|
+
return [
|
134
|
+
self._merge_combination(combo) # type: ignore[arg-type]
|
135
|
+
for combo in product(*all_indexed_streams)
|
98
136
|
]
|
99
|
-
stream_config = dpath.get(dict(self.config), path, default=[])
|
100
|
-
|
101
|
-
if not isinstance(stream_config, list):
|
102
|
-
stream_config = [stream_config]
|
103
|
-
|
104
|
-
return stream_config
|
105
137
|
|
106
138
|
def resolve_components(
|
107
139
|
self, stream_template_config: Dict[str, Any]
|
@@ -130,7 +162,27 @@ class ConfigComponentsResolver(ComponentsResolver):
|
|
130
162
|
)
|
131
163
|
|
132
164
|
path = [path.eval(self.config, **kwargs) for path in resolved_component.field_path]
|
165
|
+
parsed_value = self._parse_yaml_if_possible(value)
|
166
|
+
updated = dpath.set(updated_config, path, parsed_value)
|
133
167
|
|
134
|
-
|
168
|
+
if parsed_value and not updated and resolved_component.create_or_update:
|
169
|
+
dpath.new(updated_config, path, parsed_value)
|
135
170
|
|
136
171
|
yield updated_config
|
172
|
+
|
173
|
+
@staticmethod
|
174
|
+
def _parse_yaml_if_possible(value: Any) -> Any:
|
175
|
+
"""
|
176
|
+
Try to turn value into a Python object by YAML-parsing it.
|
177
|
+
|
178
|
+
* If value is a `str` and can be parsed by `yaml.safe_load`,
|
179
|
+
return the parsed result.
|
180
|
+
* If parsing fails (`yaml.parser.ParserError`) – or value is not
|
181
|
+
a string at all – return the original value unchanged.
|
182
|
+
"""
|
183
|
+
if isinstance(value, str):
|
184
|
+
try:
|
185
|
+
return yaml.safe_load(value)
|
186
|
+
except ParserError: # "{{ record[0] in ['cohortActiveUsers'] }}" # not valid YAML
|
187
|
+
return value
|
188
|
+
return value
|
@@ -1,28 +1,21 @@
|
|
1
1
|
#
|
2
|
-
# Copyright (c)
|
2
|
+
# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
|
3
3
|
#
|
4
4
|
|
5
|
-
import json
|
6
5
|
from dataclasses import InitVar, dataclass, field
|
7
6
|
from typing import Any, List, Mapping, MutableMapping, Optional
|
8
7
|
|
9
|
-
import orjson
|
10
|
-
|
11
|
-
from airbyte_cdk.config_observation import create_connector_config_control_message
|
12
|
-
from airbyte_cdk.entrypoint import AirbyteEntrypoint
|
13
8
|
from airbyte_cdk.models import (
|
14
9
|
AdvancedAuth,
|
15
10
|
ConnectorSpecification,
|
16
11
|
ConnectorSpecificationSerializer,
|
17
12
|
)
|
18
|
-
from airbyte_cdk.models.airbyte_protocol_serializers import AirbyteMessageSerializer
|
19
13
|
from airbyte_cdk.sources.declarative.models.declarative_component_schema import AuthFlow
|
20
14
|
from airbyte_cdk.sources.declarative.transformations.config_transformations.config_transformation import (
|
21
15
|
ConfigTransformation,
|
22
16
|
)
|
23
17
|
from airbyte_cdk.sources.declarative.validators.validator import Validator
|
24
18
|
from airbyte_cdk.sources.message.repository import InMemoryMessageRepository, MessageRepository
|
25
|
-
from airbyte_cdk.sources.source import Source
|
26
19
|
|
27
20
|
|
28
21
|
@dataclass
|
@@ -48,7 +41,6 @@ class Spec:
|
|
48
41
|
config_migrations: List[ConfigMigration] = field(default_factory=list)
|
49
42
|
config_transformations: List[ConfigTransformation] = field(default_factory=list)
|
50
43
|
config_validations: List[Validator] = field(default_factory=list)
|
51
|
-
message_repository: MessageRepository = InMemoryMessageRepository()
|
52
44
|
|
53
45
|
def generate_spec(self) -> ConnectorSpecification:
|
54
46
|
"""
|
@@ -69,34 +61,15 @@ class Spec:
|
|
69
61
|
# We remap these keys to camel case because that's the existing format expected by the rest of the platform
|
70
62
|
return ConnectorSpecificationSerializer.load(obj)
|
71
63
|
|
72
|
-
def migrate_config(
|
73
|
-
self, args: List[str], source: Source, config: MutableMapping[str, Any]
|
74
|
-
) -> None:
|
64
|
+
def migrate_config(self, config: MutableMapping[str, Any]) -> None:
|
75
65
|
"""
|
76
|
-
Apply all specified config transformations to the provided config and
|
66
|
+
Apply all specified config transformations to the provided config and emit a control message.
|
77
67
|
|
78
|
-
:param args: Command line arguments
|
79
|
-
:param source: Source instance
|
80
68
|
:param config: The user-provided config to migrate
|
81
69
|
"""
|
82
|
-
config_path = AirbyteEntrypoint(source).extract_config(args)
|
83
|
-
|
84
|
-
if not config_path:
|
85
|
-
return
|
86
|
-
|
87
|
-
mutable_config = dict(config)
|
88
70
|
for migration in self.config_migrations:
|
89
71
|
for transformation in migration.transformations:
|
90
|
-
transformation.transform(
|
91
|
-
|
92
|
-
if mutable_config != config:
|
93
|
-
with open(config_path, "w") as f:
|
94
|
-
json.dump(mutable_config, f)
|
95
|
-
self.message_repository.emit_message(
|
96
|
-
create_connector_config_control_message(mutable_config)
|
97
|
-
)
|
98
|
-
for message in self.message_repository.consume_queue():
|
99
|
-
print(orjson.dumps(AirbyteMessageSerializer.dump(message)).decode())
|
72
|
+
transformation.transform(config)
|
100
73
|
|
101
74
|
def transform_config(self, config: MutableMapping[str, Any]) -> None:
|
102
75
|
"""
|
@@ -107,7 +80,7 @@ class Spec:
|
|
107
80
|
for transformation in self.config_transformations:
|
108
81
|
transformation.transform(config)
|
109
82
|
|
110
|
-
def validate_config(self, config:
|
83
|
+
def validate_config(self, config: Mapping[str, Any]) -> None:
|
111
84
|
"""
|
112
85
|
Apply all config validations to the provided config.
|
113
86
|
|
@@ -24,6 +24,7 @@ class YamlDeclarativeSource(ConcurrentDeclarativeSource[List[AirbyteStateMessage
|
|
24
24
|
catalog: Optional[ConfiguredAirbyteCatalog] = None,
|
25
25
|
config: Optional[Mapping[str, Any]] = None,
|
26
26
|
state: Optional[List[AirbyteStateMessage]] = None,
|
27
|
+
config_path: Optional[str] = None,
|
27
28
|
) -> None:
|
28
29
|
"""
|
29
30
|
:param path_to_yaml: Path to the yaml file describing the source
|
@@ -36,6 +37,7 @@ class YamlDeclarativeSource(ConcurrentDeclarativeSource[List[AirbyteStateMessage
|
|
36
37
|
config=config or {},
|
37
38
|
state=state or [],
|
38
39
|
source_config=source_config,
|
40
|
+
config_path=config_path,
|
39
41
|
)
|
40
42
|
|
41
43
|
def _read_and_parse_yaml_file(self, path_to_yaml_file: str) -> ConnectionDefinition:
|
@@ -8,7 +8,7 @@ airbyte_cdk/cli/airbyte_cdk/_secrets.py,sha256=jLtpkhFJHavABN3UuqddeDRGS8v1iEj0e
|
|
8
8
|
airbyte_cdk/cli/airbyte_cdk/_version.py,sha256=ohZNIktLFk91sdzqFW5idaNrZAPX2dIRnz---_fcKOE,352
|
9
9
|
airbyte_cdk/cli/airbyte_cdk/exceptions.py,sha256=bsGmlWN6cXL2jCD1WYAZMqFmK1OLg2xLrcC_60KHSeA,803
|
10
10
|
airbyte_cdk/cli/source_declarative_manifest/__init__.py,sha256=-0ST722Nj65bgRokzpzPkD1NBBW5CytEHFUe38cB86Q,91
|
11
|
-
airbyte_cdk/cli/source_declarative_manifest/_run.py,sha256=
|
11
|
+
airbyte_cdk/cli/source_declarative_manifest/_run.py,sha256=jo4yydSDrB157GtTSZPWRfhZHYvFUD_btI5LW7wOrmY,11129
|
12
12
|
airbyte_cdk/cli/source_declarative_manifest/spec.json,sha256=Earc1L6ngcdIr514oFQlUoOxdF4RHqtUyStSIAquXdY,554
|
13
13
|
airbyte_cdk/config_observation.py,sha256=7SSPxtN0nXPkm4euGNcTTr1iLbwUL01jy-24V1Hzde0,3986
|
14
14
|
airbyte_cdk/connector.py,sha256=N6TUlrZOMjLAI85JrNAKkfyTqnO5xfBCw4oEfgjJd9o,4254
|
@@ -85,11 +85,11 @@ airbyte_cdk/sources/declarative/checks/check_stream.py,sha256=QeExVmpSYjr_CnghHu
|
|
85
85
|
airbyte_cdk/sources/declarative/checks/connection_checker.py,sha256=MBRJo6WJlZQHpIfOGaNOkkHUmgUl_4wDM6VPo41z5Ss,1383
|
86
86
|
airbyte_cdk/sources/declarative/concurrency_level/__init__.py,sha256=5XUqrmlstYlMM0j6crktlKQwALek0uiz2D3WdM46MyA,191
|
87
87
|
airbyte_cdk/sources/declarative/concurrency_level/concurrency_level.py,sha256=YIwCTCpOr_QSNW4ltQK0yUGWInI8PKNY216HOOegYLk,2101
|
88
|
-
airbyte_cdk/sources/declarative/concurrent_declarative_source.py,sha256=
|
88
|
+
airbyte_cdk/sources/declarative/concurrent_declarative_source.py,sha256=OKor1mDDdJZz8kgiR0KI-_pWDjCF1nuDcto_fSYerW4,28442
|
89
89
|
airbyte_cdk/sources/declarative/datetime/__init__.py,sha256=4Hw-PX1-VgESLF16cDdvuYCzGJtHntThLF4qIiULWeo,61
|
90
90
|
airbyte_cdk/sources/declarative/datetime/datetime_parser.py,sha256=_zGNGq31RNy_0QBLt_EcTvgPyhj7urPdx6oA3M5-r3o,3150
|
91
91
|
airbyte_cdk/sources/declarative/datetime/min_max_datetime.py,sha256=0BHBtDNQZfvwM45-tY5pNlTcKAFSGGNxemoi0Jic-0E,5785
|
92
|
-
airbyte_cdk/sources/declarative/declarative_component_schema.yaml,sha256=
|
92
|
+
airbyte_cdk/sources/declarative/declarative_component_schema.yaml,sha256=BQ0JKz4K8BKCpKp3C2IumJj2y2nv29lheeJUxPX6gXU,177389
|
93
93
|
airbyte_cdk/sources/declarative/declarative_source.py,sha256=qmyMnnet92eGc3C22yBtpvD5UZjqdhsAafP_zxI5wp8,1814
|
94
94
|
airbyte_cdk/sources/declarative/declarative_stream.py,sha256=dCRlddBUSaJmBNBz1pSO1r2rTw8AP5d2_vlmIeGs2gg,10767
|
95
95
|
airbyte_cdk/sources/declarative/decoders/__init__.py,sha256=JHb_0d3SE6kNY10mxA5YBEKPeSbsWYjByq1gUQxepoE,953
|
@@ -127,20 +127,20 @@ airbyte_cdk/sources/declarative/interpolation/interpolated_string.py,sha256=CQkH
|
|
127
127
|
airbyte_cdk/sources/declarative/interpolation/interpolation.py,sha256=9IoeuWam3L6GyN10L6U8xNWXmkt9cnahSDNkez1OmFY,982
|
128
128
|
airbyte_cdk/sources/declarative/interpolation/jinja.py,sha256=UQeuS4Vpyp4hlOn-R3tRyeBX0e9IoV6jQ6gH-Jz8lY0,7182
|
129
129
|
airbyte_cdk/sources/declarative/interpolation/macros.py,sha256=UYSJ5gW7TkHALYnNvUnRP3RlyGwGuRMObF3BHuNzjJM,5320
|
130
|
-
airbyte_cdk/sources/declarative/manifest_declarative_source.py,sha256=
|
130
|
+
airbyte_cdk/sources/declarative/manifest_declarative_source.py,sha256=aS_YxqUWgxyfyzwAZpV736RFg-m5a0sn-V7PPMBGVr4,24660
|
131
131
|
airbyte_cdk/sources/declarative/migrations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
132
132
|
airbyte_cdk/sources/declarative/migrations/legacy_to_per_partition_state_migration.py,sha256=V2lpYE9LJKvz6BUViHk4vaRGndxNABmPbDCtyYdkqaE,4013
|
133
133
|
airbyte_cdk/sources/declarative/migrations/state_migration.py,sha256=KWPjealMLKSMtajXgkdGgKg7EmTLR-CqqD7UIh0-eDU,794
|
134
134
|
airbyte_cdk/sources/declarative/models/__init__.py,sha256=nUFxNCiKeYRVXuZEKA7GD-lTHxsiKcQ8FitZjKhPIvE,100
|
135
135
|
airbyte_cdk/sources/declarative/models/base_model_with_deprecations.py,sha256=Imnj3yef0aqRdLfaUxkIYISUb8YkiPrRH_wBd-x8HjM,5999
|
136
|
-
airbyte_cdk/sources/declarative/models/declarative_component_schema.py,sha256=
|
136
|
+
airbyte_cdk/sources/declarative/models/declarative_component_schema.py,sha256=XVfBPgNBEEkv8u6Bj1JRiXsQWa_QoipPHFxND_YsxDY,125766
|
137
137
|
airbyte_cdk/sources/declarative/parsers/__init__.py,sha256=ZnqYNxHsKCgO38IwB34RQyRMXTs4GTvlRi3ImKnIioo,61
|
138
138
|
airbyte_cdk/sources/declarative/parsers/custom_code_compiler.py,sha256=nlVvHC511NUyDEEIRBkoeDTAvLqKNp-hRy8D19z8tdk,5941
|
139
139
|
airbyte_cdk/sources/declarative/parsers/custom_exceptions.py,sha256=wnRUP0Xeru9Rbu5OexXSDN9QWDo8YU4tT9M2LDVOgGA,802
|
140
140
|
airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py,sha256=2UdpCz3yi7ISZTyqkQXSSy3dMxeyOWqV7OlAS5b9GVg,11568
|
141
141
|
airbyte_cdk/sources/declarative/parsers/manifest_normalizer.py,sha256=laBy7ebjA-PiNwc-50U4FHvMqS_mmHvnabxgFs4CjGw,17069
|
142
142
|
airbyte_cdk/sources/declarative/parsers/manifest_reference_resolver.py,sha256=pJmg78vqE5VfUrF_KJnWjucQ4k9IWFULeAxHCowrHXE,6806
|
143
|
-
airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py,sha256=
|
143
|
+
airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py,sha256=7-q8icBPnEHbPa0vonz5KjsalYrrqJasIcLXK1TITlc,174730
|
144
144
|
airbyte_cdk/sources/declarative/partition_routers/__init__.py,sha256=TBC9AkGaUqHm2IKHMPN6punBIcY5tWGULowcLoAVkfw,1109
|
145
145
|
airbyte_cdk/sources/declarative/partition_routers/async_job_partition_router.py,sha256=VelO7zKqKtzMJ35jyFeg0ypJLQC0plqqIBNXoBW1G2E,3001
|
146
146
|
airbyte_cdk/sources/declarative/partition_routers/cartesian_product_stream_slicer.py,sha256=c5cuVFM6NFkuQqG8Z5IwkBuwDrvXZN1CunUOM_L0ezg,6892
|
@@ -194,8 +194,8 @@ airbyte_cdk/sources/declarative/requesters/request_options/request_options_provi
|
|
194
194
|
airbyte_cdk/sources/declarative/requesters/request_path.py,sha256=S3MeFvcaQrMbOkSY2W2VbXLNomqt_3eXqVd9ZhgNwUs,299
|
195
195
|
airbyte_cdk/sources/declarative/requesters/requester.py,sha256=T6tMx_Bx4iT-0YVjY7IzgRil-gaIu9n01b1iwpTh3Ek,5516
|
196
196
|
airbyte_cdk/sources/declarative/resolvers/__init__.py,sha256=NiDcz5qi8HPsfX94MUmnX0Rgs_kQXGvucOmJjNWlxKQ,1207
|
197
|
-
airbyte_cdk/sources/declarative/resolvers/components_resolver.py,sha256=
|
198
|
-
airbyte_cdk/sources/declarative/resolvers/config_components_resolver.py,sha256=
|
197
|
+
airbyte_cdk/sources/declarative/resolvers/components_resolver.py,sha256=oJIpy66ep8n-QOc8GwpllApTRcl4QtQhkTw5fWWra2w,2026
|
198
|
+
airbyte_cdk/sources/declarative/resolvers/config_components_resolver.py,sha256=sD3N7nmqDjLsau8P2DE7DYOHdFTYjC_2nIB6454BNYk,7556
|
199
199
|
airbyte_cdk/sources/declarative/resolvers/http_components_resolver.py,sha256=AiojNs8wItJFrENZBFUaDvau3sgwudO6Wkra36upSPo,4639
|
200
200
|
airbyte_cdk/sources/declarative/retrievers/__init__.py,sha256=nQepwG_RfW53sgwvK5dLPqfCx0VjsQ83nYoPjBMAaLM,527
|
201
201
|
airbyte_cdk/sources/declarative/retrievers/async_retriever.py,sha256=6oZtnCHm9NdDvjTSrVwPQOXGSdETSIR7eWH2vFjM7jI,4855
|
@@ -216,7 +216,7 @@ airbyte_cdk/sources/declarative/schema/inline_schema_loader.py,sha256=bVETE10hRs
|
|
216
216
|
airbyte_cdk/sources/declarative/schema/json_file_schema_loader.py,sha256=5Wl-fqW-pVf_dxJ4yGHMAFfC4JjKHYJhqFJT1xA57F4,4177
|
217
217
|
airbyte_cdk/sources/declarative/schema/schema_loader.py,sha256=kjt8v0N5wWKA5zyLnrDLxf1PJKdUqvQq2RVnAOAzNSY,379
|
218
218
|
airbyte_cdk/sources/declarative/spec/__init__.py,sha256=9FYO-fVOclrwjAW4qwRTbZRVopTc9rOaauAJfThdNCQ,177
|
219
|
-
airbyte_cdk/sources/declarative/spec/spec.py,sha256=
|
219
|
+
airbyte_cdk/sources/declarative/spec/spec.py,sha256=SwL_pfXZgcLYLJY-MAeFMHug9oYh2tOWjgG0C3DoLOY,3602
|
220
220
|
airbyte_cdk/sources/declarative/stream_slicers/__init__.py,sha256=sI9vhc95RwJYOnA0VKjcbtKgFcmAbWjhdWBXFbAijOs,176
|
221
221
|
airbyte_cdk/sources/declarative/stream_slicers/declarative_partition_generator.py,sha256=cjKGm4r438dd1GxrFHJ4aYrdzG2bkncnwaWxAwlXR3M,3585
|
222
222
|
airbyte_cdk/sources/declarative/stream_slicers/stream_slicer.py,sha256=SOkIPBi2Wu7yxIvA15yFzUAB95a3IzA8LPq5DEqHQQc,725
|
@@ -241,7 +241,7 @@ airbyte_cdk/sources/declarative/validators/predicate_validator.py,sha256=Q4eVncl
|
|
241
241
|
airbyte_cdk/sources/declarative/validators/validate_adheres_to_schema.py,sha256=kjcuKxWMJEzpF4GiESITGMxBAXw6YZCAsgOQMgeBo4g,1085
|
242
242
|
airbyte_cdk/sources/declarative/validators/validation_strategy.py,sha256=LwqUX89cFdHTM1-h6c8vebBA9WC38HYoGBvJfCZHr0g,467
|
243
243
|
airbyte_cdk/sources/declarative/validators/validator.py,sha256=MAwo8OievUsuzBuPxI9pbPu87yq0tJZkGbydcrHZyQc,382
|
244
|
-
airbyte_cdk/sources/declarative/yaml_declarative_source.py,sha256
|
244
|
+
airbyte_cdk/sources/declarative/yaml_declarative_source.py,sha256=-pHwGO7ZW-x8lmsqSpbrN0pOgIyjJhFDGUNwB3kQWWc,2794
|
245
245
|
airbyte_cdk/sources/file_based/README.md,sha256=iMqww4VZ882jfNQIdljjDgqreKs-mkdtSrRKA94iX6A,11085
|
246
246
|
airbyte_cdk/sources/file_based/__init__.py,sha256=EaxHv_9ot-eRlUCR47ZMZ0IOtB-n0HH24om7Bfn-uuQ,868
|
247
247
|
airbyte_cdk/sources/file_based/availability_strategy/__init__.py,sha256=ddKQfUmk-Ls7LJaG8gtrqDybG3d8S7KXOAEjLeYLrTg,399
|
@@ -419,9 +419,9 @@ airbyte_cdk/utils/slice_hasher.py,sha256=EDxgROHDbfG-QKQb59m7h_7crN1tRiawdf5uU7G
|
|
419
419
|
airbyte_cdk/utils/spec_schema_transformations.py,sha256=-5HTuNsnDBAhj-oLeQXwpTGA0HdcjFOf2zTEMUTTg_Y,816
|
420
420
|
airbyte_cdk/utils/stream_status_utils.py,sha256=ZmBoiy5HVbUEHAMrUONxZvxnvfV9CesmQJLDTAIWnWw,1171
|
421
421
|
airbyte_cdk/utils/traced_exception.py,sha256=C8uIBuCL_E4WnBAOPSxBicD06JAldoN9fGsQDp463OY,6292
|
422
|
-
airbyte_cdk-6.
|
423
|
-
airbyte_cdk-6.
|
424
|
-
airbyte_cdk-6.
|
425
|
-
airbyte_cdk-6.
|
426
|
-
airbyte_cdk-6.
|
427
|
-
airbyte_cdk-6.
|
422
|
+
airbyte_cdk-6.52.0.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
|
423
|
+
airbyte_cdk-6.52.0.dist-info/LICENSE_SHORT,sha256=aqF6D1NcESmpn-cqsxBtszTEnHKnlsp8L4x9wAh3Nxg,55
|
424
|
+
airbyte_cdk-6.52.0.dist-info/METADATA,sha256=d5e43kQPtd5tZNY-xH1y7PrUdsmFC-sqiC8pDEajuyk,6343
|
425
|
+
airbyte_cdk-6.52.0.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
|
426
|
+
airbyte_cdk-6.52.0.dist-info/entry_points.txt,sha256=AKWbEkHfpzzk9nF9tqBUaw1MbvTM4mGtEzmZQm0ZWvM,139
|
427
|
+
airbyte_cdk-6.52.0.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|