airbyte-cdk 6.50.0__py3-none-any.whl → 6.52.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -58,6 +58,7 @@ class SourceLocalYaml(YamlDeclarativeSource):
58
58
  catalog: ConfiguredAirbyteCatalog | None,
59
59
  config: MutableMapping[str, Any] | None,
60
60
  state: TState,
61
+ config_path: str | None = None,
61
62
  **kwargs: Any,
62
63
  ) -> None:
63
64
  """
@@ -76,6 +77,7 @@ class SourceLocalYaml(YamlDeclarativeSource):
76
77
  config=config,
77
78
  state=state, # type: ignore [arg-type]
78
79
  path_to_yaml="manifest.yaml",
80
+ config_path=config_path,
79
81
  )
80
82
 
81
83
 
@@ -95,7 +97,12 @@ def _get_local_yaml_source(args: list[str]) -> SourceLocalYaml:
95
97
  try:
96
98
  parsed_args = AirbyteEntrypoint.parse_args(args)
97
99
  config, catalog, state = _parse_inputs_into_config_catalog_state(parsed_args)
98
- return SourceLocalYaml(config=config, catalog=catalog, state=state)
100
+ return SourceLocalYaml(
101
+ config=config,
102
+ catalog=catalog,
103
+ state=state,
104
+ config_path=parsed_args.config if hasattr(parsed_args, "config") else None,
105
+ )
99
106
  except Exception as error:
100
107
  print(
101
108
  orjson.dumps(
@@ -204,6 +211,7 @@ def create_declarative_source(
204
211
  catalog=catalog,
205
212
  state=state,
206
213
  source_config=cast(dict[str, Any], config["__injected_declarative_manifest"]),
214
+ config_path=parsed_args.config if hasattr(parsed_args, "config") else None,
207
215
  )
208
216
  except Exception as error:
209
217
  print(
@@ -74,6 +74,7 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
74
74
  debug: bool = False,
75
75
  emit_connector_builder_messages: bool = False,
76
76
  component_factory: Optional[ModelToComponentFactory] = None,
77
+ config_path: Optional[str] = None,
77
78
  **kwargs: Any,
78
79
  ) -> None:
79
80
  # todo: We could remove state from initialization. Now that streams are grouped during the read(), a source
@@ -96,6 +97,7 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
96
97
  debug=debug,
97
98
  emit_connector_builder_messages=emit_connector_builder_messages,
98
99
  component_factory=component_factory,
100
+ config_path=config_path,
99
101
  )
100
102
 
101
103
  concurrency_level_from_manifest = self._source_config.get("concurrency_level")
@@ -4055,6 +4055,11 @@ definitions:
4055
4055
  title: Value Type
4056
4056
  description: The expected data type of the value. If omitted, the type will be inferred from the value provided.
4057
4057
  "$ref": "#/definitions/ValueType"
4058
+ create_or_update:
4059
+ title: Create or Update
4060
+ description: Determines whether to create a new path if it doesn't exist (true) or only update existing paths (false). When set to true, the resolver will create new paths in the stream template if they don't exist. When false (default), it will only update existing paths.
4061
+ type: boolean
4062
+ default: false
4058
4063
  $parameters:
4059
4064
  type: object
4060
4065
  additionalProperties: true
@@ -4106,6 +4111,12 @@ definitions:
4106
4111
  - ["data"]
4107
4112
  - ["data", "streams"]
4108
4113
  - ["data", "{{ parameters.name }}"]
4114
+ default_values:
4115
+ title: Default Values
4116
+ description: A list of default values, each matching the structure expected from the parsed component value.
4117
+ type: array
4118
+ items:
4119
+ type: object
4109
4120
  $parameters:
4110
4121
  type: object
4111
4122
  additionalProperties: true
@@ -4117,7 +4128,11 @@ definitions:
4117
4128
  type: string
4118
4129
  enum: [ConfigComponentsResolver]
4119
4130
  stream_config:
4120
- "$ref": "#/definitions/StreamConfig"
4131
+ anyOf:
4132
+ - type: array
4133
+ items:
4134
+ "$ref": "#/definitions/StreamConfig"
4135
+ - "$ref": "#/definitions/StreamConfig"
4121
4136
  components_mapping:
4122
4137
  type: array
4123
4138
  items:
@@ -1,5 +1,5 @@
1
1
  #
2
- # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
2
+ # Copyright (c) 2025 Airbyte, Inc., all rights reserved.
3
3
  #
4
4
 
5
5
  import json
@@ -8,13 +8,15 @@ import pkgutil
8
8
  from copy import deepcopy
9
9
  from importlib import metadata
10
10
  from types import ModuleType
11
- from typing import Any, Dict, Iterator, List, Mapping, Optional, Set
11
+ from typing import Any, Dict, Iterator, List, Mapping, MutableMapping, Optional, Set
12
12
 
13
+ import orjson
13
14
  import yaml
14
15
  from jsonschema.exceptions import ValidationError
15
16
  from jsonschema.validators import validate
16
17
  from packaging.version import InvalidVersion, Version
17
18
 
19
+ from airbyte_cdk.config_observation import create_connector_config_control_message
18
20
  from airbyte_cdk.connector_builder.models import (
19
21
  LogMessage as ConnectorBuilderLogMessage,
20
22
  )
@@ -29,6 +31,7 @@ from airbyte_cdk.models import (
29
31
  ConnectorSpecification,
30
32
  FailureType,
31
33
  )
34
+ from airbyte_cdk.models.airbyte_protocol_serializers import AirbyteMessageSerializer
32
35
  from airbyte_cdk.sources.declarative.checks import COMPONENTS_CHECKER_TYPE_MAPPING
33
36
  from airbyte_cdk.sources.declarative.checks.connection_checker import ConnectionChecker
34
37
  from airbyte_cdk.sources.declarative.declarative_source import DeclarativeSource
@@ -57,9 +60,10 @@ from airbyte_cdk.sources.declarative.parsers.model_to_component_factory import (
57
60
  ModelToComponentFactory,
58
61
  )
59
62
  from airbyte_cdk.sources.declarative.resolvers import COMPONENTS_RESOLVER_TYPE_MAPPING
63
+ from airbyte_cdk.sources.declarative.spec.spec import Spec
60
64
  from airbyte_cdk.sources.message import MessageRepository
61
65
  from airbyte_cdk.sources.streams.core import Stream
62
- from airbyte_cdk.sources.types import ConnectionDefinition
66
+ from airbyte_cdk.sources.types import Config, ConnectionDefinition
63
67
  from airbyte_cdk.sources.utils.slice_logger import (
64
68
  AlwaysLogSliceLogger,
65
69
  DebugSliceLogger,
@@ -99,6 +103,7 @@ class ManifestDeclarativeSource(DeclarativeSource):
99
103
  component_factory: Optional[ModelToComponentFactory] = None,
100
104
  migrate_manifest: Optional[bool] = False,
101
105
  normalize_manifest: Optional[bool] = False,
106
+ config_path: Optional[str] = None,
102
107
  ) -> None:
103
108
  """
104
109
  Args:
@@ -108,6 +113,7 @@ class ManifestDeclarativeSource(DeclarativeSource):
108
113
  emit_connector_builder_messages: True if messages should be emitted to the connector builder.
109
114
  component_factory: optional factory if ModelToComponentFactory's default behavior needs to be tweaked.
110
115
  normalize_manifest: Optional flag to indicate if the manifest should be normalized.
116
+ config_path: Optional path to the config file.
111
117
  """
112
118
  self.logger = logging.getLogger(f"airbyte.{self.name}")
113
119
  self._should_normalize = normalize_manifest
@@ -130,7 +136,6 @@ class ManifestDeclarativeSource(DeclarativeSource):
130
136
  self._slice_logger: SliceLogger = (
131
137
  AlwaysLogSliceLogger() if emit_connector_builder_messages else DebugSliceLogger()
132
138
  )
133
- self._config = config or {}
134
139
 
135
140
  # resolve all components in the manifest
136
141
  self._source_config = self._pre_process_manifest(dict(source_config))
@@ -139,6 +144,12 @@ class ManifestDeclarativeSource(DeclarativeSource):
139
144
  # apply additional post-processing to the manifest
140
145
  self._post_process_manifest()
141
146
 
147
+ spec: Optional[Mapping[str, Any]] = self._source_config.get("spec")
148
+ self._spec_component: Optional[Spec] = (
149
+ self._constructor.create_component(SpecModel, spec, dict()) if spec else None
150
+ )
151
+ self._config = self._migrate_and_transform_config(config_path, config) or {}
152
+
142
153
  @property
143
154
  def resolved_manifest(self) -> Mapping[str, Any]:
144
155
  """
@@ -199,6 +210,30 @@ class ManifestDeclarativeSource(DeclarativeSource):
199
210
  normalizer = ManifestNormalizer(self._source_config, self._declarative_component_schema)
200
211
  self._source_config = normalizer.normalize()
201
212
 
213
+ def _migrate_and_transform_config(
214
+ self,
215
+ config_path: Optional[str],
216
+ config: Optional[Config],
217
+ ) -> Optional[Config]:
218
+ if not config:
219
+ return None
220
+ if not self._spec_component:
221
+ return config
222
+ mutable_config = dict(config)
223
+ self._spec_component.migrate_config(mutable_config)
224
+ if mutable_config != config:
225
+ if config_path:
226
+ with open(config_path, "w") as f:
227
+ json.dump(mutable_config, f)
228
+ self.message_repository.emit_message(
229
+ create_connector_config_control_message(mutable_config)
230
+ )
231
+ # We have no mechanism for consuming the queue, so we print the messages to stdout
232
+ for message in self.message_repository.consume_queue():
233
+ print(orjson.dumps(AirbyteMessageSerializer.dump(message)).decode())
234
+ self._spec_component.transform_config(mutable_config)
235
+ return mutable_config
236
+
202
237
  def _migrate_manifest(self) -> None:
203
238
  """
204
239
  This method is used to migrate the manifest. It should be called after the manifest has been validated.
@@ -255,6 +290,9 @@ class ManifestDeclarativeSource(DeclarativeSource):
255
290
  )
256
291
 
257
292
  def streams(self, config: Mapping[str, Any]) -> List[Stream]:
293
+ if self._spec_component:
294
+ self._spec_component.validate_config(config)
295
+
258
296
  self._emit_manifest_debug_message(
259
297
  extra_args={
260
298
  "source_name": self.name,
@@ -262,9 +300,7 @@ class ManifestDeclarativeSource(DeclarativeSource):
262
300
  }
263
301
  )
264
302
 
265
- stream_configs = self._stream_configs(self._source_config) + self._dynamic_stream_configs(
266
- self._source_config, config
267
- )
303
+ stream_configs = self._stream_configs(self._source_config) + self.dynamic_streams
268
304
 
269
305
  api_budget_model = self._source_config.get("api_budget")
270
306
  if api_budget_model:
@@ -355,14 +391,9 @@ class ManifestDeclarativeSource(DeclarativeSource):
355
391
  }
356
392
  )
357
393
 
358
- spec = self._source_config.get("spec")
359
- if spec:
360
- if "type" not in spec:
361
- spec["type"] = "Spec"
362
- spec_component = self._constructor.create_component(SpecModel, spec, dict())
363
- return spec_component.generate_spec()
364
- else:
365
- return super().spec(logger)
394
+ return (
395
+ self._spec_component.generate_spec() if self._spec_component else super().spec(logger)
396
+ )
366
397
 
367
398
  def check(self, logger: logging.Logger, config: Mapping[str, Any]) -> AirbyteConnectionStatus:
368
399
  self._configure_logger_level(logger)
@@ -1478,6 +1478,11 @@ class ComponentMappingDefinition(BaseModel):
1478
1478
  description="The expected data type of the value. If omitted, the type will be inferred from the value provided.",
1479
1479
  title="Value Type",
1480
1480
  )
1481
+ create_or_update: Optional[bool] = Field(
1482
+ False,
1483
+ description="Determines whether to create a new path if it doesn't exist (true) or only update existing paths (false). When set to true, the resolver will create new paths in the stream template if they don't exist. When false (default), it will only update existing paths.",
1484
+ title="Create or Update",
1485
+ )
1481
1486
  parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
1482
1487
 
1483
1488
 
@@ -1489,12 +1494,17 @@ class StreamConfig(BaseModel):
1489
1494
  examples=[["data"], ["data", "streams"], ["data", "{{ parameters.name }}"]],
1490
1495
  title="Configs Pointer",
1491
1496
  )
1497
+ default_values: Optional[List[Dict[str, Any]]] = Field(
1498
+ None,
1499
+ description="A list of default values, each matching the structure expected from the parsed component value.",
1500
+ title="Default Values",
1501
+ )
1492
1502
  parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
1493
1503
 
1494
1504
 
1495
1505
  class ConfigComponentsResolver(BaseModel):
1496
1506
  type: Literal["ConfigComponentsResolver"]
1497
- stream_config: StreamConfig
1507
+ stream_config: Union[List[StreamConfig], StreamConfig]
1498
1508
  components_mapping: List[ComponentMappingDefinition]
1499
1509
  parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
1500
1510
 
@@ -3754,6 +3754,7 @@ class ModelToComponentFactory:
3754
3754
  field_path=field_path, # type: ignore[arg-type] # field_path can be str and InterpolatedString
3755
3755
  value=interpolated_value,
3756
3756
  value_type=ModelToComponentFactory._json_schema_type_name_to_type(model.value_type),
3757
+ create_or_update=model.create_or_update,
3757
3758
  parameters=model.parameters or {},
3758
3759
  )
3759
3760
 
@@ -3800,16 +3801,24 @@ class ModelToComponentFactory:
3800
3801
 
3801
3802
  return StreamConfig(
3802
3803
  configs_pointer=model_configs_pointer,
3804
+ default_values=model.default_values,
3803
3805
  parameters=model.parameters or {},
3804
3806
  )
3805
3807
 
3806
3808
  def create_config_components_resolver(
3807
3809
  self, model: ConfigComponentsResolverModel, config: Config
3808
3810
  ) -> Any:
3809
- stream_config = self._create_component_from_model(
3810
- model.stream_config, config=config, parameters=model.parameters or {}
3811
+ model_stream_configs = (
3812
+ model.stream_config if isinstance(model.stream_config, list) else [model.stream_config]
3811
3813
  )
3812
3814
 
3815
+ stream_configs = [
3816
+ self._create_component_from_model(
3817
+ stream_config, config=config, parameters=model.parameters or {}
3818
+ )
3819
+ for stream_config in model_stream_configs
3820
+ ]
3821
+
3813
3822
  components_mapping = [
3814
3823
  self._create_component_from_model(
3815
3824
  model=components_mapping_definition_model,
@@ -3822,7 +3831,7 @@ class ModelToComponentFactory:
3822
3831
  ]
3823
3832
 
3824
3833
  return ConfigComponentsResolver(
3825
- stream_config=stream_config,
3834
+ stream_configs=stream_configs,
3826
3835
  config=config,
3827
3836
  components_mapping=components_mapping,
3828
3837
  parameters=model.parameters or {},
@@ -22,6 +22,7 @@ class ComponentMappingDefinition:
22
22
  value: Union["InterpolatedString", str]
23
23
  value_type: Optional[Type[Any]]
24
24
  parameters: InitVar[Mapping[str, Any]]
25
+ create_or_update: Optional[bool] = False
25
26
 
26
27
 
27
28
  @dataclass(frozen=True)
@@ -34,6 +35,7 @@ class ResolvedComponentMappingDefinition:
34
35
  value: "InterpolatedString"
35
36
  value_type: Optional[Type[Any]]
36
37
  parameters: InitVar[Mapping[str, Any]]
38
+ create_or_update: Optional[bool] = False
37
39
 
38
40
 
39
41
  @deprecated("This class is experimental. Use at your own risk.", category=ExperimentalClassWarning)
@@ -4,10 +4,13 @@
4
4
 
5
5
  from copy import deepcopy
6
6
  from dataclasses import InitVar, dataclass, field
7
- from typing import Any, Dict, Iterable, List, Mapping, Union
7
+ from itertools import product
8
+ from typing import Any, Dict, Iterable, List, Mapping, Optional, Tuple, Union
8
9
 
9
10
  import dpath
11
+ import yaml
10
12
  from typing_extensions import deprecated
13
+ from yaml.parser import ParserError
11
14
 
12
15
  from airbyte_cdk.sources.declarative.interpolation import InterpolatedString
13
16
  from airbyte_cdk.sources.declarative.resolvers.components_resolver import (
@@ -28,6 +31,7 @@ class StreamConfig:
28
31
 
29
32
  configs_pointer: List[Union[InterpolatedString, str]]
30
33
  parameters: InitVar[Mapping[str, Any]]
34
+ default_values: Optional[List[Any]] = None
31
35
 
32
36
  def __post_init__(self, parameters: Mapping[str, Any]) -> None:
33
37
  self.configs_pointer = [
@@ -48,7 +52,7 @@ class ConfigComponentsResolver(ComponentsResolver):
48
52
  parameters (InitVar[Mapping[str, Any]]): Additional parameters for interpolation.
49
53
  """
50
54
 
51
- stream_config: StreamConfig
55
+ stream_configs: List[StreamConfig]
52
56
  config: Config
53
57
  components_mapping: List[ComponentMappingDefinition]
54
58
  parameters: InitVar[Mapping[str, Any]]
@@ -82,6 +86,7 @@ class ConfigComponentsResolver(ComponentsResolver):
82
86
  field_path=field_path,
83
87
  value=interpolated_value,
84
88
  value_type=component_mapping.value_type,
89
+ create_or_update=component_mapping.create_or_update,
85
90
  parameters=parameters,
86
91
  )
87
92
  )
@@ -90,18 +95,45 @@ class ConfigComponentsResolver(ComponentsResolver):
90
95
  f"Expected a string or InterpolatedString for value in mapping: {component_mapping}"
91
96
  )
92
97
 
98
+ @staticmethod
99
+ def _merge_combination(combo: Iterable[Tuple[int, Any]]) -> Dict[str, Any]:
100
+ """Collapse a combination of ``(idx, elem)`` into one config dict."""
101
+ result: Dict[str, Any] = {}
102
+ for config_index, (elem_index, elem) in enumerate(combo):
103
+ if isinstance(elem, dict):
104
+ result.update(elem)
105
+ else:
106
+ # keep non-dict values under an artificial name
107
+ result.setdefault(f"source_config_{config_index}", (elem_index, elem))
108
+ return result
109
+
93
110
  @property
94
- def _stream_config(self) -> Iterable[Mapping[str, Any]]:
95
- path = [
96
- node.eval(self.config) if not isinstance(node, str) else node
97
- for node in self.stream_config.configs_pointer
111
+ def _stream_config(self) -> List[Dict[str, Any]]:
112
+ """
113
+ Build every unique stream-configuration combination defined by
114
+ each ``StreamConfig`` and any ``default_values``.
115
+ """
116
+ all_indexed_streams = []
117
+ for stream_config in self.stream_configs:
118
+ path = [
119
+ node.eval(self.config) if not isinstance(node, str) else node
120
+ for node in stream_config.configs_pointer
121
+ ]
122
+ stream_configs_raw = dpath.get(dict(self.config), path, default=[])
123
+ stream_configs = (
124
+ list(stream_configs_raw)
125
+ if isinstance(stream_configs_raw, list)
126
+ else [stream_configs_raw]
127
+ )
128
+
129
+ if stream_config.default_values:
130
+ stream_configs.extend(stream_config.default_values)
131
+
132
+ all_indexed_streams.append([(i, item) for i, item in enumerate(stream_configs)])
133
+ return [
134
+ self._merge_combination(combo) # type: ignore[arg-type]
135
+ for combo in product(*all_indexed_streams)
98
136
  ]
99
- stream_config = dpath.get(dict(self.config), path, default=[])
100
-
101
- if not isinstance(stream_config, list):
102
- stream_config = [stream_config]
103
-
104
- return stream_config
105
137
 
106
138
  def resolve_components(
107
139
  self, stream_template_config: Dict[str, Any]
@@ -130,7 +162,27 @@ class ConfigComponentsResolver(ComponentsResolver):
130
162
  )
131
163
 
132
164
  path = [path.eval(self.config, **kwargs) for path in resolved_component.field_path]
165
+ parsed_value = self._parse_yaml_if_possible(value)
166
+ updated = dpath.set(updated_config, path, parsed_value)
133
167
 
134
- dpath.set(updated_config, path, value)
168
+ if parsed_value and not updated and resolved_component.create_or_update:
169
+ dpath.new(updated_config, path, parsed_value)
135
170
 
136
171
  yield updated_config
172
+
173
+ @staticmethod
174
+ def _parse_yaml_if_possible(value: Any) -> Any:
175
+ """
176
+ Try to turn value into a Python object by YAML-parsing it.
177
+
178
+ * If value is a `str` and can be parsed by `yaml.safe_load`,
179
+ return the parsed result.
180
+ * If parsing fails (`yaml.parser.ParserError`) – or value is not
181
+ a string at all – return the original value unchanged.
182
+ """
183
+ if isinstance(value, str):
184
+ try:
185
+ return yaml.safe_load(value)
186
+ except ParserError: # "{{ record[0] in ['cohortActiveUsers'] }}" # not valid YAML
187
+ return value
188
+ return value
@@ -1,28 +1,21 @@
1
1
  #
2
- # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
2
+ # Copyright (c) 2025 Airbyte, Inc., all rights reserved.
3
3
  #
4
4
 
5
- import json
6
5
  from dataclasses import InitVar, dataclass, field
7
6
  from typing import Any, List, Mapping, MutableMapping, Optional
8
7
 
9
- import orjson
10
-
11
- from airbyte_cdk.config_observation import create_connector_config_control_message
12
- from airbyte_cdk.entrypoint import AirbyteEntrypoint
13
8
  from airbyte_cdk.models import (
14
9
  AdvancedAuth,
15
10
  ConnectorSpecification,
16
11
  ConnectorSpecificationSerializer,
17
12
  )
18
- from airbyte_cdk.models.airbyte_protocol_serializers import AirbyteMessageSerializer
19
13
  from airbyte_cdk.sources.declarative.models.declarative_component_schema import AuthFlow
20
14
  from airbyte_cdk.sources.declarative.transformations.config_transformations.config_transformation import (
21
15
  ConfigTransformation,
22
16
  )
23
17
  from airbyte_cdk.sources.declarative.validators.validator import Validator
24
18
  from airbyte_cdk.sources.message.repository import InMemoryMessageRepository, MessageRepository
25
- from airbyte_cdk.sources.source import Source
26
19
 
27
20
 
28
21
  @dataclass
@@ -48,7 +41,6 @@ class Spec:
48
41
  config_migrations: List[ConfigMigration] = field(default_factory=list)
49
42
  config_transformations: List[ConfigTransformation] = field(default_factory=list)
50
43
  config_validations: List[Validator] = field(default_factory=list)
51
- message_repository: MessageRepository = InMemoryMessageRepository()
52
44
 
53
45
  def generate_spec(self) -> ConnectorSpecification:
54
46
  """
@@ -69,34 +61,15 @@ class Spec:
69
61
  # We remap these keys to camel case because that's the existing format expected by the rest of the platform
70
62
  return ConnectorSpecificationSerializer.load(obj)
71
63
 
72
- def migrate_config(
73
- self, args: List[str], source: Source, config: MutableMapping[str, Any]
74
- ) -> None:
64
+ def migrate_config(self, config: MutableMapping[str, Any]) -> None:
75
65
  """
76
- Apply all specified config transformations to the provided config and save the modified config to the given path and emit a control message.
66
+ Apply all specified config transformations to the provided config and emit a control message.
77
67
 
78
- :param args: Command line arguments
79
- :param source: Source instance
80
68
  :param config: The user-provided config to migrate
81
69
  """
82
- config_path = AirbyteEntrypoint(source).extract_config(args)
83
-
84
- if not config_path:
85
- return
86
-
87
- mutable_config = dict(config)
88
70
  for migration in self.config_migrations:
89
71
  for transformation in migration.transformations:
90
- transformation.transform(mutable_config)
91
-
92
- if mutable_config != config:
93
- with open(config_path, "w") as f:
94
- json.dump(mutable_config, f)
95
- self.message_repository.emit_message(
96
- create_connector_config_control_message(mutable_config)
97
- )
98
- for message in self.message_repository.consume_queue():
99
- print(orjson.dumps(AirbyteMessageSerializer.dump(message)).decode())
72
+ transformation.transform(config)
100
73
 
101
74
  def transform_config(self, config: MutableMapping[str, Any]) -> None:
102
75
  """
@@ -107,7 +80,7 @@ class Spec:
107
80
  for transformation in self.config_transformations:
108
81
  transformation.transform(config)
109
82
 
110
- def validate_config(self, config: MutableMapping[str, Any]) -> None:
83
+ def validate_config(self, config: Mapping[str, Any]) -> None:
111
84
  """
112
85
  Apply all config validations to the provided config.
113
86
 
@@ -24,6 +24,7 @@ class YamlDeclarativeSource(ConcurrentDeclarativeSource[List[AirbyteStateMessage
24
24
  catalog: Optional[ConfiguredAirbyteCatalog] = None,
25
25
  config: Optional[Mapping[str, Any]] = None,
26
26
  state: Optional[List[AirbyteStateMessage]] = None,
27
+ config_path: Optional[str] = None,
27
28
  ) -> None:
28
29
  """
29
30
  :param path_to_yaml: Path to the yaml file describing the source
@@ -36,6 +37,7 @@ class YamlDeclarativeSource(ConcurrentDeclarativeSource[List[AirbyteStateMessage
36
37
  config=config or {},
37
38
  state=state or [],
38
39
  source_config=source_config,
40
+ config_path=config_path,
39
41
  )
40
42
 
41
43
  def _read_and_parse_yaml_file(self, path_to_yaml_file: str) -> ConnectionDefinition:
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: airbyte-cdk
3
- Version: 6.50.0
3
+ Version: 6.52.0
4
4
  Summary: A framework for writing Airbyte Connectors.
5
5
  Home-page: https://airbyte.com
6
6
  License: MIT
@@ -8,7 +8,7 @@ airbyte_cdk/cli/airbyte_cdk/_secrets.py,sha256=jLtpkhFJHavABN3UuqddeDRGS8v1iEj0e
8
8
  airbyte_cdk/cli/airbyte_cdk/_version.py,sha256=ohZNIktLFk91sdzqFW5idaNrZAPX2dIRnz---_fcKOE,352
9
9
  airbyte_cdk/cli/airbyte_cdk/exceptions.py,sha256=bsGmlWN6cXL2jCD1WYAZMqFmK1OLg2xLrcC_60KHSeA,803
10
10
  airbyte_cdk/cli/source_declarative_manifest/__init__.py,sha256=-0ST722Nj65bgRokzpzPkD1NBBW5CytEHFUe38cB86Q,91
11
- airbyte_cdk/cli/source_declarative_manifest/_run.py,sha256=3ZXAic1MJkDyjfOGvcmmkIDY5vAqsNPp8A7Z8xjzm-A,10829
11
+ airbyte_cdk/cli/source_declarative_manifest/_run.py,sha256=jo4yydSDrB157GtTSZPWRfhZHYvFUD_btI5LW7wOrmY,11129
12
12
  airbyte_cdk/cli/source_declarative_manifest/spec.json,sha256=Earc1L6ngcdIr514oFQlUoOxdF4RHqtUyStSIAquXdY,554
13
13
  airbyte_cdk/config_observation.py,sha256=7SSPxtN0nXPkm4euGNcTTr1iLbwUL01jy-24V1Hzde0,3986
14
14
  airbyte_cdk/connector.py,sha256=N6TUlrZOMjLAI85JrNAKkfyTqnO5xfBCw4oEfgjJd9o,4254
@@ -85,11 +85,11 @@ airbyte_cdk/sources/declarative/checks/check_stream.py,sha256=QeExVmpSYjr_CnghHu
85
85
  airbyte_cdk/sources/declarative/checks/connection_checker.py,sha256=MBRJo6WJlZQHpIfOGaNOkkHUmgUl_4wDM6VPo41z5Ss,1383
86
86
  airbyte_cdk/sources/declarative/concurrency_level/__init__.py,sha256=5XUqrmlstYlMM0j6crktlKQwALek0uiz2D3WdM46MyA,191
87
87
  airbyte_cdk/sources/declarative/concurrency_level/concurrency_level.py,sha256=YIwCTCpOr_QSNW4ltQK0yUGWInI8PKNY216HOOegYLk,2101
88
- airbyte_cdk/sources/declarative/concurrent_declarative_source.py,sha256=GoZJ8Oxb3IQh6iJao6dw7NNiWVyWTrlUlWVKMWt-V0k,28362
88
+ airbyte_cdk/sources/declarative/concurrent_declarative_source.py,sha256=OKor1mDDdJZz8kgiR0KI-_pWDjCF1nuDcto_fSYerW4,28442
89
89
  airbyte_cdk/sources/declarative/datetime/__init__.py,sha256=4Hw-PX1-VgESLF16cDdvuYCzGJtHntThLF4qIiULWeo,61
90
90
  airbyte_cdk/sources/declarative/datetime/datetime_parser.py,sha256=_zGNGq31RNy_0QBLt_EcTvgPyhj7urPdx6oA3M5-r3o,3150
91
91
  airbyte_cdk/sources/declarative/datetime/min_max_datetime.py,sha256=0BHBtDNQZfvwM45-tY5pNlTcKAFSGGNxemoi0Jic-0E,5785
92
- airbyte_cdk/sources/declarative/declarative_component_schema.yaml,sha256=ttq2s4aivIbvIj2OWy0-CNb2cVXYVTzOgXS9PLbMNWU,176666
92
+ airbyte_cdk/sources/declarative/declarative_component_schema.yaml,sha256=BQ0JKz4K8BKCpKp3C2IumJj2y2nv29lheeJUxPX6gXU,177389
93
93
  airbyte_cdk/sources/declarative/declarative_source.py,sha256=qmyMnnet92eGc3C22yBtpvD5UZjqdhsAafP_zxI5wp8,1814
94
94
  airbyte_cdk/sources/declarative/declarative_stream.py,sha256=dCRlddBUSaJmBNBz1pSO1r2rTw8AP5d2_vlmIeGs2gg,10767
95
95
  airbyte_cdk/sources/declarative/decoders/__init__.py,sha256=JHb_0d3SE6kNY10mxA5YBEKPeSbsWYjByq1gUQxepoE,953
@@ -127,20 +127,20 @@ airbyte_cdk/sources/declarative/interpolation/interpolated_string.py,sha256=CQkH
127
127
  airbyte_cdk/sources/declarative/interpolation/interpolation.py,sha256=9IoeuWam3L6GyN10L6U8xNWXmkt9cnahSDNkez1OmFY,982
128
128
  airbyte_cdk/sources/declarative/interpolation/jinja.py,sha256=UQeuS4Vpyp4hlOn-R3tRyeBX0e9IoV6jQ6gH-Jz8lY0,7182
129
129
  airbyte_cdk/sources/declarative/interpolation/macros.py,sha256=UYSJ5gW7TkHALYnNvUnRP3RlyGwGuRMObF3BHuNzjJM,5320
130
- airbyte_cdk/sources/declarative/manifest_declarative_source.py,sha256=cZNUOeIogrCmCS7RXeJqQIlnsANigz1cngpLko02M2g,23191
130
+ airbyte_cdk/sources/declarative/manifest_declarative_source.py,sha256=aS_YxqUWgxyfyzwAZpV736RFg-m5a0sn-V7PPMBGVr4,24660
131
131
  airbyte_cdk/sources/declarative/migrations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
132
132
  airbyte_cdk/sources/declarative/migrations/legacy_to_per_partition_state_migration.py,sha256=V2lpYE9LJKvz6BUViHk4vaRGndxNABmPbDCtyYdkqaE,4013
133
133
  airbyte_cdk/sources/declarative/migrations/state_migration.py,sha256=KWPjealMLKSMtajXgkdGgKg7EmTLR-CqqD7UIh0-eDU,794
134
134
  airbyte_cdk/sources/declarative/models/__init__.py,sha256=nUFxNCiKeYRVXuZEKA7GD-lTHxsiKcQ8FitZjKhPIvE,100
135
135
  airbyte_cdk/sources/declarative/models/base_model_with_deprecations.py,sha256=Imnj3yef0aqRdLfaUxkIYISUb8YkiPrRH_wBd-x8HjM,5999
136
- airbyte_cdk/sources/declarative/models/declarative_component_schema.py,sha256=YhQl_vjQyWl1GJTuPyhg1aKWQazbqHROZ5v0dIg7auE,125123
136
+ airbyte_cdk/sources/declarative/models/declarative_component_schema.py,sha256=XVfBPgNBEEkv8u6Bj1JRiXsQWa_QoipPHFxND_YsxDY,125766
137
137
  airbyte_cdk/sources/declarative/parsers/__init__.py,sha256=ZnqYNxHsKCgO38IwB34RQyRMXTs4GTvlRi3ImKnIioo,61
138
138
  airbyte_cdk/sources/declarative/parsers/custom_code_compiler.py,sha256=nlVvHC511NUyDEEIRBkoeDTAvLqKNp-hRy8D19z8tdk,5941
139
139
  airbyte_cdk/sources/declarative/parsers/custom_exceptions.py,sha256=wnRUP0Xeru9Rbu5OexXSDN9QWDo8YU4tT9M2LDVOgGA,802
140
140
  airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py,sha256=2UdpCz3yi7ISZTyqkQXSSy3dMxeyOWqV7OlAS5b9GVg,11568
141
141
  airbyte_cdk/sources/declarative/parsers/manifest_normalizer.py,sha256=laBy7ebjA-PiNwc-50U4FHvMqS_mmHvnabxgFs4CjGw,17069
142
142
  airbyte_cdk/sources/declarative/parsers/manifest_reference_resolver.py,sha256=pJmg78vqE5VfUrF_KJnWjucQ4k9IWFULeAxHCowrHXE,6806
143
- airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py,sha256=j7MUNaWDpsxe5yDzE3-id_wivjNrSjsJa1VgE4oSRlo,174401
143
+ airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py,sha256=7-q8icBPnEHbPa0vonz5KjsalYrrqJasIcLXK1TITlc,174730
144
144
  airbyte_cdk/sources/declarative/partition_routers/__init__.py,sha256=TBC9AkGaUqHm2IKHMPN6punBIcY5tWGULowcLoAVkfw,1109
145
145
  airbyte_cdk/sources/declarative/partition_routers/async_job_partition_router.py,sha256=VelO7zKqKtzMJ35jyFeg0ypJLQC0plqqIBNXoBW1G2E,3001
146
146
  airbyte_cdk/sources/declarative/partition_routers/cartesian_product_stream_slicer.py,sha256=c5cuVFM6NFkuQqG8Z5IwkBuwDrvXZN1CunUOM_L0ezg,6892
@@ -194,8 +194,8 @@ airbyte_cdk/sources/declarative/requesters/request_options/request_options_provi
194
194
  airbyte_cdk/sources/declarative/requesters/request_path.py,sha256=S3MeFvcaQrMbOkSY2W2VbXLNomqt_3eXqVd9ZhgNwUs,299
195
195
  airbyte_cdk/sources/declarative/requesters/requester.py,sha256=T6tMx_Bx4iT-0YVjY7IzgRil-gaIu9n01b1iwpTh3Ek,5516
196
196
  airbyte_cdk/sources/declarative/resolvers/__init__.py,sha256=NiDcz5qi8HPsfX94MUmnX0Rgs_kQXGvucOmJjNWlxKQ,1207
197
- airbyte_cdk/sources/declarative/resolvers/components_resolver.py,sha256=KPjKc0yb9artL4ZkeqN8RmEykHH6FJgqXD7fCEnh1X0,1936
198
- airbyte_cdk/sources/declarative/resolvers/config_components_resolver.py,sha256=dz4iJV9liD_LzY_Mn4XmAStoUll60R3MIGWV4aN3pgg,5223
197
+ airbyte_cdk/sources/declarative/resolvers/components_resolver.py,sha256=oJIpy66ep8n-QOc8GwpllApTRcl4QtQhkTw5fWWra2w,2026
198
+ airbyte_cdk/sources/declarative/resolvers/config_components_resolver.py,sha256=sD3N7nmqDjLsau8P2DE7DYOHdFTYjC_2nIB6454BNYk,7556
199
199
  airbyte_cdk/sources/declarative/resolvers/http_components_resolver.py,sha256=AiojNs8wItJFrENZBFUaDvau3sgwudO6Wkra36upSPo,4639
200
200
  airbyte_cdk/sources/declarative/retrievers/__init__.py,sha256=nQepwG_RfW53sgwvK5dLPqfCx0VjsQ83nYoPjBMAaLM,527
201
201
  airbyte_cdk/sources/declarative/retrievers/async_retriever.py,sha256=6oZtnCHm9NdDvjTSrVwPQOXGSdETSIR7eWH2vFjM7jI,4855
@@ -216,7 +216,7 @@ airbyte_cdk/sources/declarative/schema/inline_schema_loader.py,sha256=bVETE10hRs
216
216
  airbyte_cdk/sources/declarative/schema/json_file_schema_loader.py,sha256=5Wl-fqW-pVf_dxJ4yGHMAFfC4JjKHYJhqFJT1xA57F4,4177
217
217
  airbyte_cdk/sources/declarative/schema/schema_loader.py,sha256=kjt8v0N5wWKA5zyLnrDLxf1PJKdUqvQq2RVnAOAzNSY,379
218
218
  airbyte_cdk/sources/declarative/spec/__init__.py,sha256=9FYO-fVOclrwjAW4qwRTbZRVopTc9rOaauAJfThdNCQ,177
219
- airbyte_cdk/sources/declarative/spec/spec.py,sha256=eOdIh_Jzlq0Tbmgrx4TSR-6kBxl7nu9F-I-zHBcS0JI,4734
219
+ airbyte_cdk/sources/declarative/spec/spec.py,sha256=SwL_pfXZgcLYLJY-MAeFMHug9oYh2tOWjgG0C3DoLOY,3602
220
220
  airbyte_cdk/sources/declarative/stream_slicers/__init__.py,sha256=sI9vhc95RwJYOnA0VKjcbtKgFcmAbWjhdWBXFbAijOs,176
221
221
  airbyte_cdk/sources/declarative/stream_slicers/declarative_partition_generator.py,sha256=cjKGm4r438dd1GxrFHJ4aYrdzG2bkncnwaWxAwlXR3M,3585
222
222
  airbyte_cdk/sources/declarative/stream_slicers/stream_slicer.py,sha256=SOkIPBi2Wu7yxIvA15yFzUAB95a3IzA8LPq5DEqHQQc,725
@@ -241,7 +241,7 @@ airbyte_cdk/sources/declarative/validators/predicate_validator.py,sha256=Q4eVncl
241
241
  airbyte_cdk/sources/declarative/validators/validate_adheres_to_schema.py,sha256=kjcuKxWMJEzpF4GiESITGMxBAXw6YZCAsgOQMgeBo4g,1085
242
242
  airbyte_cdk/sources/declarative/validators/validation_strategy.py,sha256=LwqUX89cFdHTM1-h6c8vebBA9WC38HYoGBvJfCZHr0g,467
243
243
  airbyte_cdk/sources/declarative/validators/validator.py,sha256=MAwo8OievUsuzBuPxI9pbPu87yq0tJZkGbydcrHZyQc,382
244
- airbyte_cdk/sources/declarative/yaml_declarative_source.py,sha256=nJCZkzLGP-dwvfwKsl4VqQFZQdhx6fiGCRez1gma0wE,2714
244
+ airbyte_cdk/sources/declarative/yaml_declarative_source.py,sha256=-pHwGO7ZW-x8lmsqSpbrN0pOgIyjJhFDGUNwB3kQWWc,2794
245
245
  airbyte_cdk/sources/file_based/README.md,sha256=iMqww4VZ882jfNQIdljjDgqreKs-mkdtSrRKA94iX6A,11085
246
246
  airbyte_cdk/sources/file_based/__init__.py,sha256=EaxHv_9ot-eRlUCR47ZMZ0IOtB-n0HH24om7Bfn-uuQ,868
247
247
  airbyte_cdk/sources/file_based/availability_strategy/__init__.py,sha256=ddKQfUmk-Ls7LJaG8gtrqDybG3d8S7KXOAEjLeYLrTg,399
@@ -419,9 +419,9 @@ airbyte_cdk/utils/slice_hasher.py,sha256=EDxgROHDbfG-QKQb59m7h_7crN1tRiawdf5uU7G
419
419
  airbyte_cdk/utils/spec_schema_transformations.py,sha256=-5HTuNsnDBAhj-oLeQXwpTGA0HdcjFOf2zTEMUTTg_Y,816
420
420
  airbyte_cdk/utils/stream_status_utils.py,sha256=ZmBoiy5HVbUEHAMrUONxZvxnvfV9CesmQJLDTAIWnWw,1171
421
421
  airbyte_cdk/utils/traced_exception.py,sha256=C8uIBuCL_E4WnBAOPSxBicD06JAldoN9fGsQDp463OY,6292
422
- airbyte_cdk-6.50.0.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
423
- airbyte_cdk-6.50.0.dist-info/LICENSE_SHORT,sha256=aqF6D1NcESmpn-cqsxBtszTEnHKnlsp8L4x9wAh3Nxg,55
424
- airbyte_cdk-6.50.0.dist-info/METADATA,sha256=f8KKwJtpEpfmbpMDfbO4gBZRuNEp95A3YNHSrza7nkM,6343
425
- airbyte_cdk-6.50.0.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
426
- airbyte_cdk-6.50.0.dist-info/entry_points.txt,sha256=AKWbEkHfpzzk9nF9tqBUaw1MbvTM4mGtEzmZQm0ZWvM,139
427
- airbyte_cdk-6.50.0.dist-info/RECORD,,
422
+ airbyte_cdk-6.52.0.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
423
+ airbyte_cdk-6.52.0.dist-info/LICENSE_SHORT,sha256=aqF6D1NcESmpn-cqsxBtszTEnHKnlsp8L4x9wAh3Nxg,55
424
+ airbyte_cdk-6.52.0.dist-info/METADATA,sha256=d5e43kQPtd5tZNY-xH1y7PrUdsmFC-sqiC8pDEajuyk,6343
425
+ airbyte_cdk-6.52.0.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
426
+ airbyte_cdk-6.52.0.dist-info/entry_points.txt,sha256=AKWbEkHfpzzk9nF9tqBUaw1MbvTM4mGtEzmZQm0ZWvM,139
427
+ airbyte_cdk-6.52.0.dist-info/RECORD,,