airbyte-cdk 6.13.1.dev4108__py3-none-any.whl → 6.13.1.dev41012__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (33) hide show
  1. airbyte_cdk/entrypoint.py +1 -13
  2. airbyte_cdk/sources/declarative/concurrent_declarative_source.py +51 -24
  3. airbyte_cdk/sources/declarative/declarative_component_schema.yaml +1 -77
  4. airbyte_cdk/sources/declarative/extractors/__init__.py +0 -2
  5. airbyte_cdk/sources/declarative/extractors/record_selector.py +7 -5
  6. airbyte_cdk/sources/declarative/interpolation/macros.py +0 -21
  7. airbyte_cdk/sources/declarative/models/declarative_component_schema.py +1 -45
  8. airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +13 -46
  9. airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py +35 -52
  10. airbyte_cdk/sources/declarative/requesters/paginators/no_pagination.py +7 -10
  11. airbyte_cdk/sources/declarative/requesters/paginators/paginator.py +4 -9
  12. airbyte_cdk/sources/declarative/requesters/paginators/strategies/cursor_pagination_strategy.py +6 -11
  13. airbyte_cdk/sources/declarative/requesters/paginators/strategies/offset_increment.py +11 -13
  14. airbyte_cdk/sources/declarative/requesters/paginators/strategies/page_increment.py +13 -14
  15. airbyte_cdk/sources/declarative/requesters/paginators/strategies/pagination_strategy.py +7 -6
  16. airbyte_cdk/sources/declarative/requesters/paginators/strategies/stop_condition.py +7 -10
  17. airbyte_cdk/sources/declarative/retrievers/async_retriever.py +4 -1
  18. airbyte_cdk/sources/declarative/retrievers/simple_retriever.py +64 -71
  19. airbyte_cdk/sources/declarative/stream_slicers/declarative_partition_generator.py +4 -4
  20. airbyte_cdk/sources/declarative/transformations/flatten_fields.py +1 -3
  21. airbyte_cdk/sources/file_based/config/abstract_file_based_spec.py +3 -8
  22. airbyte_cdk/sources/file_based/exceptions.py +23 -31
  23. airbyte_cdk/sources/file_based/file_based_source.py +8 -17
  24. airbyte_cdk/sources/file_based/file_based_stream_reader.py +6 -7
  25. airbyte_cdk/sources/file_based/file_types/unstructured_parser.py +2 -22
  26. airbyte_cdk/sources/file_based/stream/default_file_based_stream.py +20 -10
  27. {airbyte_cdk-6.13.1.dev4108.dist-info → airbyte_cdk-6.13.1.dev41012.dist-info}/METADATA +2 -2
  28. {airbyte_cdk-6.13.1.dev4108.dist-info → airbyte_cdk-6.13.1.dev41012.dist-info}/RECORD +31 -33
  29. {airbyte_cdk-6.13.1.dev4108.dist-info → airbyte_cdk-6.13.1.dev41012.dist-info}/WHEEL +1 -1
  30. airbyte_cdk/sources/declarative/extractors/type_transformer.py +0 -55
  31. airbyte_cdk/sources/declarative/transformations/keys_replace_transformation.py +0 -61
  32. {airbyte_cdk-6.13.1.dev4108.dist-info → airbyte_cdk-6.13.1.dev41012.dist-info}/LICENSE.txt +0 -0
  33. {airbyte_cdk-6.13.1.dev4108.dist-info → airbyte_cdk-6.13.1.dev41012.dist-info}/entry_points.txt +0 -0
airbyte_cdk/entrypoint.py CHANGED
@@ -5,7 +5,6 @@
5
5
  import argparse
6
6
  import importlib
7
7
  import ipaddress
8
- import json
9
8
  import logging
10
9
  import os.path
11
10
  import socket
@@ -47,7 +46,6 @@ logger = init_logger("airbyte")
47
46
 
48
47
  VALID_URL_SCHEMES = ["https"]
49
48
  CLOUD_DEPLOYMENT_MODE = "cloud"
50
- _HAS_LOGGED_FOR_SERIALIZATION_ERROR = False
51
49
 
52
50
 
53
51
  class AirbyteEntrypoint(object):
@@ -293,17 +291,7 @@ class AirbyteEntrypoint(object):
293
291
 
294
292
  @staticmethod
295
293
  def airbyte_message_to_string(airbyte_message: AirbyteMessage) -> str:
296
- global _HAS_LOGGED_FOR_SERIALIZATION_ERROR
297
- serialized_message = AirbyteMessageSerializer.dump(airbyte_message)
298
- try:
299
- return orjson.dumps(serialized_message).decode()
300
- except Exception as exception:
301
- if not _HAS_LOGGED_FOR_SERIALIZATION_ERROR:
302
- logger.warning(
303
- f"There was an error during the serialization of an AirbyteMessage: `{exception}`. This might impact the sync performances."
304
- )
305
- _HAS_LOGGED_FOR_SERIALIZATION_ERROR = True
306
- return json.dumps(serialized_message)
294
+ return orjson.dumps(AirbyteMessageSerializer.dump(airbyte_message)).decode()
307
295
 
308
296
  @classmethod
309
297
  def extract_state(cls, args: List[str]) -> Optional[Any]:
@@ -3,7 +3,7 @@
3
3
  #
4
4
 
5
5
  import logging
6
- from typing import Any, Generic, Iterator, List, Mapping, Optional, Tuple
6
+ from typing import Any, Callable, Generic, Iterator, List, Mapping, Optional, Tuple, Union
7
7
 
8
8
  from airbyte_cdk.models import (
9
9
  AirbyteCatalog,
@@ -28,11 +28,15 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import
28
28
  from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
29
29
  DatetimeBasedCursor as DatetimeBasedCursorModel,
30
30
  )
31
+ from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
32
+ DeclarativeStream as DeclarativeStreamModel,
33
+ )
31
34
  from airbyte_cdk.sources.declarative.parsers.model_to_component_factory import (
35
+ ComponentDefinition,
32
36
  ModelToComponentFactory,
33
37
  )
34
38
  from airbyte_cdk.sources.declarative.requesters import HttpRequester
35
- from airbyte_cdk.sources.declarative.retrievers import SimpleRetriever
39
+ from airbyte_cdk.sources.declarative.retrievers import Retriever, SimpleRetriever
36
40
  from airbyte_cdk.sources.declarative.stream_slicers.declarative_partition_generator import (
37
41
  DeclarativePartitionFactory,
38
42
  StreamSlicerPartitionGenerator,
@@ -48,6 +52,7 @@ from airbyte_cdk.sources.streams.concurrent.availability_strategy import (
48
52
  from airbyte_cdk.sources.streams.concurrent.cursor import FinalStateCursor
49
53
  from airbyte_cdk.sources.streams.concurrent.default_stream import DefaultStream
50
54
  from airbyte_cdk.sources.streams.concurrent.helpers import get_primary_key_from_stream
55
+ from airbyte_cdk.sources.types import Config, StreamState
51
56
 
52
57
 
53
58
  class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
@@ -189,11 +194,10 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
189
194
  # Some low-code sources use a combination of DeclarativeStream and regular Python streams. We can't inspect
190
195
  # these legacy Python streams the way we do low-code streams to determine if they are concurrent compatible,
191
196
  # so we need to treat them as synchronous
192
- if isinstance(declarative_stream, DeclarativeStream) and (
193
- name_to_stream_mapping[declarative_stream.name]["retriever"]["type"]
197
+ if (
198
+ isinstance(declarative_stream, DeclarativeStream)
199
+ and name_to_stream_mapping[declarative_stream.name]["retriever"]["type"]
194
200
  == "SimpleRetriever"
195
- or name_to_stream_mapping[declarative_stream.name]["retriever"]["type"]
196
- == "AsyncRetriever"
197
201
  ):
198
202
  incremental_sync_component_definition = name_to_stream_mapping[
199
203
  declarative_stream.name
@@ -230,27 +234,15 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
230
234
  stream_state=stream_state,
231
235
  )
232
236
 
233
- retriever = declarative_stream.retriever
234
-
235
- # This is an optimization so that we don't invoke any cursor or state management flows within the
236
- # low-code framework because state management is handled through the ConcurrentCursor.
237
- if declarative_stream and isinstance(retriever, SimpleRetriever):
238
- # Also a temporary hack. In the legacy Stream implementation, as part of the read,
239
- # set_initial_state() is called to instantiate incoming state on the cursor. Although we no
240
- # longer rely on the legacy low-code cursor for concurrent checkpointing, low-code components
241
- # like StopConditionPaginationStrategyDecorator and ClientSideIncrementalRecordFilterDecorator
242
- # still rely on a DatetimeBasedCursor that is properly initialized with state.
243
- if retriever.cursor:
244
- retriever.cursor.set_initial_state(stream_state=stream_state)
245
- # We zero it out here, but since this is a cursor reference, the state is still properly
246
- # instantiated for the other components that reference it
247
- retriever.cursor = None
248
-
249
237
  partition_generator = StreamSlicerPartitionGenerator(
250
238
  DeclarativePartitionFactory(
251
239
  declarative_stream.name,
252
240
  declarative_stream.get_json_schema(),
253
- retriever,
241
+ self._retriever_factory(
242
+ name_to_stream_mapping[declarative_stream.name],
243
+ config,
244
+ stream_state,
245
+ ),
254
246
  self.message_repository,
255
247
  ),
256
248
  cursor,
@@ -280,7 +272,11 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
280
272
  DeclarativePartitionFactory(
281
273
  declarative_stream.name,
282
274
  declarative_stream.get_json_schema(),
283
- declarative_stream.retriever,
275
+ self._retriever_factory(
276
+ name_to_stream_mapping[declarative_stream.name],
277
+ config,
278
+ {},
279
+ ),
284
280
  self.message_repository,
285
281
  ),
286
282
  declarative_stream.retriever.stream_slicer,
@@ -419,3 +415,34 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
419
415
  if stream.stream.name not in concurrent_stream_names
420
416
  ]
421
417
  )
418
+
419
+ def _retriever_factory(
420
+ self, stream_config: ComponentDefinition, source_config: Config, stream_state: StreamState
421
+ ) -> Callable[[], Retriever]:
422
+ def _factory_method() -> Retriever:
423
+ declarative_stream: DeclarativeStream = self._constructor.create_component(
424
+ DeclarativeStreamModel,
425
+ stream_config,
426
+ source_config,
427
+ emit_connector_builder_messages=self._emit_connector_builder_messages,
428
+ )
429
+
430
+ # This is an optimization so that we don't invoke any cursor or state management flows within the
431
+ # low-code framework because state management is handled through the ConcurrentCursor.
432
+ if (
433
+ declarative_stream
434
+ and declarative_stream.retriever
435
+ and isinstance(declarative_stream.retriever, SimpleRetriever)
436
+ ):
437
+ # Also a temporary hack. In the legacy Stream implementation, as part of the read, set_initial_state() is
438
+ # called to instantiate incoming state on the cursor. Although we no longer rely on the legacy low-code cursor
439
+ # for concurrent checkpointing, low-code components like StopConditionPaginationStrategyDecorator and
440
+ # ClientSideIncrementalRecordFilterDecorator still rely on a DatetimeBasedCursor that is properly initialized
441
+ # with state.
442
+ if declarative_stream.retriever.cursor:
443
+ declarative_stream.retriever.cursor.set_initial_state(stream_state=stream_state)
444
+ declarative_stream.retriever.cursor = None
445
+
446
+ return declarative_stream.retriever
447
+
448
+ return _factory_method
@@ -667,28 +667,6 @@ definitions:
667
667
  $parameters:
668
668
  type: object
669
669
  additionalProperties: true
670
- CustomSchemaNormalization:
671
- title: Custom Schema Normalization
672
- description: Schema normalization component whose behavior is derived from a custom code implementation of the connector.
673
- type: object
674
- additionalProperties: true
675
- required:
676
- - type
677
- - class_name
678
- properties:
679
- type:
680
- type: string
681
- enum: [ CustomSchemaNormalization ]
682
- class_name:
683
- title: Class Name
684
- description: Fully-qualified name of the class that will be implementing the custom normalization. The format is `source_<name>.<package>.<class_name>`.
685
- type: string
686
- additionalProperties: true
687
- examples:
688
- - "source_amazon_seller_partner.components.LedgerDetailedViewReportsTypeTransformer"
689
- $parameters:
690
- type: object
691
- additionalProperties: true
692
670
  CustomStateMigration:
693
671
  title: Custom State Migration
694
672
  description: Apply a custom transformation on the input state.
@@ -1263,7 +1241,6 @@ definitions:
1263
1241
  - "$ref": "#/definitions/KeysToLower"
1264
1242
  - "$ref": "#/definitions/KeysToSnakeCase"
1265
1243
  - "$ref": "#/definitions/FlattenFields"
1266
- - "$ref": "#/definitions/KeysReplace"
1267
1244
  state_migrations:
1268
1245
  title: State Migrations
1269
1246
  description: Array of state migrations to be applied on the input state
@@ -1808,7 +1785,6 @@ definitions:
1808
1785
  - "$ref": "#/definitions/KeysToLower"
1809
1786
  - "$ref": "#/definitions/KeysToSnakeCase"
1810
1787
  - "$ref": "#/definitions/FlattenFields"
1811
- - "$ref": "#/definitions/KeysReplace"
1812
1788
  schema_type_identifier:
1813
1789
  "$ref": "#/definitions/SchemaTypeIdentifier"
1814
1790
  $parameters:
@@ -1904,54 +1880,6 @@ definitions:
1904
1880
  type:
1905
1881
  type: string
1906
1882
  enum: [FlattenFields]
1907
- flatten_lists:
1908
- title: Flatten Lists
1909
- description: Whether to flatten lists or leave it as is. Default is True.
1910
- type: boolean
1911
- default: true
1912
- $parameters:
1913
- type: object
1914
- additionalProperties: true
1915
- KeysReplace:
1916
- title: Keys Replace
1917
- description: A transformation that replaces symbols in keys.
1918
- type: object
1919
- required:
1920
- - type
1921
- - old
1922
- - new
1923
- properties:
1924
- type:
1925
- type: string
1926
- enum: [KeysReplace]
1927
- old:
1928
- type: string
1929
- title: Old value
1930
- description: Old value to replace.
1931
- examples:
1932
- - " "
1933
- - "{{ record.id }}"
1934
- - "{{ config['id'] }}"
1935
- - "{{ stream_slice['id'] }}"
1936
- interpolation_context:
1937
- - config
1938
- - record
1939
- - stream_state
1940
- - stream_slice
1941
- new:
1942
- type: string
1943
- title: New value
1944
- description: New value to set.
1945
- examples:
1946
- - "_"
1947
- - "{{ record.id }}"
1948
- - "{{ config['id'] }}"
1949
- - "{{ stream_slice['id'] }}"
1950
- interpolation_context:
1951
- - config
1952
- - record
1953
- - stream_state
1954
- - stream_slice
1955
1883
  $parameters:
1956
1884
  type: object
1957
1885
  additionalProperties: true
@@ -2627,11 +2555,7 @@ definitions:
2627
2555
  - "$ref": "#/definitions/CustomRecordFilter"
2628
2556
  - "$ref": "#/definitions/RecordFilter"
2629
2557
  schema_normalization:
2630
- title: Schema Normalization
2631
- description: Responsible for normalization according to the schema.
2632
- anyOf:
2633
- - "$ref": "#/definitions/SchemaNormalization"
2634
- - "$ref": "#/definitions/CustomSchemaNormalization"
2558
+ "$ref": "#/definitions/SchemaNormalization"
2635
2559
  default: None
2636
2560
  $parameters:
2637
2561
  type: object
@@ -9,10 +9,8 @@ from airbyte_cdk.sources.declarative.extractors.record_selector import RecordSel
9
9
  from airbyte_cdk.sources.declarative.extractors.response_to_file_extractor import (
10
10
  ResponseToFileExtractor,
11
11
  )
12
- from airbyte_cdk.sources.declarative.extractors.type_transformer import TypeTransformer
13
12
 
14
13
  __all__ = [
15
- "TypeTransformer",
16
14
  "HttpSelector",
17
15
  "DpathExtractor",
18
16
  "RecordFilter",
@@ -10,14 +10,16 @@ import requests
10
10
  from airbyte_cdk.sources.declarative.extractors.http_selector import HttpSelector
11
11
  from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor
12
12
  from airbyte_cdk.sources.declarative.extractors.record_filter import RecordFilter
13
- from airbyte_cdk.sources.declarative.extractors.type_transformer import (
14
- TypeTransformer as DeclarativeTypeTransformer,
15
- )
16
13
  from airbyte_cdk.sources.declarative.interpolation import InterpolatedString
17
14
  from airbyte_cdk.sources.declarative.models import SchemaNormalization
18
15
  from airbyte_cdk.sources.declarative.transformations import RecordTransformation
19
16
  from airbyte_cdk.sources.types import Config, Record, StreamSlice, StreamState
20
- from airbyte_cdk.sources.utils.transform import TypeTransformer
17
+ from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer
18
+
19
+ SCHEMA_TRANSFORMER_TYPE_MAPPING = {
20
+ SchemaNormalization.None_: TransformConfig.NoTransform,
21
+ SchemaNormalization.Default: TransformConfig.DefaultSchemaNormalization,
22
+ }
21
23
 
22
24
 
23
25
  @dataclass
@@ -36,7 +38,7 @@ class RecordSelector(HttpSelector):
36
38
  extractor: RecordExtractor
37
39
  config: Config
38
40
  parameters: InitVar[Mapping[str, Any]]
39
- schema_normalization: Union[TypeTransformer, DeclarativeTypeTransformer]
41
+ schema_normalization: TypeTransformer
40
42
  name: str
41
43
  _name: Union[InterpolatedString, str] = field(init=False, repr=False, default="")
42
44
  record_filter: Optional[RecordFilter] = None
@@ -94,26 +94,6 @@ def max(*args: typing.Any) -> typing.Any:
94
94
  return builtins.max(*args)
95
95
 
96
96
 
97
- def min(*args: typing.Any) -> typing.Any:
98
- """
99
- Returns smallest object of an iterable, or two or more arguments.
100
-
101
- min(iterable, *[, default=obj, key=func]) -> value
102
- min(arg1, arg2, *args, *[, key=func]) -> value
103
-
104
- Usage:
105
- `"{{ min(2,3) }}"
106
-
107
- With a single iterable argument, return its smallest item. The
108
- default keyword-only argument specifies an object to return if
109
- the provided iterable is empty.
110
- With two or more arguments, return the smallest argument.
111
- :param args: args to compare
112
- :return: smallest argument
113
- """
114
- return builtins.min(*args)
115
-
116
-
117
97
  def day_delta(num_days: int, format: str = "%Y-%m-%dT%H:%M:%S.%f%z") -> str:
118
98
  """
119
99
  Returns datetime of now() + num_days
@@ -167,7 +147,6 @@ _macros_list = [
167
147
  today_utc,
168
148
  timestamp,
169
149
  max,
170
- min,
171
150
  day_delta,
172
151
  duration,
173
152
  format_datetime,
@@ -268,22 +268,6 @@ class CustomSchemaLoader(BaseModel):
268
268
  parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
269
269
 
270
270
 
271
- class CustomSchemaNormalization(BaseModel):
272
- class Config:
273
- extra = Extra.allow
274
-
275
- type: Literal["CustomSchemaNormalization"]
276
- class_name: str = Field(
277
- ...,
278
- description="Fully-qualified name of the class that will be implementing the custom normalization. The format is `source_<name>.<package>.<class_name>`.",
279
- examples=[
280
- "source_amazon_seller_partner.components.LedgerDetailedViewReportsTypeTransformer"
281
- ],
282
- title="Class Name",
283
- )
284
- parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
285
-
286
-
287
271
  class CustomStateMigration(BaseModel):
288
272
  class Config:
289
273
  extra = Extra.allow
@@ -737,30 +721,8 @@ class KeysToSnakeCase(BaseModel):
737
721
  parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
738
722
 
739
723
 
740
- class KeysReplace(BaseModel):
741
- type: Literal["KeysReplace"]
742
- old: str = Field(
743
- ...,
744
- description="Old value to replace.",
745
- examples=[" ", "{{ record.id }}", "{{ config['id'] }}", "{{ stream_slice['id'] }}"],
746
- title="Old value",
747
- )
748
- new: str = Field(
749
- ...,
750
- description="New value to set.",
751
- examples=["_", "{{ record.id }}", "{{ config['id'] }}", "{{ stream_slice['id'] }}"],
752
- title="New value",
753
- )
754
- parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
755
-
756
-
757
724
  class FlattenFields(BaseModel):
758
725
  type: Literal["FlattenFields"]
759
- flatten_lists: Optional[bool] = Field(
760
- True,
761
- description="Whether to flatten lists or leave it as is. Default is True.",
762
- title="Flatten Lists",
763
- )
764
726
  parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
765
727
 
766
728
 
@@ -1551,11 +1513,7 @@ class RecordSelector(BaseModel):
1551
1513
  description="Responsible for filtering records to be emitted by the Source.",
1552
1514
  title="Record Filter",
1553
1515
  )
1554
- schema_normalization: Optional[Union[SchemaNormalization, CustomSchemaNormalization]] = Field(
1555
- SchemaNormalization.None_,
1556
- description="Responsible for normalization according to the schema.",
1557
- title="Schema Normalization",
1558
- )
1516
+ schema_normalization: Optional[SchemaNormalization] = SchemaNormalization.None_
1559
1517
  parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
1560
1518
 
1561
1519
 
@@ -1743,7 +1701,6 @@ class DeclarativeStream(BaseModel):
1743
1701
  KeysToLower,
1744
1702
  KeysToSnakeCase,
1745
1703
  FlattenFields,
1746
- KeysReplace,
1747
1704
  ]
1748
1705
  ]
1749
1706
  ] = Field(
@@ -1918,7 +1875,6 @@ class DynamicSchemaLoader(BaseModel):
1918
1875
  KeysToLower,
1919
1876
  KeysToSnakeCase,
1920
1877
  FlattenFields,
1921
- KeysReplace,
1922
1878
  ]
1923
1879
  ]
1924
1880
  ] = Field(
@@ -82,6 +82,9 @@ from airbyte_cdk.sources.declarative.extractors import (
82
82
  from airbyte_cdk.sources.declarative.extractors.record_filter import (
83
83
  ClientSideIncrementalRecordFilterDecorator,
84
84
  )
85
+ from airbyte_cdk.sources.declarative.extractors.record_selector import (
86
+ SCHEMA_TRANSFORMER_TYPE_MAPPING,
87
+ )
85
88
  from airbyte_cdk.sources.declarative.incremental import (
86
89
  ChildPartitionResumableFullRefreshCursor,
87
90
  CursorFactory,
@@ -97,9 +100,7 @@ from airbyte_cdk.sources.declarative.interpolation.interpolated_mapping import I
97
100
  from airbyte_cdk.sources.declarative.migrations.legacy_to_per_partition_state_migration import (
98
101
  LegacyToPerPartitionStateMigration,
99
102
  )
100
- from airbyte_cdk.sources.declarative.models import (
101
- CustomStateMigration,
102
- )
103
+ from airbyte_cdk.sources.declarative.models import CustomStateMigration
103
104
  from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
104
105
  AddedFieldDefinition as AddedFieldDefinitionModel,
105
106
  )
@@ -184,9 +185,6 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import
184
185
  from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
185
186
  CustomSchemaLoader as CustomSchemaLoader,
186
187
  )
187
- from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
188
- CustomSchemaNormalization as CustomSchemaNormalizationModel,
189
- )
190
188
  from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
191
189
  CustomTransformation as CustomTransformationModel,
192
190
  )
@@ -256,9 +254,6 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import
256
254
  from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
257
255
  JwtPayload as JwtPayloadModel,
258
256
  )
259
- from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
260
- KeysReplace as KeysReplaceModel,
261
- )
262
257
  from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
263
258
  KeysToLower as KeysToLowerModel,
264
259
  )
@@ -313,9 +308,6 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import
313
308
  from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
314
309
  ResponseToFileExtractor as ResponseToFileExtractorModel,
315
310
  )
316
- from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
317
- SchemaNormalization as SchemaNormalizationModel,
318
- )
319
311
  from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
320
312
  SchemaTypeIdentifier as SchemaTypeIdentifierModel,
321
313
  )
@@ -425,9 +417,6 @@ from airbyte_cdk.sources.declarative.transformations.add_fields import AddedFiel
425
417
  from airbyte_cdk.sources.declarative.transformations.flatten_fields import (
426
418
  FlattenFields,
427
419
  )
428
- from airbyte_cdk.sources.declarative.transformations.keys_replace_transformation import (
429
- KeysReplaceTransformation,
430
- )
431
420
  from airbyte_cdk.sources.declarative.transformations.keys_to_lower_transformation import (
432
421
  KeysToLowerTransformation,
433
422
  )
@@ -450,11 +439,6 @@ from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer
450
439
 
451
440
  ComponentDefinition = Mapping[str, Any]
452
441
 
453
- SCHEMA_TRANSFORMER_TYPE_MAPPING = {
454
- SchemaNormalizationModel.None_: TransformConfig.NoTransform,
455
- SchemaNormalizationModel.Default: TransformConfig.DefaultSchemaNormalization,
456
- }
457
-
458
442
 
459
443
  class ModelToComponentFactory:
460
444
  EPOCH_DATETIME_FORMAT = "%s"
@@ -503,7 +487,6 @@ class ModelToComponentFactory:
503
487
  CustomRequesterModel: self.create_custom_component,
504
488
  CustomRetrieverModel: self.create_custom_component,
505
489
  CustomSchemaLoader: self.create_custom_component,
506
- CustomSchemaNormalizationModel: self.create_custom_component,
507
490
  CustomStateMigration: self.create_custom_component,
508
491
  CustomPaginationStrategyModel: self.create_custom_component,
509
492
  CustomPartitionRouterModel: self.create_custom_component,
@@ -526,7 +509,6 @@ class ModelToComponentFactory:
526
509
  GzipParserModel: self.create_gzip_parser,
527
510
  KeysToLowerModel: self.create_keys_to_lower_transformation,
528
511
  KeysToSnakeCaseModel: self.create_keys_to_snake_transformation,
529
- KeysReplaceModel: self.create_keys_replace_transformation,
530
512
  FlattenFieldsModel: self.create_flatten_fields,
531
513
  IterableDecoderModel: self.create_iterable_decoder,
532
514
  XmlDecoderModel: self.create_xml_decoder,
@@ -648,19 +630,10 @@ class ModelToComponentFactory:
648
630
  ) -> KeysToSnakeCaseTransformation:
649
631
  return KeysToSnakeCaseTransformation()
650
632
 
651
- def create_keys_replace_transformation(
652
- self, model: KeysReplaceModel, config: Config, **kwargs: Any
653
- ) -> KeysReplaceTransformation:
654
- return KeysReplaceTransformation(
655
- old=model.old, new=model.new, parameters=model.parameters or {}
656
- )
657
-
658
633
  def create_flatten_fields(
659
634
  self, model: FlattenFieldsModel, config: Config, **kwargs: Any
660
635
  ) -> FlattenFields:
661
- return FlattenFields(
662
- flatten_lists=model.flatten_lists if model.flatten_lists is not None else True
663
- )
636
+ return FlattenFields()
664
637
 
665
638
  @staticmethod
666
639
  def _json_schema_type_name_to_type(value_type: Optional[ValueType]) -> Optional[Type[Any]]:
@@ -1587,12 +1560,7 @@ class ModelToComponentFactory:
1587
1560
  )
1588
1561
 
1589
1562
  def create_http_requester(
1590
- self,
1591
- model: HttpRequesterModel,
1592
- config: Config,
1593
- decoder: Decoder = JsonDecoder(parameters={}),
1594
- *,
1595
- name: str,
1563
+ self, model: HttpRequesterModel, decoder: Decoder, config: Config, *, name: str
1596
1564
  ) -> HttpRequester:
1597
1565
  authenticator = (
1598
1566
  self._create_component_from_model(
@@ -2008,11 +1976,12 @@ class ModelToComponentFactory:
2008
1976
  config: Config,
2009
1977
  *,
2010
1978
  name: str,
2011
- transformations: List[RecordTransformation] | None = None,
2012
- decoder: Decoder | None = None,
2013
- client_side_incremental_sync: Dict[str, Any] | None = None,
1979
+ transformations: List[RecordTransformation],
1980
+ decoder: Optional[Decoder] = None,
1981
+ client_side_incremental_sync: Optional[Dict[str, Any]] = None,
2014
1982
  **kwargs: Any,
2015
1983
  ) -> RecordSelector:
1984
+ assert model.schema_normalization is not None # for mypy
2016
1985
  extractor = self._create_component_from_model(
2017
1986
  model=model.extractor, decoder=decoder, config=config
2018
1987
  )
@@ -2030,10 +1999,8 @@ class ModelToComponentFactory:
2030
1999
  else None,
2031
2000
  **client_side_incremental_sync,
2032
2001
  )
2033
- schema_normalization = (
2034
- TypeTransformer(SCHEMA_TRANSFORMER_TYPE_MAPPING[model.schema_normalization])
2035
- if isinstance(model.schema_normalization, SchemaNormalizationModel)
2036
- else self._create_component_from_model(model.schema_normalization, config=config) # type: ignore[arg-type] # custom normalization model expected here
2002
+ schema_normalization = TypeTransformer(
2003
+ SCHEMA_TRANSFORMER_TYPE_MAPPING[model.schema_normalization]
2037
2004
  )
2038
2005
 
2039
2006
  return RecordSelector(
@@ -2041,7 +2008,7 @@ class ModelToComponentFactory:
2041
2008
  name=name,
2042
2009
  config=config,
2043
2010
  record_filter=record_filter,
2044
- transformations=transformations or [],
2011
+ transformations=transformations,
2045
2012
  schema_normalization=schema_normalization,
2046
2013
  parameters=model.parameters or {},
2047
2014
  )