airbyte-cdk 0.72.0__py3-none-any.whl → 6.13.1.dev4106__py3-none-any.whl

airbyte_cdk/sources/declarative/schema/dynamic_schema_loader.py

@@ -0,0 +1,236 @@
+#
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+#
+
+
+from copy import deepcopy
+from dataclasses import InitVar, dataclass, field
+from typing import Any, List, Mapping, MutableMapping, Optional, Union
+
+import dpath
+from typing_extensions import deprecated
+
+from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
+from airbyte_cdk.sources.declarative.retrievers.retriever import Retriever
+from airbyte_cdk.sources.declarative.schema.schema_loader import SchemaLoader
+from airbyte_cdk.sources.declarative.transformations import RecordTransformation
+from airbyte_cdk.sources.source import ExperimentalClassWarning
+from airbyte_cdk.sources.types import Config, StreamSlice, StreamState
+
+AIRBYTE_DATA_TYPES: Mapping[str, Mapping[str, Any]] = {
+    "string": {"type": ["null", "string"]},
+    "boolean": {"type": ["null", "boolean"]},
+    "date": {"type": ["null", "string"], "format": "date"},
+    "timestamp_without_timezone": {
+        "type": ["null", "string"],
+        "format": "date-time",
+        "airbyte_type": "timestamp_without_timezone",
+    },
+    "timestamp_with_timezone": {"type": ["null", "string"], "format": "date-time"},
+    "time_without_timezone": {
+        "type": ["null", "string"],
+        "format": "time",
+        "airbyte_type": "time_without_timezone",
+    },
+    "time_with_timezone": {
+        "type": ["null", "string"],
+        "format": "time",
+        "airbyte_type": "time_with_timezone",
+    },
+    "integer": {"type": ["null", "integer"]},
+    "number": {"type": ["null", "number"]},
+    "array": {"type": ["null", "array"]},
+    "object": {"type": ["null", "object"]},
+}
+
+
+@deprecated("This class is experimental. Use at your own risk.", category=ExperimentalClassWarning)
+@dataclass(frozen=True)
+class TypesMap:
+    """
+    Represents a mapping between a current type and its corresponding target type.
+    """
+
+    target_type: Union[List[str], str]
+    current_type: Union[List[str], str]
+
+
+@deprecated("This class is experimental. Use at your own risk.", category=ExperimentalClassWarning)
+@dataclass
+class SchemaTypeIdentifier:
+    """
+    Identifies schema details for dynamic schema extraction and processing.
+    """
+
+    key_pointer: List[Union[InterpolatedString, str]]
+    parameters: InitVar[Mapping[str, Any]]
+    type_pointer: Optional[List[Union[InterpolatedString, str]]] = None
+    types_mapping: Optional[List[TypesMap]] = None
+    schema_pointer: Optional[List[Union[InterpolatedString, str]]] = None
+
+    def __post_init__(self, parameters: Mapping[str, Any]) -> None:
+        self.schema_pointer = (
+            self._update_pointer(self.schema_pointer, parameters) if self.schema_pointer else []
+        )  # type: ignore[assignment]  # This is reqired field in model
+        self.key_pointer = self._update_pointer(self.key_pointer, parameters)  # type: ignore[assignment]  # This is reqired field in model
+        self.type_pointer = (
+            self._update_pointer(self.type_pointer, parameters) if self.type_pointer else None
+        )
+
+    @staticmethod
+    def _update_pointer(
+        pointer: Optional[List[Union[InterpolatedString, str]]], parameters: Mapping[str, Any]
+    ) -> Optional[List[Union[InterpolatedString, str]]]:
+        return (
+            [
+                InterpolatedString.create(path, parameters=parameters)
+                if isinstance(path, str)
+                else path
+                for path in pointer
+            ]
+            if pointer
+            else None
+        )
+
+
+@deprecated("This class is experimental. Use at your own risk.", category=ExperimentalClassWarning)
+@dataclass
+class DynamicSchemaLoader(SchemaLoader):
+    """
+    Dynamically loads a JSON Schema by extracting data from retrieved records.
+    """
+
+    retriever: Retriever
+    config: Config
+    parameters: InitVar[Mapping[str, Any]]
+    schema_type_identifier: SchemaTypeIdentifier
+    schema_transformations: List[RecordTransformation] = field(default_factory=lambda: [])
+
+    def get_json_schema(self) -> Mapping[str, Any]:
+        """
+        Constructs a JSON Schema based on retrieved data.
+        """
+        properties = {}
+        retrieved_record = next(self.retriever.read_records({}), None)  # type: ignore[call-overload] # read_records return Iterable data type
+
+        raw_schema = (
+            self._extract_data(
+                retrieved_record,  # type: ignore[arg-type] # Expected that retrieved_record will be only Mapping[str, Any]
+                self.schema_type_identifier.schema_pointer,
+            )
+            if retrieved_record
+            else []
+        )
+
+        for property_definition in raw_schema:
+            key = self._get_key(property_definition, self.schema_type_identifier.key_pointer)
+            value = self._get_type(
+                property_definition,
+                self.schema_type_identifier.type_pointer,
+            )
+            properties[key] = value
+
+        transformed_properties = self._transform(properties, {})
+
+        return {
+            "$schema": "http://json-schema.org/draft-07/schema#",
+            "type": "object",
+            "properties": transformed_properties,
+        }
+
+    def _transform(
+        self,
+        properties: Mapping[str, Any],
+        stream_state: StreamState,
+        stream_slice: Optional[StreamSlice] = None,
+    ) -> Mapping[str, Any]:
+        for transformation in self.schema_transformations:
+            transformation.transform(
+                properties,  # type: ignore  # properties has type Mapping[str, Any], but Dict[str, Any] expected
+                config=self.config,
+            )
+        return properties
+
+    def _get_key(
+        self,
+        raw_schema: MutableMapping[str, Any],
+        field_key_path: List[Union[InterpolatedString, str]],
+    ) -> str:
+        """
+        Extracts the key field from the schema using the specified path.
+        """
+        field_key = self._extract_data(raw_schema, field_key_path)
+        if not isinstance(field_key, str):
+            raise ValueError(f"Expected key to be a string. Got {field_key}")
+        return field_key
+
+    def _get_type(
+        self,
+        raw_schema: MutableMapping[str, Any],
+        field_type_path: Optional[List[Union[InterpolatedString, str]]],
+    ) -> Union[Mapping[str, Any], List[Mapping[str, Any]]]:
+        """
+        Determines the JSON Schema type for a field, supporting nullable and combined types.
+        """
+        raw_field_type = (
+            self._extract_data(raw_schema, field_type_path, default="string")
+            if field_type_path
+            else "string"
+        )
+        mapped_field_type = self._replace_type_if_not_valid(raw_field_type)
+        if (
+            isinstance(mapped_field_type, list)
+            and len(mapped_field_type) == 2
+            and all(isinstance(item, str) for item in mapped_field_type)
+        ):
+            first_type = self._get_airbyte_type(mapped_field_type[0])
+            second_type = self._get_airbyte_type(mapped_field_type[1])
+            return {"oneOf": [first_type, second_type]}
+        elif isinstance(mapped_field_type, str):
+            return self._get_airbyte_type(mapped_field_type)
+        else:
+            raise ValueError(
+                f"Invalid data type. Available string or two items list of string. Got {mapped_field_type}."
+            )
+
+    def _replace_type_if_not_valid(
+        self, field_type: Union[List[str], str]
+    ) -> Union[List[str], str]:
+        """
+        Replaces a field type if it matches a type mapping in `types_map`.
+        """
+        if self.schema_type_identifier.types_mapping:
+            for types_map in self.schema_type_identifier.types_mapping:
+                if field_type == types_map.current_type:
+                    return types_map.target_type
+        return field_type
+
+    @staticmethod
+    def _get_airbyte_type(field_type: str) -> Mapping[str, Any]:
+        """
+        Maps a field type to its corresponding Airbyte type definition.
+        """
+        if field_type not in AIRBYTE_DATA_TYPES:
+            raise ValueError(f"Invalid Airbyte data type: {field_type}")
+
+        return deepcopy(AIRBYTE_DATA_TYPES[field_type])
+
+    def _extract_data(
+        self,
+        body: Mapping[str, Any],
+        extraction_path: Optional[List[Union[InterpolatedString, str]]] = None,
+        default: Any = None,
+    ) -> Any:
+        """
+        Extracts data from the body based on the provided extraction path.
+        """
+
+        if not extraction_path:
+            return body
+
+        path = [
+            node.eval(self.config) if not isinstance(node, str) else node
+            for node in extraction_path
+        ]
+
+        return dpath.get(body, path, default=default)  # type: ignore # extracted will be a MutableMapping, given input data structure

airbyte_cdk/sources/declarative/schema/json_file_schema_loader.py

@@ -6,11 +6,11 @@ import json
 import pkgutil
 import sys
 from dataclasses import InitVar, dataclass, field
-from typing import Any, Mapping, Union
+from typing import Any, Mapping, Tuple, Union
 
 from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
 from airbyte_cdk.sources.declarative.schema.schema_loader import SchemaLoader
-from airbyte_cdk.sources.declarative.types import Config
+from airbyte_cdk.sources.types import Config
 from airbyte_cdk.sources.utils.schema_helpers import ResourceSchemaLoader
 
 
@@ -18,7 +18,7 @@ def _default_file_path() -> str:
     # Schema files are always in "source_<connector_name>/schemas/<stream_name>.json
     # The connector's module name can be inferred by looking at the modules loaded and look for the one starting with source_
     source_modules = [
-        k for k, v in sys.modules.items() if "source_" in k
+        k for k, v in sys.modules.items() if "source_" in k and "airbyte_cdk" not in k
     ]  # example: ['source_exchange_rates', 'source_exchange_rates.source']
     if source_modules:
         module = source_modules[0].split(".")[0]
@@ -43,9 +43,9 @@ class JsonFileSchemaLoader(ResourceSchemaLoader, SchemaLoader):
 
     config: Config
     parameters: InitVar[Mapping[str, Any]]
-    file_path: Union[InterpolatedString, str] = field(default=None)
+    file_path: Union[InterpolatedString, str] = field(default="")
 
-    def __post_init__(self, parameters: Mapping[str, Any]):
+    def __post_init__(self, parameters: Mapping[str, Any]) -> None:
         if not self.file_path:
             self.file_path = _default_file_path()
         self.file_path = InterpolatedString.create(self.file_path, parameters=parameters)
@@ -66,11 +66,11 @@ class JsonFileSchemaLoader(ResourceSchemaLoader, SchemaLoader):
         self.package_name = resource
         return self._resolve_schema_references(raw_schema)
 
-    def _get_json_filepath(self):
-        return self.file_path.eval(self.config)
+    def _get_json_filepath(self) -> Any:
+        return self.file_path.eval(self.config)  # type: ignore # file_path is always cast to an interpolated string
 
     @staticmethod
-    def extract_resource_and_schema_path(json_schema_path: str) -> (str, str):
+    def extract_resource_and_schema_path(json_schema_path: str) -> Tuple[str, str]:
         """
         When the connector is running on a docker container, package_data is accessible from the resource (source_<name>), so we extract
         the resource from the first part of the schema path and the remaining path is used to find the schema file. This is a slight

airbyte_cdk/sources/declarative/spec/spec.py

@@ -5,7 +5,11 @@
 from dataclasses import InitVar, dataclass
 from typing import Any, Mapping, Optional
 
-from airbyte_cdk.models.airbyte_protocol import ConnectorSpecification
+from airbyte_cdk.models import (
+    AdvancedAuth,
+    ConnectorSpecification,
+    ConnectorSpecificationSerializer,
+)
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import AuthFlow
 
 
@@ -29,13 +33,16 @@ class Spec:
         Returns the connector specification according the spec block defined in the low code connector manifest.
         """
 
-        obj = {"connectionSpecification": self.connection_specification}
+        obj: dict[str, Mapping[str, Any] | str | AdvancedAuth] = {
+            "connectionSpecification": self.connection_specification
+        }
 
         if self.documentation_url:
             obj["documentationUrl"] = self.documentation_url
         if self.advanced_auth:
-            obj["advanced_auth"] = self.advanced_auth
-            obj["advanced_auth"].auth_flow_type = obj["advanced_auth"].auth_flow_type.value  # Get enum value
+            self.advanced_auth.auth_flow_type = self.advanced_auth.auth_flow_type.value  # type: ignore # We know this is always assigned to an AuthFlow which has the auth_flow_type field
+            # Map CDK AuthFlow model to protocol AdvancedAuth model
+            obj["advanced_auth"] = self.advanced_auth.dict()
 
         # We remap these keys to camel case because that's the existing format expected by the rest of the platform
-        return ConnectorSpecification.parse_obj(obj)
+        return ConnectorSpecificationSerializer.load(obj)

airbyte_cdk/sources/declarative/stream_slicers/__init__.py

@@ -2,7 +2,6 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
 
-from airbyte_cdk.sources.declarative.stream_slicers.cartesian_product_stream_slicer import CartesianProductStreamSlicer
 from airbyte_cdk.sources.declarative.stream_slicers.stream_slicer import StreamSlicer
 
-__all__ = ["CartesianProductStreamSlicer", "StreamSlicer"]
+__all__ = ["StreamSlicer"]

airbyte_cdk/sources/declarative/stream_slicers/declarative_partition_generator.py

@@ -0,0 +1,88 @@
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+
+from typing import Any, Iterable, Mapping, Optional
+
+from airbyte_cdk.sources.declarative.retrievers import Retriever
+from airbyte_cdk.sources.message import MessageRepository
+from airbyte_cdk.sources.streams.concurrent.partitions.partition import Partition
+from airbyte_cdk.sources.streams.concurrent.partitions.partition_generator import PartitionGenerator
+from airbyte_cdk.sources.streams.concurrent.partitions.stream_slicer import StreamSlicer
+from airbyte_cdk.sources.types import Record, StreamSlice
+from airbyte_cdk.utils.slice_hasher import SliceHasher
+
+
+class DeclarativePartitionFactory:
+    def __init__(
+        self,
+        stream_name: str,
+        json_schema: Mapping[str, Any],
+        retriever: Retriever,
+        message_repository: MessageRepository,
+    ) -> None:
+        """
+        The DeclarativePartitionFactory takes a retriever_factory and not a retriever directly. The reason is that our components are not
+        thread safe and classes like `DefaultPaginator` may not work because multiple threads can access and modify a shared field across each other.
+        In order to avoid these problems, we will create one retriever per thread which should make the processing thread-safe.
+        """
+        self._stream_name = stream_name
+        self._json_schema = json_schema
+        self._retriever = retriever
+        self._message_repository = message_repository
+
+    def create(self, stream_slice: StreamSlice) -> Partition:
+        return DeclarativePartition(
+            self._stream_name,
+            self._json_schema,
+            self._retriever,
+            self._message_repository,
+            stream_slice,
+        )
+
+
+class DeclarativePartition(Partition):
+    def __init__(
+        self,
+        stream_name: str,
+        json_schema: Mapping[str, Any],
+        retriever: Retriever,
+        message_repository: MessageRepository,
+        stream_slice: StreamSlice,
+    ):
+        self._stream_name = stream_name
+        self._json_schema = json_schema
+        self._retriever = retriever
+        self._message_repository = message_repository
+        self._stream_slice = stream_slice
+        self._hash = SliceHasher.hash(self._stream_name, self._stream_slice)
+
+    def read(self) -> Iterable[Record]:
+        for stream_data in self._retriever.read_records(self._json_schema, self._stream_slice):
+            if isinstance(stream_data, Mapping):
+                yield Record(
+                    data=stream_data,
+                    stream_name=self.stream_name(),
+                    associated_slice=self._stream_slice,
+                )
+            else:
+                self._message_repository.emit_message(stream_data)
+
+    def to_slice(self) -> Optional[Mapping[str, Any]]:
+        return self._stream_slice
+
+    def stream_name(self) -> str:
+        return self._stream_name
+
+    def __hash__(self) -> int:
+        return self._hash
+
+
+class StreamSlicerPartitionGenerator(PartitionGenerator):
+    def __init__(
+        self, partition_factory: DeclarativePartitionFactory, stream_slicer: StreamSlicer
+    ) -> None:
+        self._partition_factory = partition_factory
+        self._stream_slicer = stream_slicer
+
+    def generate(self) -> Iterable[Partition]:
+        for stream_slice in self._stream_slicer.stream_slices():
+            yield self._partition_factory.create(stream_slice)

airbyte_cdk/sources/declarative/stream_slicers/stream_slicer.py

@@ -2,16 +2,17 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
 
-from abc import abstractmethod
-from dataclasses import dataclass
-from typing import Iterable
+from abc import ABC
 
-from airbyte_cdk.sources.declarative.requesters.request_options.request_options_provider import RequestOptionsProvider
-from airbyte_cdk.sources.declarative.types import StreamSlice
+from airbyte_cdk.sources.declarative.requesters.request_options.request_options_provider import (
+    RequestOptionsProvider,
+)
+from airbyte_cdk.sources.streams.concurrent.partitions.stream_slicer import (
+    StreamSlicer as ConcurrentStreamSlicer,
+)
 
 
-@dataclass
-class StreamSlicer(RequestOptionsProvider):
+class StreamSlicer(ConcurrentStreamSlicer, RequestOptionsProvider, ABC):
     """
     Slices the stream into a subset of records.
     Slices enable state checkpointing and data retrieval parallelization.
@@ -21,10 +22,4 @@ class StreamSlicer(RequestOptionsProvider):
     See the stream slicing section of the docs for more information.
     """
 
-    @abstractmethod
-    def stream_slices(self) -> Iterable[StreamSlice]:
-        """
-        Defines stream slices
-
-        :return: List of stream slices
-        """
+    pass

airbyte_cdk/sources/declarative/transformations/add_fields.py

@@ -3,12 +3,13 @@
 #
 
 from dataclasses import InitVar, dataclass, field
-from typing import Any, List, Mapping, Optional, Type, Union
+from typing import Any, Dict, List, Mapping, Optional, Type, Union
+
+import dpath
 
-import dpath.util
 from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
 from airbyte_cdk.sources.declarative.transformations import RecordTransformation
-from airbyte_cdk.sources.declarative.types import Config, FieldPointer, Record, StreamSlice, StreamState
+from airbyte_cdk.sources.types import Config, FieldPointer, StreamSlice, StreamState
 
 
 @dataclass(frozen=True)
@@ -85,12 +86,16 @@ class AddFields(RecordTransformation):
 
     fields: List[AddedFieldDefinition]
     parameters: InitVar[Mapping[str, Any]]
-    _parsed_fields: List[ParsedAddFieldDefinition] = field(init=False, repr=False, default_factory=list)
+    _parsed_fields: List[ParsedAddFieldDefinition] = field(
+        init=False, repr=False, default_factory=list
+    )
 
     def __post_init__(self, parameters: Mapping[str, Any]) -> None:
         for add_field in self.fields:
             if len(add_field.path) < 1:
-                raise ValueError(f"Expected a non-zero-length path for the AddFields transformation {add_field}")
+                raise ValueError(
+                    f"Expected a non-zero-length path for the AddFields transformation {add_field}"
+                )
 
             if not isinstance(add_field.value, InterpolatedString):
                 if not isinstance(add_field.value, str):
@@ -106,25 +111,28 @@ class AddFields(RecordTransformation):
                     )
             else:
                 self._parsed_fields.append(
-                    ParsedAddFieldDefinition(add_field.path, add_field.value, value_type=add_field.value_type, parameters={})
+                    ParsedAddFieldDefinition(
+                        add_field.path,
+                        add_field.value,
+                        value_type=add_field.value_type,
+                        parameters={},
+                    )
                 )
 
     def transform(
         self,
-        record: Record,
+        record: Dict[str, Any],
         config: Optional[Config] = None,
         stream_state: Optional[StreamState] = None,
         stream_slice: Optional[StreamSlice] = None,
-    ) -> Record:
+    ) -> None:
         if config is None:
             config = {}
         kwargs = {"record": record, "stream_state": stream_state, "stream_slice": stream_slice}
         for parsed_field in self._parsed_fields:
             valid_types = (parsed_field.value_type,) if parsed_field.value_type else None
             value = parsed_field.value.eval(config, valid_types=valid_types, **kwargs)
-            dpath.util.new(record, parsed_field.path, value)
-
-        return record
+            dpath.new(record, parsed_field.path, value)
 
     def __eq__(self, other: Any) -> bool:
         return bool(self.__dict__ == other.__dict__)

airbyte_cdk/sources/declarative/transformations/flatten_fields.py

@@ -0,0 +1,52 @@
+#
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+#
+
+from dataclasses import dataclass
+from typing import Any, Dict, Optional
+
+from airbyte_cdk.sources.declarative.transformations import RecordTransformation
+from airbyte_cdk.sources.types import Config, StreamSlice, StreamState
+
+
+@dataclass
+class FlattenFields(RecordTransformation):
+    flatten_lists: bool = True
+
+    def transform(
+        self,
+        record: Dict[str, Any],
+        config: Optional[Config] = None,
+        stream_state: Optional[StreamState] = None,
+        stream_slice: Optional[StreamSlice] = None,
+    ) -> None:
+        transformed_record = self.flatten_record(record)
+        record.clear()
+        record.update(transformed_record)
+
+    def flatten_record(self, record: Dict[str, Any]) -> Dict[str, Any]:
+        stack = [(record, "_")]
+        transformed_record: Dict[str, Any] = {}
+        force_with_parent_name = False
+
+        while stack:
+            current_record, parent_key = stack.pop()
+
+            if isinstance(current_record, dict):
+                for current_key, value in current_record.items():
+                    new_key = (
+                        f"{parent_key}.{current_key}"
+                        if (current_key in transformed_record or force_with_parent_name)
+                        else current_key
+                    )
+                    stack.append((value, new_key))
+
+            elif isinstance(current_record, list) and self.flatten_lists:
+                for i, item in enumerate(current_record):
+                    force_with_parent_name = True
+                    stack.append((item, f"{parent_key}.{i}"))
+
+            else:
+                transformed_record[parent_key] = current_record
+
+        return transformed_record

airbyte_cdk/sources/declarative/transformations/keys_replace_transformation.py

@@ -0,0 +1,61 @@
+#
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+#
+
+from dataclasses import InitVar, dataclass
+from typing import Any, Dict, Mapping, Optional
+
+from airbyte_cdk import InterpolatedString
+from airbyte_cdk.sources.declarative.transformations import RecordTransformation
+from airbyte_cdk.sources.types import Config, StreamSlice, StreamState
+
+
+@dataclass
+class KeysReplaceTransformation(RecordTransformation):
+    """
+    Transformation that applies keys names replacement.
+
+    Example usage:
+    - type: KeysReplace
+      old: " "
+      new: "_"
+    Result:
+    from: {"created time": ..., "customer id": ..., "user id": ...}
+    to: {"created_time": ..., "customer_id": ..., "user_id": ...}
+    """
+
+    old: str
+    new: str
+    parameters: InitVar[Mapping[str, Any]]
+
+    def __post_init__(self, parameters: Mapping[str, Any]) -> None:
+        self._old = InterpolatedString.create(self.old, parameters=parameters)
+        self._new = InterpolatedString.create(self.new, parameters=parameters)
+
+    def transform(
+        self,
+        record: Dict[str, Any],
+        config: Optional[Config] = None,
+        stream_state: Optional[StreamState] = None,
+        stream_slice: Optional[StreamSlice] = None,
+    ) -> None:
+        if config is None:
+            config = {}
+
+        kwargs = {"record": record, "stream_state": stream_state, "stream_slice": stream_slice}
+        old_key = str(self._old.eval(config, **kwargs))
+        new_key = str(self._new.eval(config, **kwargs))
+
+        def _transform(data: Dict[str, Any]) -> Dict[str, Any]:
+            result = {}
+            for key, value in data.items():
+                updated_key = key.replace(old_key, new_key)
+                if isinstance(value, dict):
+                    result[updated_key] = _transform(value)
+                else:
+                    result[updated_key] = value
+            return result
+
+        transformed_record = _transform(record)
+        record.clear()
+        record.update(transformed_record)

airbyte_cdk/sources/declarative/transformations/keys_to_lower_transformation.py

@@ -0,0 +1,22 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+from dataclasses import dataclass
+from typing import Any, Dict, Optional
+
+from airbyte_cdk.sources.declarative.transformations import RecordTransformation
+from airbyte_cdk.sources.types import Config, StreamSlice, StreamState
+
+
+@dataclass
+class KeysToLowerTransformation(RecordTransformation):
+    def transform(
+        self,
+        record: Dict[str, Any],
+        config: Optional[Config] = None,
+        stream_state: Optional[StreamState] = None,
+        stream_slice: Optional[StreamSlice] = None,
+    ) -> None:
+        for key in set(record.keys()):
+            record[key.lower()] = record.pop(key)