airbyte-cdk 0.0.0.dev0__py3-none-any.whl

airbyte_cdk/sources/declarative/transformations/__init__.py

@@ -0,0 +1,17 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+
+# RecordTransformation is depended upon by every class in this module (since it's the abc everything implements). For this reason,
+# the order of imports matters i.e: this file must fully import RecordTransformation before importing anything which depends on RecordTransformation
+# Otherwise there will be a circular dependency (load order will be init.py --> RemoveFields (which tries to import RecordTransformation) -->
+# init.py --> circular dep error, since loading this file causes it to try to import itself down the line.
+# so we add the split directive below to tell isort to sort imports while keeping RecordTransformation as the first import
+from .transformation import RecordTransformation
+
+# isort: split
+from .add_fields import AddFields
+from .remove_fields import RemoveFields
+
+__all__ = ["AddFields", "RecordTransformation", "RemoveFields"]

airbyte_cdk/sources/declarative/transformations/add_fields.py

@@ -0,0 +1,146 @@
+#
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+#
+
+from dataclasses import InitVar, dataclass, field
+from typing import Any, Dict, List, Mapping, Optional, Type, Union
+
+import dpath
+
+from airbyte_cdk.sources.declarative.interpolation.interpolated_boolean import InterpolatedBoolean
+from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
+from airbyte_cdk.sources.declarative.transformations import RecordTransformation
+from airbyte_cdk.sources.types import Config, FieldPointer, StreamSlice, StreamState
+
+
+@dataclass(frozen=True)
+class AddedFieldDefinition:
+    """Defines the field to add on a record"""
+
+    path: FieldPointer
+    value: Union[InterpolatedString, str]
+    value_type: Optional[Type[Any]]
+    parameters: InitVar[Mapping[str, Any]]
+
+
+@dataclass(frozen=True)
+class ParsedAddFieldDefinition:
+    """Defines the field to add on a record"""
+
+    path: FieldPointer
+    value: InterpolatedString
+    value_type: Optional[Type[Any]]
+    parameters: InitVar[Mapping[str, Any]]
+
+
+@dataclass
+class AddFields(RecordTransformation):
+    """
+    Transformation which adds field to an output record. The path of the added field can be nested. Adding nested fields will create all
+    necessary parent objects (like mkdir -p). Adding fields to an array will extend the array to that index (filling intermediate
+    indices with null values). So if you add a field at index 5 to the array ["value"], it will become ["value", null, null, null, null,
+    "new_value"].
+
+
+    This transformation has access to the following contextual values:
+        record: the record about to be output by the connector
+        config: the input configuration provided to a connector
+        stream_state: the current state of the stream
+        stream_slice: the current stream slice being read
+
+
+
+    Examples of instantiating this transformation via YAML:
+    - type: AddFields
+      fields:
+        # hardcoded constant
+        - path: ["path"]
+          value: "static_value"
+
+        # nested path
+        - path: ["path", "to", "field"]
+          value: "static"
+
+        # from config
+        - path: ["shop_id"]
+          value: "{{ config.shop_id }}"
+
+        # from stream_interval
+        - path: ["date"]
+          value: "{{ stream_interval.start_date }}"
+
+        # from record
+        - path: ["unnested_value"]
+          value: {{ record.nested.field }}
+
+        # from stream_slice
+        - path: ["start_date"]
+          value: {{ stream_slice.start_date }}
+
+        # by supplying any valid Jinja template directive or expression https://jinja.palletsprojects.com/en/3.1.x/templates/#
+        - path: ["two_times_two"]
+          value: {{ 2 * 2 }}
+
+    Attributes:
+        fields (List[AddedFieldDefinition]): A list of transformations (path and corresponding value) that will be added to the record
+    """
+
+    fields: List[AddedFieldDefinition]
+    parameters: InitVar[Mapping[str, Any]]
+    condition: str = ""
+    _parsed_fields: List[ParsedAddFieldDefinition] = field(
+        init=False, repr=False, default_factory=list
+    )
+
+    def __post_init__(self, parameters: Mapping[str, Any]) -> None:
+        self._filter_interpolator = InterpolatedBoolean(
+            condition=self.condition, parameters=parameters
+        )
+
+        for add_field in self.fields:
+            if len(add_field.path) < 1:
+                raise ValueError(
+                    f"Expected a non-zero-length path for the AddFields transformation {add_field}"
+                )
+
+            if not isinstance(add_field.value, InterpolatedString):
+                if not isinstance(add_field.value, str):
+                    raise f"Expected a string value for the AddFields transformation: {add_field}"
+                else:
+                    self._parsed_fields.append(
+                        ParsedAddFieldDefinition(
+                            add_field.path,
+                            InterpolatedString.create(add_field.value, parameters=parameters),
+                            value_type=add_field.value_type,
+                            parameters=parameters,
+                        )
+                    )
+            else:
+                self._parsed_fields.append(
+                    ParsedAddFieldDefinition(
+                        add_field.path,
+                        add_field.value,
+                        value_type=add_field.value_type,
+                        parameters={},
+                    )
+                )
+
+    def transform(
+        self,
+        record: Dict[str, Any],
+        config: Optional[Config] = None,
+        stream_state: Optional[StreamState] = None,
+        stream_slice: Optional[StreamSlice] = None,
+    ) -> None:
+        if config is None:
+            config = {}
+        kwargs = {"record": record, "stream_slice": stream_slice}
+        for parsed_field in self._parsed_fields:
+            valid_types = (parsed_field.value_type,) if parsed_field.value_type else None
+            value = parsed_field.value.eval(config, valid_types=valid_types, **kwargs)
+            is_empty_condition = not self.condition
+            if is_empty_condition or self._filter_interpolator.eval(config, value=value, **kwargs):
+                dpath.new(record, parsed_field.path, value)
+
+    def __eq__(self, other: Any) -> bool:
+        return bool(self.__dict__ == other.__dict__)

airbyte_cdk/sources/declarative/transformations/dpath_flatten_fields.py

@@ -0,0 +1,61 @@
+from dataclasses import InitVar, dataclass
+from typing import Any, Dict, List, Mapping, Optional, Union
+
+import dpath
+
+from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
+from airbyte_cdk.sources.declarative.transformations import RecordTransformation
+from airbyte_cdk.sources.types import Config, StreamSlice, StreamState
+
+
+@dataclass
+class DpathFlattenFields(RecordTransformation):
+    """
+    Flatten fields only for provided path.
+
+    field_path: List[Union[InterpolatedString, str]] path to the field to flatten.
+    delete_origin_value: bool = False whether to delete origin field or keep it. Default is False.
+    replace_record: bool = False whether to replace origin record or not. Default is False.
+
+    """
+
+    config: Config
+    field_path: List[Union[InterpolatedString, str]]
+    parameters: InitVar[Mapping[str, Any]]
+    delete_origin_value: bool = False
+    replace_record: bool = False
+
+    def __post_init__(self, parameters: Mapping[str, Any]) -> None:
+        self._field_path = [
+            InterpolatedString.create(path, parameters=parameters) for path in self.field_path
+        ]
+        for path_index in range(len(self.field_path)):
+            if isinstance(self.field_path[path_index], str):
+                self._field_path[path_index] = InterpolatedString.create(
+                    self.field_path[path_index], parameters=parameters
+                )
+
+    def transform(
+        self,
+        record: Dict[str, Any],
+        config: Optional[Config] = None,
+        stream_state: Optional[StreamState] = None,
+        stream_slice: Optional[StreamSlice] = None,
+    ) -> None:
+        path = [path.eval(self.config) for path in self._field_path]
+        if "*" in path:
+            matched = dpath.values(record, path)
+            extracted = matched[0] if matched else None
+        else:
+            extracted = dpath.get(record, path, default=[])
+
+        if isinstance(extracted, dict):
+            if self.replace_record and extracted:
+                dpath.delete(record, "**")
+                record.update(extracted)
+            else:
+                conflicts = set(extracted.keys()) & set(record.keys())
+                if not conflicts:
+                    if self.delete_origin_value:
+                        dpath.delete(record, path)
+                    record.update(extracted)

airbyte_cdk/sources/declarative/transformations/flatten_fields.py

@@ -0,0 +1,52 @@
+#
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+#
+
+from dataclasses import dataclass
+from typing import Any, Dict, Optional
+
+from airbyte_cdk.sources.declarative.transformations import RecordTransformation
+from airbyte_cdk.sources.types import Config, StreamSlice, StreamState
+
+
+@dataclass
+class FlattenFields(RecordTransformation):
+    flatten_lists: bool = True
+
+    def transform(
+        self,
+        record: Dict[str, Any],
+        config: Optional[Config] = None,
+        stream_state: Optional[StreamState] = None,
+        stream_slice: Optional[StreamSlice] = None,
+    ) -> None:
+        transformed_record = self.flatten_record(record)
+        record.clear()
+        record.update(transformed_record)
+
+    def flatten_record(self, record: Dict[str, Any]) -> Dict[str, Any]:
+        stack = [(record, "_")]
+        transformed_record: Dict[str, Any] = {}
+        force_with_parent_name = False
+
+        while stack:
+            current_record, parent_key = stack.pop()
+
+            if isinstance(current_record, dict):
+                for current_key, value in current_record.items():
+                    new_key = (
+                        f"{parent_key}.{current_key}"
+                        if (current_key in transformed_record or force_with_parent_name)
+                        else current_key
+                    )
+                    stack.append((value, new_key))
+
+            elif isinstance(current_record, list) and self.flatten_lists:
+                for i, item in enumerate(current_record):
+                    force_with_parent_name = True
+                    stack.append((item, f"{parent_key}.{i}"))
+
+            else:
+                transformed_record[parent_key] = current_record
+
+        return transformed_record

airbyte_cdk/sources/declarative/transformations/keys_replace_transformation.py

@@ -0,0 +1,61 @@
+#
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+#
+
+from dataclasses import InitVar, dataclass
+from typing import Any, Dict, Mapping, Optional
+
+from airbyte_cdk import InterpolatedString
+from airbyte_cdk.sources.declarative.transformations import RecordTransformation
+from airbyte_cdk.sources.types import Config, StreamSlice, StreamState
+
+
+@dataclass
+class KeysReplaceTransformation(RecordTransformation):
+    """
+    Transformation that applies keys names replacement.
+
+    Example usage:
+    - type: KeysReplace
+      old: " "
+      new: "_"
+    Result:
+    from: {"created time": ..., "customer id": ..., "user id": ...}
+    to: {"created_time": ..., "customer_id": ..., "user_id": ...}
+    """
+
+    old: str
+    new: str
+    parameters: InitVar[Mapping[str, Any]]
+
+    def __post_init__(self, parameters: Mapping[str, Any]) -> None:
+        self._old = InterpolatedString.create(self.old, parameters=parameters)
+        self._new = InterpolatedString.create(self.new, parameters=parameters)
+
+    def transform(
+        self,
+        record: Dict[str, Any],
+        config: Optional[Config] = None,
+        stream_state: Optional[StreamState] = None,
+        stream_slice: Optional[StreamSlice] = None,
+    ) -> None:
+        if config is None:
+            config = {}
+
+        kwargs = {"record": record, "stream_state": stream_state, "stream_slice": stream_slice}
+        old_key = str(self._old.eval(config, **kwargs))
+        new_key = str(self._new.eval(config, **kwargs))
+
+        def _transform(data: Dict[str, Any]) -> Dict[str, Any]:
+            result = {}
+            for key, value in data.items():
+                updated_key = key.replace(old_key, new_key)
+                if isinstance(value, dict):
+                    result[updated_key] = _transform(value)
+                else:
+                    result[updated_key] = value
+            return result
+
+        transformed_record = _transform(record)
+        record.clear()
+        record.update(transformed_record)

airbyte_cdk/sources/declarative/transformations/keys_to_lower_transformation.py

@@ -0,0 +1,22 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+from dataclasses import dataclass
+from typing import Any, Dict, Optional
+
+from airbyte_cdk.sources.declarative.transformations import RecordTransformation
+from airbyte_cdk.sources.types import Config, StreamSlice, StreamState
+
+
+@dataclass
+class KeysToLowerTransformation(RecordTransformation):
+    def transform(
+        self,
+        record: Dict[str, Any],
+        config: Optional[Config] = None,
+        stream_state: Optional[StreamState] = None,
+        stream_slice: Optional[StreamSlice] = None,
+    ) -> None:
+        for key in set(record.keys()):
+            record[key.lower()] = record.pop(key)

airbyte_cdk/sources/declarative/transformations/keys_to_snake_transformation.py

@@ -0,0 +1,68 @@
+#
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+#
+
+import re
+from dataclasses import dataclass
+from typing import Any, Dict, List, Optional
+
+import anyascii
+
+from airbyte_cdk.sources.declarative.transformations import RecordTransformation
+from airbyte_cdk.sources.types import Config, StreamSlice, StreamState
+
+
+@dataclass
+class KeysToSnakeCaseTransformation(RecordTransformation):
+    token_pattern: re.Pattern[str] = re.compile(
+        r"[A-Z]+[a-z]*|[a-z]+|\d+|(?P<NoToken>[^a-zA-Z\d]+)"
+    )
+
+    def transform(
+        self,
+        record: Dict[str, Any],
+        config: Optional[Config] = None,
+        stream_state: Optional[StreamState] = None,
+        stream_slice: Optional[StreamSlice] = None,
+    ) -> None:
+        transformed_record = self._transform_record(record)
+        record.clear()
+        record.update(transformed_record)
+
+    def _transform_record(self, record: Dict[str, Any]) -> Dict[str, Any]:
+        transformed_record = {}
+        for key, value in record.items():
+            transformed_key = self.process_key(key)
+            transformed_value = value
+
+            if isinstance(value, dict):
+                transformed_value = self._transform_record(value)
+
+            transformed_record[transformed_key] = transformed_value
+        return transformed_record
+
+    def process_key(self, key: str) -> str:
+        key = self.normalize_key(key)
+        tokens = self.tokenize_key(key)
+        tokens = self.filter_tokens(tokens)
+        return self.tokens_to_snake_case(tokens)
+
+    def normalize_key(self, key: str) -> str:
+        return str(anyascii.anyascii(key))
+
+    def tokenize_key(self, key: str) -> List[str]:
+        tokens = []
+        for match in self.token_pattern.finditer(key):
+            token = match.group(0) if match.group("NoToken") is None else ""
+            tokens.append(token)
+        return tokens
+
+    def filter_tokens(self, tokens: List[str]) -> List[str]:
+        if len(tokens) >= 3:
+            tokens = tokens[:1] + [t for t in tokens[1:-1] if t] + tokens[-1:]
+        if tokens and tokens[0].isdigit():
+            tokens.insert(0, "")
+        return tokens
+
+    def tokens_to_snake_case(self, tokens: List[str]) -> str:
+        return "_".join(token.lower() for token in tokens)

airbyte_cdk/sources/declarative/transformations/remove_fields.py

@@ -0,0 +1,75 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+from dataclasses import InitVar, dataclass
+from typing import Any, Dict, List, Mapping, Optional
+
+import dpath
+import dpath.exceptions
+
+from airbyte_cdk.sources.declarative.interpolation.interpolated_boolean import InterpolatedBoolean
+from airbyte_cdk.sources.declarative.transformations import RecordTransformation
+from airbyte_cdk.sources.types import Config, FieldPointer, StreamSlice, StreamState
+
+
+@dataclass
+class RemoveFields(RecordTransformation):
+    """
+    A transformation which removes fields from a record. The fields removed are designated using FieldPointers.
+    During transformation, if a field or any of its parents does not exist in the record, no error is thrown.
+
+    If an input field pointer references an item in a list (e.g: ["k", 0] in the object {"k": ["a", "b", "c"]}) then
+    the object at that index is set to None rather than being not entirely removed from the list. TODO change this behavior.
+
+    It's possible to remove objects nested in lists e.g: removing [".", 0, "k"] from {".": [{"k": "V"}]} results in {".": [{}]}
+
+    Usage syntax:
+
+    ```yaml
+        my_stream:
+            <other parameters..>
+            transformations:
+                - type: RemoveFields
+                  field_pointers:
+                    - ["path", "to", "field1"]
+                    - ["path2"]
+    ```
+
+    Attributes:
+        field_pointers (List[FieldPointer]): pointers to the fields that should be removed
+    """
+
+    field_pointers: List[FieldPointer]
+    parameters: InitVar[Mapping[str, Any]]
+    condition: str = ""
+
+    def __post_init__(self, parameters: Mapping[str, Any]) -> None:
+        self._filter_interpolator = InterpolatedBoolean(
+            condition=self.condition, parameters=parameters
+        )
+
+    def transform(
+        self,
+        record: Dict[str, Any],
+        config: Optional[Config] = None,
+        stream_state: Optional[StreamState] = None,
+        stream_slice: Optional[StreamSlice] = None,
+    ) -> None:
+        """
+        :param record: The record to be transformed
+        :return: the input record with the requested fields removed
+        """
+        for pointer in self.field_pointers:
+            # the dpath library by default doesn't delete fields from arrays
+            try:
+                dpath.delete(
+                    record,
+                    pointer,
+                    afilter=(lambda x: self._filter_interpolator.eval(config or {}, property=x))
+                    if self.condition
+                    else None,
+                )
+            except dpath.exceptions.PathNotFound:
+                # if the (potentially nested) property does not exist, silently skip
+                pass

airbyte_cdk/sources/declarative/transformations/transformation.py

@@ -0,0 +1,37 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+from abc import abstractmethod
+from dataclasses import dataclass
+from typing import Any, Dict, Optional
+
+from airbyte_cdk.sources.types import Config, StreamSlice, StreamState
+
+
+@dataclass
+class RecordTransformation:
+    """
+    Implementations of this class define transformations that can be applied to records of a stream.
+    """
+
+    @abstractmethod
+    def transform(
+        self,
+        record: Dict[str, Any],
+        config: Optional[Config] = None,
+        stream_state: Optional[StreamState] = None,
+        stream_slice: Optional[StreamSlice] = None,
+    ) -> None:
+        """
+        Transform a record by adding, deleting, or mutating fields directly from the record reference passed in argument.
+
+        :param record: The input record to be transformed
+        :param config: The user-provided configuration as specified by the source's spec
+        :param stream_state: The stream state
+        :param stream_slice: The stream slice
+        :return: The transformed record
+        """
+
+    def __eq__(self, other: object) -> bool:
+        return other.__dict__ == self.__dict__

airbyte_cdk/sources/declarative/types.py

@@ -0,0 +1,25 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+from __future__ import annotations
+
+from airbyte_cdk.sources.types import (
+    Config,
+    ConnectionDefinition,
+    FieldPointer,
+    Record,
+    StreamSlice,
+    StreamState,
+)
+
+# Note: This package originally contained class definitions for low-code CDK types, but we promoted them into the Python CDK.
+# We've migrated connectors in the repository to reference the new location, but these assignments are used to retain backwards
+# compatibility for sources created by OSS customers or on forks. This can be removed when we start bumping major versions.
+
+FieldPointer = FieldPointer
+Config = Config
+ConnectionDefinition = ConnectionDefinition
+StreamState = StreamState
+Record = Record
+StreamSlice = StreamSlice

airbyte_cdk/sources/declarative/yaml_declarative_source.py

@@ -0,0 +1,67 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+import pkgutil
+from typing import Any, List, Mapping, Optional
+
+import yaml
+
+from airbyte_cdk.models import AirbyteStateMessage, ConfiguredAirbyteCatalog
+from airbyte_cdk.sources.declarative.concurrent_declarative_source import (
+    ConcurrentDeclarativeSource,
+)
+from airbyte_cdk.sources.types import ConnectionDefinition
+
+
+class YamlDeclarativeSource(ConcurrentDeclarativeSource[List[AirbyteStateMessage]]):
+    """Declarative source defined by a yaml file"""
+
+    def __init__(
+        self,
+        path_to_yaml: str,
+        debug: bool = False,
+        catalog: Optional[ConfiguredAirbyteCatalog] = None,
+        config: Optional[Mapping[str, Any]] = None,
+        state: Optional[List[AirbyteStateMessage]] = None,
+    ) -> None:
+        """
+        :param path_to_yaml: Path to the yaml file describing the source
+        """
+        self._path_to_yaml = path_to_yaml
+        source_config = self._read_and_parse_yaml_file(path_to_yaml)
+
+        super().__init__(
+            catalog=catalog or ConfiguredAirbyteCatalog(streams=[]),
+            config=config or {},
+            state=state or [],
+            source_config=source_config,
+        )
+
+    def _read_and_parse_yaml_file(self, path_to_yaml_file: str) -> ConnectionDefinition:
+        try:
+            # For testing purposes, we want to allow to just pass a file
+            with open(path_to_yaml_file, "r") as f:
+                return yaml.safe_load(f)  # type: ignore  # we assume the yaml represents a ConnectionDefinition
+        except FileNotFoundError:
+            # Running inside the container, the working directory during an operation is not structured the same as the static files
+            package = self.__class__.__module__.split(".")[0]
+
+            yaml_config = pkgutil.get_data(package, path_to_yaml_file)
+            if yaml_config:
+                decoded_yaml = yaml_config.decode()
+                return self._parse(decoded_yaml)
+            return {}
+
+    def _emit_manifest_debug_message(self, extra_args: dict[str, Any]) -> None:
+        extra_args["path_to_yaml"] = self._path_to_yaml
+
+    @staticmethod
+    def _parse(connection_definition_str: str) -> ConnectionDefinition:
+        """
+        Parses a yaml file into a manifest. Component references still exist in the manifest which will be
+        resolved during the creating of the DeclarativeSource.
+        :param connection_definition_str: yaml string to parse
+        :return: The ConnectionDefinition parsed from connection_definition_str
+        """
+        return yaml.safe_load(connection_definition_str)  # type: ignore # yaml.safe_load doesn't return a type but know it is a Mapping