airbyte-cdk 0.0.0.dev0__py3-none-any.whl

airbyte_cdk/sources/utils/transform.py

@@ -0,0 +1,277 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+import logging
+from enum import Flag, auto
+from typing import Any, Callable, Dict, Generator, Mapping, Optional, cast
+
+from jsonschema import Draft7Validator, RefResolver, ValidationError, Validator, validators
+
+MAX_NESTING_DEPTH = 3
+json_to_python_simple = {
+    "string": str,
+    "number": float,
+    "integer": int,
+    "boolean": bool,
+    "null": type(None),
+}
+json_to_python = {**json_to_python_simple, **{"object": dict, "array": list}}
+python_to_json = {v: k for k, v in json_to_python.items()}
+
+logger = logging.getLogger("airbyte")
+
+_TRUTHY_STRINGS = ("y", "yes", "t", "true", "on", "1")
+_FALSEY_STRINGS = ("n", "no", "f", "false", "off", "0")
+
+
+def _strtobool(value: str, /) -> int:
+    """Mimic the behavior of distutils.util.strtobool.
+
+    From: https://docs.python.org/2/distutils/apiref.html#distutils.util.strtobool
+
+    > Convert a string representation of truth to true (1) or false (0).
+    > True values are y, yes, t, true, on and 1; false values are n, no, f, false, off and 0. Raises
+    > `ValueError` if val is anything else.
+    """
+    normalized_str = value.lower().strip()
+    if normalized_str in _TRUTHY_STRINGS:
+        return 1
+
+    if normalized_str in _FALSEY_STRINGS:
+        return 0
+
+    raise ValueError(f"Invalid boolean value: {normalized_str}")
+
+
+class TransformConfig(Flag):
+    """
+    TypeTransformer class config. Configs can be combined using bitwise or operator e.g.
+        ```
+        TransformConfig.DefaultSchemaNormalization | TransformConfig.CustomSchemaNormalization
+        ```
+    """
+
+    # No action taken, default behavior. Cannot be combined with any other options.
+    NoTransform = auto()
+    # Applies default type casting with default_convert method which converts
+    # values by applying simple type casting to specified jsonschema type.
+    DefaultSchemaNormalization = auto()
+    # Allow registering custom type transformation callback. Can be combined
+    # with DefaultSchemaNormalization. In this case default type casting would
+    # be applied before custom one.
+    CustomSchemaNormalization = auto()
+
+
+class TypeTransformer:
+    """
+    Class for transforming object before output.
+    """
+
+    _custom_normalizer: Optional[Callable[[Any, Dict[str, Any]], Any]] = None
+
+    def __init__(self, config: TransformConfig):
+        """
+        Initialize TypeTransformer instance.
+        :param config Transform config that would be applied to object
+        """
+        if TransformConfig.NoTransform in config and config != TransformConfig.NoTransform:
+            raise Exception("NoTransform option cannot be combined with other flags.")
+        self._config = config
+        all_validators = {
+            key: self.__get_normalizer(key, orig_validator)
+            for key, orig_validator in Draft7Validator.VALIDATORS.items()
+            # Do not validate field we do not transform for maximum performance.
+            if key in ["type", "array", "$ref", "properties", "items"]
+        }
+        self._normalizer = validators.create(
+            meta_schema=Draft7Validator.META_SCHEMA, validators=all_validators
+        )
+
+    def registerCustomTransform(
+        self, normalization_callback: Callable[[Any, dict[str, Any]], Any]
+    ) -> Callable[[Any, dict[str, Any]], Any]:
+        """
+        Register custom normalization callback.
+        :param normalization_callback function to be used for value
+        normalization. Takes original value and part type schema. Should return
+        normalized value. See docs/connector-development/cdk-python/schemas.md
+        for details.
+        :return Same callback, this is useful for using registerCustomTransform function as decorator.
+        """
+        if TransformConfig.CustomSchemaNormalization not in self._config:
+            raise Exception(
+                "Please set TransformConfig.CustomSchemaNormalization config before registering custom normalizer"
+            )
+        self._custom_normalizer = normalization_callback
+        return normalization_callback
+
+    def __normalize(self, original_item: Any, subschema: Dict[str, Any]) -> Any:
+        """
+        Applies different transform function to object's field according to config.
+        :param original_item original value of field.
+        :param subschema part of the jsonschema containing field type/format data.
+        :return Final field value.
+        """
+        if TransformConfig.DefaultSchemaNormalization in self._config:
+            original_item = self.default_convert(original_item, subschema)
+
+        if self._custom_normalizer:
+            original_item = self._custom_normalizer(original_item, subschema)
+        return original_item
+
+    @staticmethod
+    def default_convert(original_item: Any, subschema: Dict[str, Any]) -> Any:
+        """
+        Default transform function that is used when TransformConfig.DefaultSchemaNormalization flag set.
+        :param original_item original value of field.
+        :param subschema part of the jsonschema containing field type/format data.
+        :return transformed field value.
+        """
+        target_type = subschema.get("type", [])
+        if original_item is None and "null" in target_type:
+            return None
+        if isinstance(target_type, list):
+            # jsonschema type could either be a single string or array of type
+            # strings. In case if there is some disambigous and more than one
+            # type (except null) do not do any conversion and return original
+            # value. If type array has one type and null i.e. {"type":
+            # ["integer", "null"]}, convert value to specified type.
+            target_type = [t for t in target_type if t != "null"]
+            if len(target_type) != 1:
+                return original_item
+            target_type = target_type[0]
+        try:
+            if target_type == "string":
+                return str(original_item)
+            elif target_type == "number":
+                return float(original_item)
+            elif target_type == "integer":
+                return int(original_item)
+            elif target_type == "boolean":
+                if isinstance(original_item, str):
+                    return _strtobool(original_item) == 1
+                return bool(original_item)
+            elif target_type == "array":
+                item_types = set(subschema.get("items", {}).get("type", set()))
+                if (
+                    item_types.issubset(json_to_python_simple)
+                    and type(original_item) in json_to_python_simple.values()
+                ):
+                    return [original_item]
+        except (ValueError, TypeError):
+            return original_item
+        return original_item
+
+    def __get_normalizer(
+        self,
+        schema_key: str,
+        original_validator: Callable,  # type: ignore[type-arg]
+    ) -> Callable[[Any, Any, Any, dict[str, Any]], Generator[Any, Any, None]]:
+        """
+        Traverse through object fields using native jsonschema validator and apply normalization function.
+        :param schema_key related json schema key that currently being validated/normalized.
+        :original_validator: native jsonschema validator callback.
+        """
+
+        def normalizator(
+            validator_instance: Validator,
+            property_value: Any,
+            instance: Any,
+            schema: Dict[str, Any],
+        ) -> Generator[Any, Any, None]:
+            """
+            Jsonschema validator callable it uses for validating instance. We
+            override default Draft7Validator to perform value transformation
+            before validation take place. We do not take any action except
+            logging warn if object does not conform to json schema, just using
+            jsonschema algorithm to traverse through object fields.
+            Look
+            https://python-jsonschema.readthedocs.io/en/stable/creating/?highlight=validators.create#jsonschema.validators.create
+            validators parameter for detailed description.
+            :
+            """
+
+            def resolve(subschema: dict[str, Any]) -> dict[str, Any]:
+                if "$ref" in subschema:
+                    _, resolved = cast(
+                        RefResolver,
+                        validator_instance.resolver,
+                    ).resolve(subschema["$ref"])
+                    return cast(dict[str, Any], resolved)
+                return subschema
+
+            # Transform object and array values before running json schema type checking for each element.
+            # Recursively normalize every value of the "instance" sub-object,
+            # if "instance" is an incorrect type - skip recursive normalization of "instance"
+            if schema_key == "properties" and isinstance(instance, dict):
+                for k, subschema in property_value.items():
+                    if k in instance:
+                        subschema = resolve(subschema)
+                        instance[k] = self.__normalize(instance[k], subschema)
+            # Recursively normalize every item of the "instance" sub-array,
+            # if "instance" is an incorrect type - skip recursive normalization of "instance"
+            elif schema_key == "items" and isinstance(instance, list):
+                subschema = resolve(property_value)
+                for index, item in enumerate(instance):
+                    instance[index] = self.__normalize(item, subschema)
+
+            # Running native jsonschema traverse algorithm after field normalization is done.
+            yield from original_validator(
+                validator_instance,
+                property_value,
+                instance,
+                schema,
+            )
+
+        return normalizator
+
+    def transform(
+        self,
+        record: Dict[str, Any],
+        schema: Mapping[str, Any],
+    ) -> None:
+        """
+        Normalize and validate according to config.
+        :param record: record instance for normalization/transformation. All modification are done by modifying existent object.
+        :param schema: object's jsonschema for normalization.
+        """
+        if TransformConfig.NoTransform in self._config:
+            return
+        normalizer = self._normalizer(schema)
+        for e in normalizer.iter_errors(record):
+            """
+            just calling normalizer.validate() would throw an exception on
+            first validation occurrences and stop processing rest of schema.
+            """
+            logger.warning(self.get_error_message(e))
+
+    def get_error_message(self, e: ValidationError) -> str:
+        """
+        Construct a sanitized error message from a ValidationError instance.
+        """
+        field_path = ".".join(map(str, e.path))
+        type_structure = self._get_type_structure(e.instance)
+
+        return f"Failed to transform value from type '{type_structure}' to type '{e.validator_value}' at path: '{field_path}'"
+
+    def _get_type_structure(self, input_data: Any, current_depth: int = 0) -> Any:
+        """
+        Get the structure of a given input data for use in error message construction.
+        """
+        # Handle null values
+        if input_data is None:
+            return "null"
+
+        # Avoid recursing too deep
+        if current_depth >= MAX_NESTING_DEPTH:
+            return "object" if isinstance(input_data, dict) else python_to_json[type(input_data)]
+
+        if isinstance(input_data, dict):
+            return {
+                key: self._get_type_structure(field_value, current_depth + 1)
+                for key, field_value in input_data.items()
+            }
+
+        else:
+            return python_to_json[type(input_data)]

airbyte_cdk/sources/utils/types.py

@@ -0,0 +1,7 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+from typing import Union
+
+JsonType = Union[dict[str, "JsonType"], list["JsonType"], str, int, float, bool, None]

airbyte_cdk/sql/_util/hashing.py

@@ -0,0 +1,34 @@
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+"""Hashing utils for Airbyte."""
+
+from __future__ import annotations
+
+import hashlib
+from collections.abc import Mapping
+
+HASH_SEED = "Airbyte:"
+"""Additional seed for randomizing one-way hashed strings."""
+
+
+def one_way_hash(
+    obj: Mapping[str, str] | list[str] | object,
+    /,
+) -> str:
+    """Return a one-way hash of the given string.
+
+    To ensure a unique domain of hashes, we prepend a seed to the string before hashing.
+    """
+    string_to_hash: str
+    if isinstance(obj, Mapping):
+        # Recursively sort and convert nested dictionaries to tuples of key-value pairs
+        string_to_hash = str(sorted((k, one_way_hash(v)) for k, v in obj.items()))
+
+    elif isinstance(obj, list):
+        # Recursively hash elements of the list
+        string_to_hash = str([one_way_hash(item) for item in obj])
+
+    else:
+        # Convert the object to a string
+        string_to_hash = str(obj)
+
+    return hashlib.sha256((HASH_SEED + str(string_to_hash)).encode()).hexdigest()

airbyte_cdk/sql/_util/name_normalizers.py

@@ -0,0 +1,92 @@
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+"""Name normalizer classes."""
+
+from __future__ import annotations
+
+import abc
+import functools
+import re
+from typing import TYPE_CHECKING
+
+from airbyte_cdk.sql import exceptions as exc
+
+if TYPE_CHECKING:
+    from collections.abc import Iterable
+
+
+class NameNormalizerBase(abc.ABC):
+    """Abstract base class for name normalizers."""
+
+    @staticmethod
+    @abc.abstractmethod
+    def normalize(name: str) -> str:
+        """Return the normalized name."""
+        ...
+
+    @classmethod
+    def normalize_set(cls, str_iter: Iterable[str]) -> set[str]:
+        """Converts string iterable to a set of lower case strings."""
+        return {cls.normalize(s) for s in str_iter}
+
+    @classmethod
+    def normalize_list(cls, str_iter: Iterable[str]) -> list[str]:
+        """Converts string iterable to a list of lower case strings."""
+        return [cls.normalize(s) for s in str_iter]
+
+    @classmethod
+    def check_matched(cls, name1: str, name2: str) -> bool:
+        """Return True if the two names match after each is normalized."""
+        return cls.normalize(name1) == cls.normalize(name2)
+
+    @classmethod
+    def check_normalized(cls, name: str) -> bool:
+        """Return True if the name is already normalized."""
+        return cls.normalize(name) == name
+
+
+class LowerCaseNormalizer(NameNormalizerBase):
+    """A name normalizer that converts names to lower case."""
+
+    @staticmethod
+    @functools.cache
+    def normalize(name: str) -> str:
+        """Return the normalized name.
+
+        - All non-alphanumeric characters are replaced with underscores.
+        - Any names that start with a numeric ("1", "2", "123", "1b" etc.) are prefixed
+          with and underscore ("_1", "_2", "_123", "_1b" etc.)
+
+        Examples:
+        - "Hello World!" -> "hello_world"
+        - "Hello, World!" -> "hello__world"
+        - "Hello - World" -> "hello___world"
+        - "___Hello, World___" -> "___hello__world___"
+        - "Average Sales (%)" -> "average_sales____"
+        - "Average Sales (#)" -> "average_sales____"
+        - "+1" -> "_1"
+        - "-1" -> "_1"
+        """
+        result = name
+
+        # Replace all non-alphanumeric characters with underscores.
+        result = re.sub("[^A-Za-z0-9]", "_", result.lower())
+
+        # Check if name starts with a number and prepend "_" if it does.
+        if result and result[0].isdigit():
+            # Most databases do not allow identifiers to start with a number.
+            result = f"_{result}"
+
+        if not result.replace("_", ""):
+            raise exc.AirbyteNameNormalizationError(
+                message="Name cannot be empty after normalization.",
+                raw_name=name,
+                normalization_result=result,
+            )
+
+        return result
+
+
+__all__ = [
+    "NameNormalizerBase",
+    "LowerCaseNormalizer",
+]

airbyte_cdk/sql/constants.py

@@ -0,0 +1,32 @@
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+"""Constants shared across the Airbyte codebase."""
+
+from __future__ import annotations
+
+DEBUG_MODE = False  # Set to True to enable additional debug logging.
+
+AB_EXTRACTED_AT_COLUMN = "_airbyte_extracted_at"
+"""A column that stores the timestamp when the record was extracted."""
+
+AB_META_COLUMN = "_airbyte_meta"
+"""A column that stores metadata about the record."""
+
+AB_RAW_ID_COLUMN = "_airbyte_raw_id"
+"""A column that stores a unique identifier for each row in the source data.
+
+Note: The interpretation of this column is slightly different from in Airbyte Dv2 destinations.
+In Airbyte Dv2 destinations, this column points to a row in a separate 'raw' table. In Airbyte,
+this column is simply used as a unique identifier for each record as it is received.
+
+Airbyte uses ULIDs for this column, which are identifiers that can be sorted by time
+received. This allows us to determine the debug the order of records as they are received, even if
+the source provides records that are tied or received out of order from the perspective of their
+`emitted_at` (`_airbyte_extracted_at`) timestamps.
+"""
+
+AB_INTERNAL_COLUMNS = {
+    AB_RAW_ID_COLUMN,
+    AB_EXTRACTED_AT_COLUMN,
+    AB_META_COLUMN,
+}
+"""A set of internal columns that are reserved for Airbyte's internal use."""

airbyte_cdk/sql/exceptions.py

@@ -0,0 +1,235 @@
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+
+"""All exceptions used in Airbyte.
+
+This design is modeled after structlog's exceptions, in that we bias towards auto-generated
+property prints rather than sentence-like string concatenation.
+
+E.g. Instead of this:
+
+> `Subprocess failed with exit code '1'`
+
+We do this:
+
+> `Subprocess failed. (exit_code=1)`
+
+The benefit of this approach is that we can easily support structured logging, and we can
+easily add new properties to exceptions without having to update all the places where they
+are raised. We can also support any arbitrary number of properties in exceptions, without spending
+time on building sentence-like string constructions with optional inputs.
+
+
+In addition, the following principles are applied for exception class design:
+
+- All exceptions inherit from a common base class.
+- All exceptions have a message attribute.
+- The first line of the docstring is used as the default message.
+- The default message can be overridden by explicitly setting the message attribute.
+- Exceptions may optionally have a guidance attribute.
+- Exceptions may optionally have a help_url attribute.
+- Rendering is automatically handled by the base class.
+- Any helpful context not defined by the exception class can be passed in the `context` dict arg.
+- Within reason, avoid sending PII to the exception constructor.
+- Exceptions are dataclasses, so they can be instantiated with keyword arguments.
+- Use the 'from' syntax to chain exceptions when it is helpful to do so.
+  E.g. `raise AirbyteConnectorNotFoundError(...) from FileNotFoundError(connector_path)`
+- Any exception that adds a new property should also be decorated as `@dataclass`.
+"""
+
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass
+from pathlib import Path
+from textwrap import indent
+from typing import Any
+
+NEW_ISSUE_URL = "https://github.com/airbytehq/airbyte/issues/new/choose"
+DOCS_URL_BASE = "https://https://docs.airbyte.com/"
+DOCS_URL = f"{DOCS_URL_BASE}/airbyte.html"
+
+VERTICAL_SEPARATOR = "\n" + "-" * 60
+
+
+# Base error class
+
+
+@dataclass
+class AirbyteError(Exception):
+    """Base class for exceptions in Airbyte."""
+
+    guidance: str | None = None
+    help_url: str | None = None
+    log_text: str | list[str] | None = None
+    log_file: Path | None = None
+    context: dict[str, Any] | None = None
+    message: str | None = None
+    original_exception: Exception | None = None
+
+    def get_message(self) -> str:
+        """Return the best description for the exception.
+
+        We resolve the following in order:
+        1. The message sent to the exception constructor (if provided).
+        2. The first line of the class's docstring.
+        """
+        if self.message:
+            return self.message
+
+        return self.__doc__.split("\n")[0] if self.__doc__ else ""
+
+    def __str__(self) -> str:
+        """Return a string representation of the exception."""
+        special_properties = [
+            "message",
+            "guidance",
+            "help_url",
+            "log_text",
+            "context",
+            "log_file",
+            "original_exception",
+        ]
+        display_properties = {
+            k: v
+            for k, v in self.__dict__.items()
+            if k not in special_properties and not k.startswith("_") and v is not None
+        }
+        display_properties.update(self.context or {})
+        context_str = "\n    ".join(
+            f"{str(k).replace('_', ' ').title()}: {v!r}" for k, v in display_properties.items()
+        )
+        exception_str = (
+            f"{self.get_message()} ({self.__class__.__name__})"
+            + VERTICAL_SEPARATOR
+            + f"\n{self.__class__.__name__}: {self.get_message()}"
+        )
+
+        if self.guidance:
+            exception_str += f"\n    {self.guidance}"
+
+        if self.help_url:
+            exception_str += f"\n    More info: {self.help_url}"
+
+        if context_str:
+            exception_str += "\n    " + context_str
+
+        if self.log_file:
+            exception_str += f"\n    Log file: {self.log_file.absolute()!s}"
+
+        if self.log_text:
+            if isinstance(self.log_text, list):
+                self.log_text = "\n".join(self.log_text)
+
+            exception_str += f"\n    Log output: \n    {indent(self.log_text, '    ')}"
+
+        if self.original_exception:
+            exception_str += VERTICAL_SEPARATOR + f"\nCaused by: {self.original_exception!s}"
+
+        return exception_str
+
+    def __repr__(self) -> str:
+        """Return a string representation of the exception."""
+        class_name = self.__class__.__name__
+        properties_str = ", ".join(
+            f"{k}={v!r}" for k, v in self.__dict__.items() if not k.startswith("_")
+        )
+        return f"{class_name}({properties_str})"
+
+    def safe_logging_dict(self) -> dict[str, Any]:
+        """Return a dictionary of the exception's properties which is safe for logging.
+
+        We avoid any properties which could potentially contain PII.
+        """
+        result = {
+            # The class name is safe to log:
+            "class": self.__class__.__name__,
+            # We discourage interpolated strings in 'message' so that this should never contain PII:
+            "message": self.get_message(),
+        }
+        safe_attrs = ["connector_name", "stream_name", "violation", "exit_code"]
+        for attr in safe_attrs:
+            if hasattr(self, attr):
+                result[attr] = getattr(self, attr)
+
+        return result
+
+
+# Airbyte Internal Errors (these are probably bugs)
+
+
+@dataclass
+class AirbyteInternalError(AirbyteError):
+    """An internal error occurred in Airbyte."""
+
+    guidance = "Please consider reporting this error to the Airbyte team."
+    help_url = NEW_ISSUE_URL
+
+
+# Airbyte Input Errors (replaces ValueError for user input)
+
+
+@dataclass
+class AirbyteInputError(AirbyteError, ValueError):
+    """The input provided to Airbyte did not match expected validation rules.
+
+    This inherits from ValueError so that it can be used as a drop-in replacement for
+    ValueError in the Airbyte API.
+    """
+
+    guidance = "Please check the provided value and try again."
+    help_url = DOCS_URL
+    input_value: str | None = None
+
+
+# Normalization Errors
+
+
+@dataclass
+class AirbyteNameNormalizationError(AirbyteError, ValueError):
+    """Error occurred while normalizing a table or column name."""
+
+    guidance = (
+        "Please consider renaming the source object if possible, or "
+        "raise an issue in GitHub if not."
+    )
+    help_url = NEW_ISSUE_URL
+
+    raw_name: str | None = None
+    normalization_result: str | None = None
+
+
+@dataclass
+class AirbyteConnectorError(AirbyteError):
+    """Error when running the connector."""
+
+    connector_name: str | None = None
+
+    def __post_init__(self) -> None:
+        """Set the log file path for the connector."""
+        self.log_file = self._get_log_file()
+        if not self.guidance and self.log_file:
+            self.guidance = "Please review the log file for more information."
+
+    def _get_log_file(self) -> Path | None:
+        """Return the log file path for the connector."""
+        if self.connector_name:
+            logger = logging.getLogger(f"airbyte.{self.connector_name}")
+
+            log_paths: list[Path] = [
+                Path(handler.baseFilename).absolute()
+                for handler in logger.handlers
+                if isinstance(handler, logging.FileHandler)
+            ]
+
+            if log_paths:
+                return log_paths[0]
+
+        return None
+
+
+@dataclass
+class AirbyteStreamNotFoundError(AirbyteConnectorError):
+    """Connector stream not found."""
+
+    stream_name: str | None = None
+    available_streams: list[str] | None = None