airbyte-cdk 0.72.0__py3-none-any.whl → 6.13.1.dev4106__py3-none-any.whl

airbyte_cdk/sources/utils/slice_logger.py

@@ -27,7 +27,10 @@ class SliceLogger(ABC):
         printable_slice = dict(_slice) if _slice else _slice
         return AirbyteMessage(
             type=MessageType.LOG,
-            log=AirbyteLogMessage(level=Level.INFO, message=f"{SliceLogger.SLICE_LOG_PREFIX}{json.dumps(printable_slice, default=str)}"),
+            log=AirbyteLogMessage(
+                level=Level.INFO,
+                message=f"{SliceLogger.SLICE_LOG_PREFIX}{json.dumps(printable_slice, default=str)}",
+            ),
         )
 
     @abstractmethod

airbyte_cdk/sources/utils/transform.py

@@ -5,11 +5,17 @@
 import logging
 from distutils.util import strtobool
 from enum import Flag, auto
-from typing import Any, Callable, Dict, Mapping, Optional
+from typing import Any, Callable, Dict, Generator, Mapping, Optional, cast
 
-from jsonschema import Draft7Validator, ValidationError, validators
+from jsonschema import Draft7Validator, RefResolver, ValidationError, Validator, validators
 
-json_to_python_simple = {"string": str, "number": float, "integer": int, "boolean": bool, "null": type(None)}
+json_to_python_simple = {
+    "string": str,
+    "number": float,
+    "integer": int,
+    "boolean": bool,
+    "null": type(None),
+}
 json_to_python = {**json_to_python_simple, **{"object": dict, "array": list}}
 python_to_json = {v: k for k, v in json_to_python.items()}
 
@@ -24,7 +30,7 @@ class TransformConfig(Flag):
         ```
     """
 
-    # No action taken, default behaviour. Cannot be combined with any other options.
+    # No action taken, default behavior. Cannot be combined with any other options.
     NoTransform = auto()
     # Applies default type casting with default_convert method which converts
     # values by applying simple type casting to specified jsonschema type.
@@ -56,19 +62,25 @@ class TypeTransformer:
             # Do not validate field we do not transform for maximum performance.
             if key in ["type", "array", "$ref", "properties", "items"]
         }
-        self._normalizer = validators.create(meta_schema=Draft7Validator.META_SCHEMA, validators=all_validators)
+        self._normalizer = validators.create(
+            meta_schema=Draft7Validator.META_SCHEMA, validators=all_validators
+        )
 
-    def registerCustomTransform(self, normalization_callback: Callable[[Any, Dict[str, Any]], Any]) -> Callable:
+    def registerCustomTransform(
+        self, normalization_callback: Callable[[Any, dict[str, Any]], Any]
+    ) -> Callable[[Any, dict[str, Any]], Any]:
         """
         Register custom normalization callback.
         :param normalization_callback function to be used for value
         normalization. Takes original value and part type schema. Should return
         normalized value. See docs/connector-development/cdk-python/schemas.md
         for details.
-        :return Same callbeck, this is usefull for using registerCustomTransform function as decorator.
+        :return Same callback, this is useful for using registerCustomTransform function as decorator.
         """
         if TransformConfig.CustomSchemaNormalization not in self._config:
-            raise Exception("Please set TransformConfig.CustomSchemaNormalization config before registering custom normalizer")
+            raise Exception(
+                "Please set TransformConfig.CustomSchemaNormalization config before registering custom normalizer"
+            )
         self._custom_normalizer = normalization_callback
         return normalization_callback
 
@@ -120,20 +132,32 @@ class TypeTransformer:
                 return bool(original_item)
             elif target_type == "array":
                 item_types = set(subschema.get("items", {}).get("type", set()))
-                if item_types.issubset(json_to_python_simple) and type(original_item) in json_to_python_simple.values():
+                if (
+                    item_types.issubset(json_to_python_simple)
+                    and type(original_item) in json_to_python_simple.values()
+                ):
                     return [original_item]
         except (ValueError, TypeError):
             return original_item
         return original_item
 
-    def __get_normalizer(self, schema_key: str, original_validator: Callable):
+    def __get_normalizer(
+        self,
+        schema_key: str,
+        original_validator: Callable,  # type: ignore[type-arg]
+    ) -> Callable[[Any, Any, Any, dict[str, Any]], Generator[Any, Any, None]]:
         """
         Traverse through object fields using native jsonschema validator and apply normalization function.
         :param schema_key related json schema key that currently being validated/normalized.
         :original_validator: native jsonschema validator callback.
         """
 
-        def normalizator(validator_instance: Callable, property_value: Any, instance: Any, schema: Dict[str, Any]):
+        def normalizator(
+            validator_instance: Validator,
+            property_value: Any,
+            instance: Any,
+            schema: Dict[str, Any],
+        ) -> Generator[Any, Any, None]:
             """
             Jsonschema validator callable it uses for validating instance. We
             override default Draft7Validator to perform value transformation
@@ -146,10 +170,13 @@ class TypeTransformer:
             :
             """
 
-            def resolve(subschema):
+            def resolve(subschema: dict[str, Any]) -> dict[str, Any]:
                 if "$ref" in subschema:
-                    _, resolved = validator_instance.resolver.resolve(subschema["$ref"])
-                    return resolved
+                    _, resolved = cast(
+                        RefResolver,
+                        validator_instance.resolver,
+                    ).resolve(subschema["$ref"])
+                    return cast(dict[str, Any], resolved)
                 return subschema
 
             # Transform object and array values before running json schema type checking for each element.
@@ -168,11 +195,20 @@ class TypeTransformer:
                     instance[index] = self.__normalize(item, subschema)
 
             # Running native jsonschema traverse algorithm after field normalization is done.
-            yield from original_validator(validator_instance, property_value, instance, schema)
+            yield from original_validator(
+                validator_instance,
+                property_value,
+                instance,
+                schema,
+            )
 
         return normalizator
 
-    def transform(self, record: Dict[str, Any], schema: Mapping[str, Any]):
+    def transform(
+        self,
+        record: Dict[str, Any],
+        schema: Mapping[str, Any],
+    ) -> None:
         """
         Normalize and validate according to config.
         :param record: record instance for normalization/transformation. All modification are done by modifying existent object.
@@ -184,13 +220,11 @@ class TypeTransformer:
         for e in normalizer.iter_errors(record):
             """
             just calling normalizer.validate() would throw an exception on
-            first validation occurences and stop processing rest of schema.
+            first validation occurrences and stop processing rest of schema.
             """
             logger.warning(self.get_error_message(e))
 
     def get_error_message(self, e: ValidationError) -> str:
         instance_json_type = python_to_json[type(e.instance)]
         key_path = "." + ".".join(map(str, e.path))
-        return (
-            f"Failed to transform value {repr(e.instance)} of type '{instance_json_type}' to '{e.validator_value}', key path: '{key_path}'"
-        )
+        return f"Failed to transform value {repr(e.instance)} of type '{instance_json_type}' to '{e.validator_value}', key path: '{key_path}'"

airbyte_cdk/sql/_util/hashing.py

@@ -0,0 +1,34 @@
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+"""Hashing utils for Airbyte."""
+
+from __future__ import annotations
+
+import hashlib
+from collections.abc import Mapping
+
+HASH_SEED = "Airbyte:"
+"""Additional seed for randomizing one-way hashed strings."""
+
+
+def one_way_hash(
+    obj: Mapping[str, str] | list[str] | object,
+    /,
+) -> str:
+    """Return a one-way hash of the given string.
+
+    To ensure a unique domain of hashes, we prepend a seed to the string before hashing.
+    """
+    string_to_hash: str
+    if isinstance(obj, Mapping):
+        # Recursively sort and convert nested dictionaries to tuples of key-value pairs
+        string_to_hash = str(sorted((k, one_way_hash(v)) for k, v in obj.items()))
+
+    elif isinstance(obj, list):
+        # Recursively hash elements of the list
+        string_to_hash = str([one_way_hash(item) for item in obj])
+
+    else:
+        # Convert the object to a string
+        string_to_hash = str(obj)
+
+    return hashlib.sha256((HASH_SEED + str(string_to_hash)).encode()).hexdigest()

airbyte_cdk/sql/_util/name_normalizers.py

@@ -0,0 +1,92 @@
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+"""Name normalizer classes."""
+
+from __future__ import annotations
+
+import abc
+import functools
+import re
+from typing import TYPE_CHECKING
+
+from airbyte_cdk.sql import exceptions as exc
+
+if TYPE_CHECKING:
+    from collections.abc import Iterable
+
+
+class NameNormalizerBase(abc.ABC):
+    """Abstract base class for name normalizers."""
+
+    @staticmethod
+    @abc.abstractmethod
+    def normalize(name: str) -> str:
+        """Return the normalized name."""
+        ...
+
+    @classmethod
+    def normalize_set(cls, str_iter: Iterable[str]) -> set[str]:
+        """Converts string iterable to a set of lower case strings."""
+        return {cls.normalize(s) for s in str_iter}
+
+    @classmethod
+    def normalize_list(cls, str_iter: Iterable[str]) -> list[str]:
+        """Converts string iterable to a list of lower case strings."""
+        return [cls.normalize(s) for s in str_iter]
+
+    @classmethod
+    def check_matched(cls, name1: str, name2: str) -> bool:
+        """Return True if the two names match after each is normalized."""
+        return cls.normalize(name1) == cls.normalize(name2)
+
+    @classmethod
+    def check_normalized(cls, name: str) -> bool:
+        """Return True if the name is already normalized."""
+        return cls.normalize(name) == name
+
+
+class LowerCaseNormalizer(NameNormalizerBase):
+    """A name normalizer that converts names to lower case."""
+
+    @staticmethod
+    @functools.cache
+    def normalize(name: str) -> str:
+        """Return the normalized name.
+
+        - All non-alphanumeric characters are replaced with underscores.
+        - Any names that start with a numeric ("1", "2", "123", "1b" etc.) are prefixed
+          with and underscore ("_1", "_2", "_123", "_1b" etc.)
+
+        Examples:
+        - "Hello World!" -> "hello_world"
+        - "Hello, World!" -> "hello__world"
+        - "Hello - World" -> "hello___world"
+        - "___Hello, World___" -> "___hello__world___"
+        - "Average Sales (%)" -> "average_sales____"
+        - "Average Sales (#)" -> "average_sales____"
+        - "+1" -> "_1"
+        - "-1" -> "_1"
+        """
+        result = name
+
+        # Replace all non-alphanumeric characters with underscores.
+        result = re.sub("[^A-Za-z0-9]", "_", result.lower())
+
+        # Check if name starts with a number and prepend "_" if it does.
+        if result and result[0].isdigit():
+            # Most databases do not allow identifiers to start with a number.
+            result = f"_{result}"
+
+        if not result.replace("_", ""):
+            raise exc.AirbyteNameNormalizationError(
+                message="Name cannot be empty after normalization.",
+                raw_name=name,
+                normalization_result=result,
+            )
+
+        return result
+
+
+__all__ = [
+    "NameNormalizerBase",
+    "LowerCaseNormalizer",
+]

airbyte_cdk/sql/constants.py

@@ -0,0 +1,32 @@
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+"""Constants shared across the Airbyte codebase."""
+
+from __future__ import annotations
+
+DEBUG_MODE = False  # Set to True to enable additional debug logging.
+
+AB_EXTRACTED_AT_COLUMN = "_airbyte_extracted_at"
+"""A column that stores the timestamp when the record was extracted."""
+
+AB_META_COLUMN = "_airbyte_meta"
+"""A column that stores metadata about the record."""
+
+AB_RAW_ID_COLUMN = "_airbyte_raw_id"
+"""A column that stores a unique identifier for each row in the source data.
+
+Note: The interpretation of this column is slightly different from in Airbyte Dv2 destinations.
+In Airbyte Dv2 destinations, this column points to a row in a separate 'raw' table. In Airbyte,
+this column is simply used as a unique identifier for each record as it is received.
+
+Airbyte uses ULIDs for this column, which are identifiers that can be sorted by time
+received. This allows us to determine the debug the order of records as they are received, even if
+the source provides records that are tied or received out of order from the perspective of their
+`emitted_at` (`_airbyte_extracted_at`) timestamps.
+"""
+
+AB_INTERNAL_COLUMNS = {
+    AB_RAW_ID_COLUMN,
+    AB_EXTRACTED_AT_COLUMN,
+    AB_META_COLUMN,
+}
+"""A set of internal columns that are reserved for Airbyte's internal use."""

airbyte_cdk/sql/exceptions.py

@@ -0,0 +1,235 @@
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+
+"""All exceptions used in Airbyte.
+
+This design is modeled after structlog's exceptions, in that we bias towards auto-generated
+property prints rather than sentence-like string concatenation.
+
+E.g. Instead of this:
+
+> `Subprocess failed with exit code '1'`
+
+We do this:
+
+> `Subprocess failed. (exit_code=1)`
+
+The benefit of this approach is that we can easily support structured logging, and we can
+easily add new properties to exceptions without having to update all the places where they
+are raised. We can also support any arbitrary number of properties in exceptions, without spending
+time on building sentence-like string constructions with optional inputs.
+
+
+In addition, the following principles are applied for exception class design:
+
+- All exceptions inherit from a common base class.
+- All exceptions have a message attribute.
+- The first line of the docstring is used as the default message.
+- The default message can be overridden by explicitly setting the message attribute.
+- Exceptions may optionally have a guidance attribute.
+- Exceptions may optionally have a help_url attribute.
+- Rendering is automatically handled by the base class.
+- Any helpful context not defined by the exception class can be passed in the `context` dict arg.
+- Within reason, avoid sending PII to the exception constructor.
+- Exceptions are dataclasses, so they can be instantiated with keyword arguments.
+- Use the 'from' syntax to chain exceptions when it is helpful to do so.
+  E.g. `raise AirbyteConnectorNotFoundError(...) from FileNotFoundError(connector_path)`
+- Any exception that adds a new property should also be decorated as `@dataclass`.
+"""
+
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass
+from pathlib import Path
+from textwrap import indent
+from typing import Any
+
+NEW_ISSUE_URL = "https://github.com/airbytehq/airbyte/issues/new/choose"
+DOCS_URL_BASE = "https://https://docs.airbyte.com/"
+DOCS_URL = f"{DOCS_URL_BASE}/airbyte.html"
+
+VERTICAL_SEPARATOR = "\n" + "-" * 60
+
+
+# Base error class
+
+
+@dataclass
+class AirbyteError(Exception):
+    """Base class for exceptions in Airbyte."""
+
+    guidance: str | None = None
+    help_url: str | None = None
+    log_text: str | list[str] | None = None
+    log_file: Path | None = None
+    context: dict[str, Any] | None = None
+    message: str | None = None
+    original_exception: Exception | None = None
+
+    def get_message(self) -> str:
+        """Return the best description for the exception.
+
+        We resolve the following in order:
+        1. The message sent to the exception constructor (if provided).
+        2. The first line of the class's docstring.
+        """
+        if self.message:
+            return self.message
+
+        return self.__doc__.split("\n")[0] if self.__doc__ else ""
+
+    def __str__(self) -> str:
+        """Return a string representation of the exception."""
+        special_properties = [
+            "message",
+            "guidance",
+            "help_url",
+            "log_text",
+            "context",
+            "log_file",
+            "original_exception",
+        ]
+        display_properties = {
+            k: v
+            for k, v in self.__dict__.items()
+            if k not in special_properties and not k.startswith("_") and v is not None
+        }
+        display_properties.update(self.context or {})
+        context_str = "\n    ".join(
+            f"{str(k).replace('_', ' ').title()}: {v!r}" for k, v in display_properties.items()
+        )
+        exception_str = (
+            f"{self.get_message()} ({self.__class__.__name__})"
+            + VERTICAL_SEPARATOR
+            + f"\n{self.__class__.__name__}: {self.get_message()}"
+        )
+
+        if self.guidance:
+            exception_str += f"\n    {self.guidance}"
+
+        if self.help_url:
+            exception_str += f"\n    More info: {self.help_url}"
+
+        if context_str:
+            exception_str += "\n    " + context_str
+
+        if self.log_file:
+            exception_str += f"\n    Log file: {self.log_file.absolute()!s}"
+
+        if self.log_text:
+            if isinstance(self.log_text, list):
+                self.log_text = "\n".join(self.log_text)
+
+            exception_str += f"\n    Log output: \n    {indent(self.log_text, '    ')}"
+
+        if self.original_exception:
+            exception_str += VERTICAL_SEPARATOR + f"\nCaused by: {self.original_exception!s}"
+
+        return exception_str
+
+    def __repr__(self) -> str:
+        """Return a string representation of the exception."""
+        class_name = self.__class__.__name__
+        properties_str = ", ".join(
+            f"{k}={v!r}" for k, v in self.__dict__.items() if not k.startswith("_")
+        )
+        return f"{class_name}({properties_str})"
+
+    def safe_logging_dict(self) -> dict[str, Any]:
+        """Return a dictionary of the exception's properties which is safe for logging.
+
+        We avoid any properties which could potentially contain PII.
+        """
+        result = {
+            # The class name is safe to log:
+            "class": self.__class__.__name__,
+            # We discourage interpolated strings in 'message' so that this should never contain PII:
+            "message": self.get_message(),
+        }
+        safe_attrs = ["connector_name", "stream_name", "violation", "exit_code"]
+        for attr in safe_attrs:
+            if hasattr(self, attr):
+                result[attr] = getattr(self, attr)
+
+        return result
+
+
+# Airbyte Internal Errors (these are probably bugs)
+
+
+@dataclass
+class AirbyteInternalError(AirbyteError):
+    """An internal error occurred in Airbyte."""
+
+    guidance = "Please consider reporting this error to the Airbyte team."
+    help_url = NEW_ISSUE_URL
+
+
+# Airbyte Input Errors (replaces ValueError for user input)
+
+
+@dataclass
+class AirbyteInputError(AirbyteError, ValueError):
+    """The input provided to Airbyte did not match expected validation rules.
+
+    This inherits from ValueError so that it can be used as a drop-in replacement for
+    ValueError in the Airbyte API.
+    """
+
+    guidance = "Please check the provided value and try again."
+    help_url = DOCS_URL
+    input_value: str | None = None
+
+
+# Normalization Errors
+
+
+@dataclass
+class AirbyteNameNormalizationError(AirbyteError, ValueError):
+    """Error occurred while normalizing a table or column name."""
+
+    guidance = (
+        "Please consider renaming the source object if possible, or "
+        "raise an issue in GitHub if not."
+    )
+    help_url = NEW_ISSUE_URL
+
+    raw_name: str | None = None
+    normalization_result: str | None = None
+
+
+@dataclass
+class AirbyteConnectorError(AirbyteError):
+    """Error when running the connector."""
+
+    connector_name: str | None = None
+
+    def __post_init__(self) -> None:
+        """Set the log file path for the connector."""
+        self.log_file = self._get_log_file()
+        if not self.guidance and self.log_file:
+            self.guidance = "Please review the log file for more information."
+
+    def _get_log_file(self) -> Path | None:
+        """Return the log file path for the connector."""
+        if self.connector_name:
+            logger = logging.getLogger(f"airbyte.{self.connector_name}")
+
+            log_paths: list[Path] = [
+                Path(handler.baseFilename).absolute()
+                for handler in logger.handlers
+                if isinstance(handler, logging.FileHandler)
+            ]
+
+            if log_paths:
+                return log_paths[0]
+
+        return None
+
+
+@dataclass
+class AirbyteStreamNotFoundError(AirbyteConnectorError):
+    """Connector stream not found."""
+
+    stream_name: str | None = None
+    available_streams: list[str] | None = None

airbyte_cdk/sql/secrets.py

@@ -0,0 +1,123 @@
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+"""Base classes and methods for working with secrets in Airbyte."""
+
+from __future__ import annotations
+
+import json
+from typing import TYPE_CHECKING, Any
+
+from pydantic_core import CoreSchema, core_schema
+
+from airbyte_cdk.sql import exceptions as exc
+
+if TYPE_CHECKING:
+    from pydantic import GetCoreSchemaHandler, GetJsonSchemaHandler, ValidationInfo
+    from pydantic.json_schema import JsonSchemaValue
+
+
+class SecretString(str):
+    """A string that represents a secret.
+
+    This class is used to mark a string as a secret. When a secret is printed, it
+    will be masked to prevent accidental exposure of sensitive information when debugging
+    or when printing containing objects like dictionaries.
+
+    To create a secret string, simply instantiate the class with any string value:
+
+        ```python
+        secret = SecretString("my_secret_password")
+        ```
+
+    """
+
+    __slots__ = ()
+
+    def __repr__(self) -> str:
+        """Override the representation of the secret string to return a masked value.
+
+        The secret string is always masked with `****` to prevent accidental exposure, unless
+        explicitly converted to a string. For instance, printing a config dictionary that contains
+        a secret will automatically mask the secret value instead of printing it in plain text.
+
+        However, if you explicitly convert the cast the secret as a string, such as when used
+        in an f-string, the secret will be exposed. This is the desired behavior to allow
+        secrets to be used in a controlled manner.
+        """
+        return "<SecretString: ****>"
+
+    def is_empty(self) -> bool:
+        """Check if the secret is an empty string."""
+        return len(self) == 0
+
+    def is_json(self) -> bool:
+        """Check if the secret string is a valid JSON string."""
+        try:
+            json.loads(self)
+        except (json.JSONDecodeError, Exception):
+            return False
+
+        return True
+
+    def __bool__(self) -> bool:
+        """Override the boolean value of the secret string.
+
+        Always returns `True` without inspecting contents.
+        """
+        return True
+
+    def parse_json(self) -> Any:
+        """Parse the secret string as JSON."""
+        try:
+            return json.loads(self)
+        except json.JSONDecodeError as ex:
+            raise exc.AirbyteInputError(
+                message="Failed to parse secret as JSON.",
+                context={
+                    "Message": ex.msg,
+                    "Position": ex.pos,
+                    "SecretString_Length": len(self),  # Debug secret blank or an unexpected format.
+                },
+            ) from None
+
+    # Pydantic compatibility
+
+    @classmethod
+    def validate(
+        cls,
+        v: Any,  # noqa: ANN401  # Must allow `Any` to match Pydantic signature
+        info: ValidationInfo,
+    ) -> SecretString:
+        """Validate the input value is valid as a secret string."""
+        _ = info  # Unused
+        if not isinstance(v, str):
+            raise exc.AirbyteInputError(
+                message="A valid `str` or `SecretString` object is required.",
+            )
+        return cls(v)
+
+    @classmethod
+    def __get_pydantic_core_schema__(  # noqa: PLW3201  # Pydantic dunder
+        cls,
+        source_type: Any,  # noqa: ANN401  # Must allow `Any` to match Pydantic signature
+        handler: GetCoreSchemaHandler,
+    ) -> CoreSchema:
+        """Return a modified core schema for the secret string."""
+        return core_schema.with_info_after_validator_function(
+            function=cls.validate, schema=handler(str), field_name=handler.field_name
+        )
+
+    @classmethod
+    def __get_pydantic_json_schema__(  # noqa: PLW3201  # Pydantic dunder method
+        cls, _core_schema: core_schema.CoreSchema, handler: GetJsonSchemaHandler
+    ) -> JsonSchemaValue:
+        """Return a modified JSON schema for the secret string.
+
+        - `writeOnly=True` is the official way to prevent secrets from being exposed inadvertently.
+        - `Format=password` is a popular and readable convention to indicate the field is sensitive.
+        """
+        _ = _core_schema, handler  # Unused
+        return {
+            "type": "string",
+            "format": "password",
+            "writeOnly": True,
+        }