airbyte-cdk 0.0.0.dev0__py3-none-any.whl

airbyte_cdk/sources/declarative/parsers/__init__.py

@@ -0,0 +1,3 @@
+#
+# Copyright (c) 2021 Airbyte, Inc., all rights reserved.
+#

airbyte_cdk/sources/declarative/parsers/custom_code_compiler.py

@@ -0,0 +1,157 @@
+"""Contains functions to compile custom code from text."""
+
+import hashlib
+import os
+import sys
+from collections.abc import Mapping
+from types import ModuleType
+from typing import Any, cast
+
+from typing_extensions import Literal
+
+ChecksumType = Literal["md5", "sha256"]
+CHECKSUM_FUNCTIONS = {
+    "md5": hashlib.md5,
+    "sha256": hashlib.sha256,
+}
+COMPONENTS_MODULE_NAME = "components"
+SDM_COMPONENTS_MODULE_NAME = "source_declarative_manifest.components"
+INJECTED_MANIFEST = "__injected_declarative_manifest"
+INJECTED_COMPONENTS_PY = "__injected_components_py"
+INJECTED_COMPONENTS_PY_CHECKSUMS = "__injected_components_py_checksums"
+ENV_VAR_ALLOW_CUSTOM_CODE = "AIRBYTE_ENABLE_UNSAFE_CODE"
+
+
+class AirbyteCodeTamperedError(Exception):
+    """Raised when the connector's components module does not match its checksum.
+
+    This is a fatal error, as it can be a sign of code tampering.
+    """
+
+
+class AirbyteCustomCodeNotPermittedError(Exception):
+    """Raised when custom code is attempted to be run in an environment that does not support it."""
+
+    def __init__(self) -> None:
+        super().__init__(
+            "Custom connector code is not permitted in this environment. "
+            "If you need to run custom code, please ask your administrator to set the `AIRBYTE_ENABLE_UNSAFE_CODE` "
+            "environment variable to 'true' in your Airbyte environment. "
+            "If you see this message in Airbyte Cloud, your workspace does not allow executing "
+            "custom connector code."
+        )
+
+
+def _hash_text(input_text: str, hash_type: str = "md5") -> str:
+    """Return the hash of the input text using the specified hash type."""
+    if not input_text:
+        raise ValueError("Hash input text cannot be empty.")
+
+    hash_object = CHECKSUM_FUNCTIONS[hash_type]()
+    hash_object.update(input_text.encode())
+    return hash_object.hexdigest()
+
+
+def custom_code_execution_permitted() -> bool:
+    """Return `True` if custom code execution is permitted, otherwise `False`.
+
+    Custom code execution is permitted if the `AIRBYTE_ENABLE_UNSAFE_CODE` environment variable is set to 'true'.
+    """
+    return os.environ.get(ENV_VAR_ALLOW_CUSTOM_CODE, "").lower() == "true"
+
+
+def validate_python_code(
+    code_text: str,
+    checksums: dict[str, str] | None,
+) -> None:
+    """Validate the provided Python code text against the provided checksums.
+
+    Currently we fail if no checksums are provided, although this may change in the future.
+    """
+    if not code_text:
+        # No code provided, nothing to validate.
+        return
+
+    if not checksums:
+        raise ValueError(f"A checksum is required to validate the code. Received: {checksums}")
+
+    for checksum_type, checksum in checksums.items():
+        if checksum_type not in CHECKSUM_FUNCTIONS:
+            raise ValueError(
+                f"Unsupported checksum type: {checksum_type}. Supported checksum types are: {CHECKSUM_FUNCTIONS.keys()}"
+            )
+
+        calculated_checksum = _hash_text(code_text, checksum_type)
+        if calculated_checksum != checksum:
+            raise AirbyteCodeTamperedError(
+                f"{checksum_type} checksum does not match."
+                + str(
+                    {
+                        "expected_checksum": checksum,
+                        "actual_checksum": calculated_checksum,
+                        "code_text": code_text,
+                    }
+                ),
+            )
+
+
+def get_registered_components_module(
+    config: Mapping[str, Any] | None,
+) -> ModuleType | None:
+    """Get a components module object based on the provided config.
+
+    If custom python components is provided, this will be loaded. Otherwise, we will
+    attempt to load from the `components` module already imported/registered in sys.modules.
+
+    If custom `components.py` text is provided in config, it will be registered with sys.modules
+    so that it can be later imported by manifest declarations which reference the provided classes.
+
+    Returns `None` if no components is provided and the `components` module is not found.
+    """
+    if config and config.get(INJECTED_COMPONENTS_PY, None):
+        if not custom_code_execution_permitted():
+            raise AirbyteCustomCodeNotPermittedError
+
+        # Create a new module object and execute the provided Python code text within it
+        python_text: str = config[INJECTED_COMPONENTS_PY]
+        return register_components_module_from_string(
+            components_py_text=python_text,
+            checksums=config.get(INJECTED_COMPONENTS_PY_CHECKSUMS, None),
+        )
+
+    # Check for `components` or `source_declarative_manifest.components`.
+    if SDM_COMPONENTS_MODULE_NAME in sys.modules:
+        return cast(ModuleType, sys.modules.get(SDM_COMPONENTS_MODULE_NAME))
+
+    if COMPONENTS_MODULE_NAME in sys.modules:
+        return cast(ModuleType, sys.modules.get(COMPONENTS_MODULE_NAME))
+
+    # Could not find module 'components' in `sys.modules`
+    # and INJECTED_COMPONENTS_PY was not provided in config.
+    return None
+
+
+def register_components_module_from_string(
+    components_py_text: str,
+    checksums: dict[str, Any] | None,
+) -> ModuleType:
+    """Load and return the components module from a provided string containing the python code."""
+    # First validate the code
+    validate_python_code(
+        code_text=components_py_text,
+        checksums=checksums,
+    )
+
+    # Create a new module object
+    components_module = ModuleType(name=COMPONENTS_MODULE_NAME)
+
+    # Execute the module text in the module's namespace
+    exec(components_py_text, components_module.__dict__)
+
+    # Register the module in `sys.modules`` so it can be imported as
+    # `source_declarative_manifest.components` and/or `components`.
+    sys.modules[SDM_COMPONENTS_MODULE_NAME] = components_module
+    sys.modules[COMPONENTS_MODULE_NAME] = components_module
+
+    # Now you can import and use the module
+    return components_module

airbyte_cdk/sources/declarative/parsers/custom_exceptions.py

@@ -0,0 +1,21 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+
+class CircularReferenceException(Exception):
+    """
+    Raised when a circular reference is detected in a manifest.
+    """
+
+    def __init__(self, reference: str) -> None:
+        super().__init__(f"Circular reference found: {reference}")
+
+
+class UndefinedReferenceException(Exception):
+    """
+    Raised when refering to an undefined reference.
+    """
+
+    def __init__(self, path: str, reference: str) -> None:
+        super().__init__(f"Undefined reference {reference} from {path}")

airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py

@@ -0,0 +1,172 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+import copy
+import typing
+from typing import Any, Mapping
+
+PARAMETERS_STR = "$parameters"
+
+
+DEFAULT_MODEL_TYPES: Mapping[str, str] = {
+    # CompositeErrorHandler
+    "CompositeErrorHandler.error_handlers": "DefaultErrorHandler",
+    # CursorPagination
+    "CursorPagination.decoder": "JsonDecoder",
+    # DatetimeBasedCursor
+    "DatetimeBasedCursor.end_datetime": "MinMaxDatetime",
+    "DatetimeBasedCursor.end_time_option": "RequestOption",
+    "DatetimeBasedCursor.start_datetime": "MinMaxDatetime",
+    "DatetimeBasedCursor.start_time_option": "RequestOption",
+    # CustomIncrementalSync
+    "CustomIncrementalSync.end_datetime": "MinMaxDatetime",
+    "CustomIncrementalSync.end_time_option": "RequestOption",
+    "CustomIncrementalSync.start_datetime": "MinMaxDatetime",
+    "CustomIncrementalSync.start_time_option": "RequestOption",
+    # DeclarativeSource
+    "DeclarativeSource.check": "CheckStream",
+    "DeclarativeSource.spec": "Spec",
+    "DeclarativeSource.streams": "DeclarativeStream",
+    # DeclarativeStream
+    "DeclarativeStream.retriever": "SimpleRetriever",
+    "DeclarativeStream.schema_loader": "JsonFileSchemaLoader",
+    # DynamicDeclarativeStream
+    "DynamicDeclarativeStream.stream_template": "DeclarativeStream",
+    "DynamicDeclarativeStream.components_resolver": "ConfigComponentResolver",
+    # HttpComponentsResolver
+    "HttpComponentsResolver.retriever": "SimpleRetriever",
+    "HttpComponentsResolver.components_mapping": "ComponentMappingDefinition",
+    # ConfigComponentResolver
+    "ConfigComponentsResolver.stream_config": "StreamConfig",
+    "ConfigComponentsResolver.components_mapping": "ComponentMappingDefinition",
+    # DefaultErrorHandler
+    "DefaultErrorHandler.response_filters": "HttpResponseFilter",
+    # DefaultPaginator
+    "DefaultPaginator.decoder": "JsonDecoder",
+    "DefaultPaginator.page_size_option": "RequestOption",
+    # DpathExtractor
+    "DpathExtractor.decoder": "JsonDecoder",
+    # HttpRequester
+    "HttpRequester.error_handler": "DefaultErrorHandler",
+    # ListPartitionRouter
+    "ListPartitionRouter.request_option": "RequestOption",
+    # ParentStreamConfig
+    "ParentStreamConfig.request_option": "RequestOption",
+    "ParentStreamConfig.stream": "DeclarativeStream",
+    # RecordSelector
+    "RecordSelector.extractor": "DpathExtractor",
+    "RecordSelector.record_filter": "RecordFilter",
+    # SimpleRetriever
+    "SimpleRetriever.paginator": "NoPagination",
+    "SimpleRetriever.record_selector": "RecordSelector",
+    "SimpleRetriever.requester": "HttpRequester",
+    # SubstreamPartitionRouter
+    "SubstreamPartitionRouter.parent_stream_configs": "ParentStreamConfig",
+    # AddFields
+    "AddFields.fields": "AddedFieldDefinition",
+    # CustomPartitionRouter
+    "CustomPartitionRouter.parent_stream_configs": "ParentStreamConfig",
+    # DynamicSchemaLoader
+    "DynamicSchemaLoader.retriever": "SimpleRetriever",
+    # SchemaTypeIdentifier
+    "SchemaTypeIdentifier.types_map": "TypesMap",
+}
+
+# We retain a separate registry for custom components to automatically insert the type if it is missing. This is intended to
+# be a short term fix because once we have migrated, then type and class_name should be requirements for all custom components.
+CUSTOM_COMPONENTS_MAPPING: Mapping[str, str] = {
+    "CompositeErrorHandler.backoff_strategies": "CustomBackoffStrategy",
+    "DeclarativeStream.retriever": "CustomRetriever",
+    "DeclarativeStream.transformations": "CustomTransformation",
+    "DefaultErrorHandler.backoff_strategies": "CustomBackoffStrategy",
+    "DefaultPaginator.pagination_strategy": "CustomPaginationStrategy",
+    "HttpRequester.authenticator": "CustomAuthenticator",
+    "HttpRequester.error_handler": "CustomErrorHandler",
+    "RecordSelector.extractor": "CustomRecordExtractor",
+    "SimpleRetriever.partition_router": "CustomPartitionRouter",
+}
+
+
+class ManifestComponentTransformer:
+    def propagate_types_and_parameters(
+        self,
+        parent_field_identifier: str,
+        declarative_component: Mapping[str, Any],
+        parent_parameters: Mapping[str, Any],
+    ) -> Mapping[str, Any]:
+        """
+        Recursively transforms the specified declarative component and subcomponents to propagate parameters and insert the
+        default component type if it was not already present. The resulting transformed components are a deep copy of the input
+        components, not an in-place transformation.
+
+        :param declarative_component: The current component that is having type and parameters added
+        :param parent_field_identifier: The name of the field of the current component coming from the parent component
+        :param parent_parameters: The parameters set on parent components defined before the current component
+        :return: A deep copy of the transformed component with types and parameters persisted to it
+        """
+        propagated_component = dict(copy.deepcopy(declarative_component))
+        if "type" not in propagated_component:
+            # If the component has class_name we assume that this is a reference to a custom component. This is a slight change to
+            # existing behavior because we originally allowed for either class or type to be specified. After the pydantic migration,
+            # class_name will only be a valid field on custom components and this change reflects that. I checked, and we currently
+            # have no low-code connectors that use class_name except for custom components.
+            if "class_name" in propagated_component:
+                found_type = CUSTOM_COMPONENTS_MAPPING.get(parent_field_identifier)
+            else:
+                found_type = DEFAULT_MODEL_TYPES.get(parent_field_identifier)
+            if found_type:
+                propagated_component["type"] = found_type
+
+        # When there is no resolved type, we're not processing a component (likely a regular object) and don't need to propagate parameters
+        # When the type refers to a json schema, we're not processing a component as well. This check is currently imperfect as there could
+        # be json_schema are not objects but we believe this is not likely in our case because:
+        # * records are Mapping so objects hence SchemaLoader root should be an object
+        # * connection_specification is a Mapping
+        if "type" not in propagated_component or self._is_json_schema_object(propagated_component):
+            return propagated_component
+
+        # Combines parameters defined at the current level with parameters from parent components. Parameters at the current
+        # level take precedence
+        current_parameters = dict(copy.deepcopy(parent_parameters))
+        component_parameters = propagated_component.pop(PARAMETERS_STR, {})
+        current_parameters = {**current_parameters, **component_parameters}
+
+        # Parameters should be applied to the current component fields with the existing field taking precedence over parameters if
+        # both exist
+        for parameter_key, parameter_value in current_parameters.items():
+            propagated_component[parameter_key] = (
+                propagated_component.get(parameter_key) or parameter_value
+            )
+
+        for field_name, field_value in propagated_component.items():
+            if isinstance(field_value, dict):
+                # We exclude propagating a parameter that matches the current field name because that would result in an infinite cycle
+                excluded_parameter = current_parameters.pop(field_name, None)
+                parent_type_field_identifier = f"{propagated_component.get('type')}.{field_name}"
+                propagated_component[field_name] = self.propagate_types_and_parameters(
+                    parent_type_field_identifier, field_value, current_parameters
+                )
+                if excluded_parameter:
+                    current_parameters[field_name] = excluded_parameter
+            elif isinstance(field_value, typing.List):
+                # We exclude propagating a parameter that matches the current field name because that would result in an infinite cycle
+                excluded_parameter = current_parameters.pop(field_name, None)
+                for i, element in enumerate(field_value):
+                    if isinstance(element, dict):
+                        parent_type_field_identifier = (
+                            f"{propagated_component.get('type')}.{field_name}"
+                        )
+                        field_value[i] = self.propagate_types_and_parameters(
+                            parent_type_field_identifier, element, current_parameters
+                        )
+                if excluded_parameter:
+                    current_parameters[field_name] = excluded_parameter
+
+        if current_parameters:
+            propagated_component[PARAMETERS_STR] = current_parameters
+        return propagated_component
+
+    @staticmethod
+    def _is_json_schema_object(propagated_component: Mapping[str, Any]) -> bool:
+        return propagated_component.get("type") == "object"

airbyte_cdk/sources/declarative/parsers/manifest_reference_resolver.py

@@ -0,0 +1,213 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+import re
+from typing import Any, Mapping, Set, Tuple, Union
+
+from airbyte_cdk.sources.declarative.parsers.custom_exceptions import (
+    CircularReferenceException,
+    UndefinedReferenceException,
+)
+
+REF_TAG = "$ref"
+
+
+class ManifestReferenceResolver:
+    """
+    An incoming manifest can contain references to values previously defined.
+    This parser will dereference these values to produce a complete ConnectionDefinition.
+
+    References can be defined using a #/<arg> string.
+    ```
+    key: 1234
+    reference: "#/key"
+    ```
+    will produce the following definition:
+    ```
+    key: 1234
+    reference: 1234
+    ```
+    This also works with objects:
+    ```
+    key_value_pairs:
+      k1: v1
+      k2: v2
+    same_key_value_pairs: "#/key_value_pairs"
+    ```
+    will produce the following definition:
+    ```
+    key_value_pairs:
+      k1: v1
+      k2: v2
+    same_key_value_pairs:
+      k1: v1
+      k2: v2
+    ```
+
+    The $ref keyword can be used to refer to an object and enhance it with addition key-value pairs
+    ```
+    key_value_pairs:
+      k1: v1
+      k2: v2
+    same_key_value_pairs:
+      $ref: "#/key_value_pairs"
+      k3: v3
+    ```
+    will produce the following definition:
+    ```
+    key_value_pairs:
+      k1: v1
+      k2: v2
+    same_key_value_pairs:
+      k1: v1
+      k2: v2
+      k3: v3
+    ```
+
+    References can also point to nested values.
+    Nested references are ambiguous because one could define a key containing with `.`
+    in this example, we want to refer to the limit key in the dict object:
+    ```
+    dict:
+        limit: 50
+    limit_ref: "#/dict/limit"
+    ```
+    will produce the following definition:
+    ```
+    dict
+        limit: 50
+    limit-ref: 50
+    ```
+
+    whereas here we want to access the `nested/path` value.
+    ```
+    nested:
+        path: "first one"
+    nested/path: "uh oh"
+    value: "#/nested/path
+    ```
+    will produce the following definition:
+    ```
+    nested:
+        path: "first one"
+    nested/path: "uh oh"
+    value: "uh oh"
+    ```
+
+    to resolve the ambiguity, we try looking for the reference key at the top level, and then traverse the structs downward
+    until we find a key with the given path, or until there is nothing to traverse.
+    """
+
+    def preprocess_manifest(self, manifest: Mapping[str, Any]) -> Mapping[str, Any]:
+        """
+        :param manifest: incoming manifest that could have references to previously defined components
+        :return:
+        """
+        return self._evaluate_node(manifest, manifest, set())  # type: ignore[no-any-return]
+
+    def _evaluate_node(self, node: Any, manifest: Mapping[str, Any], visited: Set[Any]) -> Any:
+        if isinstance(node, dict):
+            evaluated_dict = {
+                k: self._evaluate_node(v, manifest, visited)
+                for k, v in node.items()
+                if not self._is_ref_key(k)
+            }
+            if REF_TAG in node:
+                # The node includes a $ref key, so we splat the referenced value(s) into the evaluated dict
+                evaluated_ref = self._evaluate_node(node[REF_TAG], manifest, visited)
+                if not isinstance(evaluated_ref, dict):
+                    return evaluated_ref
+                else:
+                    # The values defined on the component take precedence over the reference values
+                    return evaluated_ref | evaluated_dict
+            else:
+                return evaluated_dict
+        elif isinstance(node, list):
+            return [self._evaluate_node(v, manifest, visited) for v in node]
+        elif self._is_ref(node):
+            if node in visited:
+                raise CircularReferenceException(node)
+            visited.add(node)
+            ret = self._evaluate_node(self._lookup_ref_value(node, manifest), manifest, visited)
+            visited.remove(node)
+            return ret
+        else:
+            return node
+
+    def _lookup_ref_value(self, ref: str, manifest: Mapping[str, Any]) -> Any:
+        ref_match = re.match(r"#/(.*)", ref)
+        if not ref_match:
+            raise ValueError(f"Invalid reference format {ref}")
+        try:
+            path = ref_match.groups()[0]
+            return self._read_ref_value(path, manifest)
+        except (AttributeError, KeyError, IndexError):
+            raise UndefinedReferenceException(path, ref)
+
+    @staticmethod
+    def _is_ref(node: Any) -> bool:
+        return isinstance(node, str) and node.startswith("#/")
+
+    @staticmethod
+    def _is_ref_key(key: str) -> bool:
+        return bool(key == REF_TAG)
+
+    @staticmethod
+    def _read_ref_value(ref: str, manifest_node: Mapping[str, Any]) -> Any:
+        """
+        Read the value at the referenced location of the manifest.
+
+        References are ambiguous because one could define a key containing `/`
+        In this example, we want to refer to the `limit` key in the `dict` object:
+            dict:
+                limit: 50
+            limit_ref: "#/dict/limit"
+
+        Whereas here we want to access the `nested/path` value.
+          nested:
+            path: "first one"
+          nested/path: "uh oh"
+          value: "#/nested/path"
+
+        To resolve the ambiguity, we try looking for the reference key at the top level, and then traverse the structs downward
+        until we find a key with the given path, or until there is nothing to traverse.
+
+        Consider the path foo/bar/baz. To resolve the ambiguity, we first try 'foo/bar/baz' in its entirety as a top-level key. If this
+        fails, we try 'foo' as the top-level key, and if this succeeds, pass 'bar/baz' on as the key to be tried at the next level.
+        """
+        while ref:
+            try:
+                return manifest_node[ref]
+            except (KeyError, TypeError):
+                head, ref = _parse_path(ref)
+                manifest_node = manifest_node[head]  # type: ignore # Couldn't figure out how to fix this since manifest_node can get reassigned into other types like lists
+        return manifest_node
+
+
+def _parse_path(ref: str) -> Tuple[Union[str, int], str]:
+    """
+    Return the next path component, together with the rest of the path.
+
+    A path component may be a string key, or an int index.
+
+    >>> _parse_path("foo/bar")
+    "foo", "bar"
+    >>> _parse_path("foo/7/8/bar")
+    "foo", "7/8/bar"
+    >>> _parse_path("7/8/bar")
+    7, "8/bar"
+    >>> _parse_path("8/bar")
+    8, "bar"
+    >>> _parse_path("8foo/bar")
+    "8foo", "bar"
+    """
+    match = re.match(r"([^/]*)/?(.*)", ref)
+    if match:
+        first, rest = match.groups()
+        try:
+            return int(first), rest
+        except ValueError:
+            return first, rest
+    else:
+        raise ValueError(f"Invalid path {ref} specified")