airbyte-cdk 0.0.0.dev0__py3-none-any.whl

airbyte_cdk/config_observation.py

@@ -0,0 +1,104 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+from __future__ import (  # Used to evaluate type hints at runtime, a NameError: name 'ConfigObserver' is not defined is thrown otherwise
+    annotations,
+)
+
+import time
+from copy import copy
+from typing import Any, List, MutableMapping
+
+import orjson
+
+from airbyte_cdk.models import (
+    AirbyteControlConnectorConfigMessage,
+    AirbyteControlMessage,
+    AirbyteMessage,
+    AirbyteMessageSerializer,
+    OrchestratorType,
+    Type,
+)
+
+
+class ObservedDict(dict):  # type: ignore # disallow_any_generics is set to True, and dict is equivalent to dict[Any]
+    def __init__(
+        self,
+        non_observed_mapping: MutableMapping[Any, Any],
+        observer: ConfigObserver,
+        update_on_unchanged_value: bool = True,
+    ) -> None:
+        non_observed_mapping = copy(non_observed_mapping)
+        self.observer = observer
+        self.update_on_unchanged_value = update_on_unchanged_value
+        for item, value in non_observed_mapping.items():
+            # Observe nested dicts
+            if isinstance(value, MutableMapping):
+                non_observed_mapping[item] = ObservedDict(value, observer)
+
+            # Observe nested list of dicts
+            if isinstance(value, List):
+                for i, sub_value in enumerate(value):
+                    if isinstance(sub_value, MutableMapping):
+                        value[i] = ObservedDict(sub_value, observer)
+        super().__init__(non_observed_mapping)
+
+    def __setitem__(self, item: Any, value: Any) -> None:
+        """Override dict.__setitem__ by:
+        1. Observing the new value if it is a dict
+        2. Call observer update if the new value is different from the previous one
+        """
+        previous_value = self.get(item)
+        if isinstance(value, MutableMapping):
+            value = ObservedDict(value, self.observer)
+        if isinstance(value, List):
+            for i, sub_value in enumerate(value):
+                if isinstance(sub_value, MutableMapping):
+                    value[i] = ObservedDict(sub_value, self.observer)
+        super(ObservedDict, self).__setitem__(item, value)
+        if self.update_on_unchanged_value or value != previous_value:
+            self.observer.update()
+
+
+class ConfigObserver:
+    """This class is made to track mutations on ObservedDict config.
+    When update is called a CONNECTOR_CONFIG control message is emitted on stdout.
+    """
+
+    def set_config(self, config: ObservedDict) -> None:
+        self.config = config
+
+    def update(self) -> None:
+        emit_configuration_as_airbyte_control_message(self.config)
+
+
+def observe_connector_config(
+    non_observed_connector_config: MutableMapping[str, Any],
+) -> ObservedDict:
+    if isinstance(non_observed_connector_config, ObservedDict):
+        raise ValueError("This connector configuration is already observed")
+    connector_config_observer = ConfigObserver()
+    observed_connector_config = ObservedDict(
+        non_observed_connector_config, connector_config_observer
+    )
+    connector_config_observer.set_config(observed_connector_config)
+    return observed_connector_config
+
+
+def emit_configuration_as_airbyte_control_message(config: MutableMapping[str, Any]) -> None:
+    """
+    WARNING: deprecated - emit_configuration_as_airbyte_control_message is being deprecated in favor of the MessageRepository mechanism.
+    See the airbyte_cdk.sources.message package
+    """
+    airbyte_message = create_connector_config_control_message(config)
+    print(orjson.dumps(AirbyteMessageSerializer.dump(airbyte_message)).decode())
+
+
+def create_connector_config_control_message(config: MutableMapping[str, Any]) -> AirbyteMessage:
+    control_message = AirbyteControlMessage(
+        type=OrchestratorType.CONNECTOR_CONFIG,
+        emitted_at=time.time() * 1000,
+        connectorConfig=AirbyteControlConnectorConfigMessage(config=config),
+    )
+    return AirbyteMessage(type=Type.CONTROL, control=control_message)

airbyte_cdk/connector.py

@@ -0,0 +1,123 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+
+import json
+import logging
+import os
+import pkgutil
+from abc import ABC, abstractmethod
+from typing import Any, Generic, Mapping, Optional, Protocol, TypeVar
+
+import yaml
+
+from airbyte_cdk.models import (
+    AirbyteConnectionStatus,
+    ConnectorSpecification,
+    ConnectorSpecificationSerializer,
+)
+
+
+def load_optional_package_file(package: str, filename: str) -> Optional[bytes]:
+    """Gets a resource from a package, returning None if it does not exist"""
+    try:
+        return pkgutil.get_data(package, filename)
+    except FileNotFoundError:
+        return None
+
+
+TConfig = TypeVar("TConfig", bound=Mapping[str, Any])
+
+
+class BaseConnector(ABC, Generic[TConfig]):
+    # configure whether the `check_config_against_spec_or_exit()` needs to be called
+    check_config_against_spec: bool = True
+
+    @abstractmethod
+    def configure(self, config: Mapping[str, Any], temp_dir: str) -> TConfig:
+        """
+        Persist config in temporary directory to run the Source job
+        """
+
+    @staticmethod
+    def read_config(config_path: str) -> Mapping[str, Any]:
+        config = BaseConnector._read_json_file(config_path)
+        if isinstance(config, Mapping):
+            return config
+        else:
+            raise ValueError(
+                f"The content of {config_path} is not an object and therefore is not a valid config. Please ensure the file represent a config."
+            )
+
+    @staticmethod
+    def _read_json_file(file_path: str) -> Any:
+        with open(file_path, "r") as file:
+            contents = file.read()
+
+        try:
+            return json.loads(contents)
+        except json.JSONDecodeError as error:
+            raise ValueError(
+                f"Could not read json file {file_path}: {error}. Please ensure that it is a valid JSON."
+            )
+
+    @staticmethod
+    def write_config(config: TConfig, config_path: str) -> None:
+        with open(config_path, "w") as fh:
+            fh.write(json.dumps(config))
+
+    def spec(self, logger: logging.Logger) -> ConnectorSpecification:
+        """
+        Returns the spec for this integration. The spec is a JSON-Schema object describing the required configurations (e.g: username and password)
+        required to run this integration. By default, this will be loaded from a "spec.yaml" or a "spec.json" in the package root.
+        """
+
+        package = self.__class__.__module__.split(".")[0]
+
+        yaml_spec = load_optional_package_file(package, "spec.yaml")
+        json_spec = load_optional_package_file(package, "spec.json")
+
+        if yaml_spec and json_spec:
+            raise RuntimeError(
+                "Found multiple spec files in the package. Only one of spec.yaml or spec.json should be provided."
+            )
+
+        if yaml_spec:
+            spec_obj = yaml.load(yaml_spec, Loader=yaml.SafeLoader)
+        elif json_spec:
+            try:
+                spec_obj = json.loads(json_spec)
+            except json.JSONDecodeError as error:
+                raise ValueError(
+                    f"Could not read json spec file: {error}. Please ensure that it is a valid JSON."
+                )
+        else:
+            raise FileNotFoundError("Unable to find spec.yaml or spec.json in the package.")
+
+        return ConnectorSpecificationSerializer.load(spec_obj)
+
+    @abstractmethod
+    def check(self, logger: logging.Logger, config: TConfig) -> AirbyteConnectionStatus:
+        """
+        Tests if the input configuration can be used to successfully connect to the integration e.g: if a provided Stripe API token can be used to connect
+        to the Stripe API.
+        """
+
+
+class _WriteConfigProtocol(Protocol):
+    @staticmethod
+    def write_config(config: Mapping[str, Any], config_path: str) -> None: ...
+
+
+class DefaultConnectorMixin:
+    # can be overridden to change an input config
+    def configure(
+        self: _WriteConfigProtocol, config: Mapping[str, Any], temp_dir: str
+    ) -> Mapping[str, Any]:
+        config_path = os.path.join(temp_dir, "config.json")
+        self.write_config(config, config_path)
+        return config
+
+
+class Connector(DefaultConnectorMixin, BaseConnector[Mapping[str, Any]], ABC): ...

airbyte_cdk/connector_builder/README.md

@@ -0,0 +1,53 @@
+# Connector Builder Backend
+
+This is the backend for requests from the
+[Connector Builder](https://docs.airbyte.com/connector-development/connector-builder-ui/overview/).
+
+## Local development
+
+### Locally running the Connector Builder backend
+
+```bash
+python main.py read --config path/to/config --catalog path/to/catalog
+```
+
+Note:
+
+- Requires the keys `__injected_declarative_manifest` and `__command` in its config, where
+  `__injected_declarative_manifest` is a JSON manifest and `__command` is one of the commands
+  handled by the ConnectorBuilderHandler (`stream_read` or `resolve_manifest`), i.e.
+
+```json
+{
+  "config": <normal config>,
+  "__injected_declarative_manifest": {...},
+  "__command": <"resolve_manifest" | "test_read">
+}
+```
+
+\*See
+[ConnectionSpecification](https://docs.airbyte.com/understanding-airbyte/airbyte-protocol/#actor-specification)
+for details on the `"config"` key if needed.
+
+- When the `__command` is `resolve_manifest`, the argument to `catalog` should be an empty string.
+- The config can optionally contain an object under the `__test_read_config` key which can define
+  custom test read limits with `max_records`, `max_slices`, and `max_pages_per_slice` properties.
+  All custom limits are optional; a default will be used for any limit that is not provided.
+
+### Locally running the docker image
+
+#### Build
+
+First, make sure you build the latest Docker image:
+
+```bash
+docker build -t airbyte/source-declarative-manifest:dev .
+```
+
+#### Run
+
+Then run any of the connector commands as follows:
+
+```bash
+docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-declarative-manifest:dev read --config /secrets/config.json
+```

airbyte_cdk/connector_builder/__init__.py

@@ -0,0 +1,3 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#

airbyte_cdk/connector_builder/connector_builder_handler.py

@@ -0,0 +1,121 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+
+from dataclasses import asdict, dataclass, field
+from typing import Any, List, Mapping
+
+from airbyte_cdk.connector_builder.test_reader import TestReader
+from airbyte_cdk.models import (
+    AirbyteMessage,
+    AirbyteRecordMessage,
+    AirbyteStateMessage,
+    ConfiguredAirbyteCatalog,
+    Type,
+)
+from airbyte_cdk.models import Type as MessageType
+from airbyte_cdk.sources.declarative.declarative_source import DeclarativeSource
+from airbyte_cdk.sources.declarative.manifest_declarative_source import ManifestDeclarativeSource
+from airbyte_cdk.sources.declarative.parsers.model_to_component_factory import (
+    ModelToComponentFactory,
+)
+from airbyte_cdk.utils.airbyte_secrets_utils import filter_secrets
+from airbyte_cdk.utils.datetime_helpers import ab_datetime_now
+from airbyte_cdk.utils.traced_exception import AirbyteTracedException
+
+DEFAULT_MAXIMUM_NUMBER_OF_PAGES_PER_SLICE = 5
+DEFAULT_MAXIMUM_NUMBER_OF_SLICES = 5
+DEFAULT_MAXIMUM_RECORDS = 100
+
+MAX_PAGES_PER_SLICE_KEY = "max_pages_per_slice"
+MAX_SLICES_KEY = "max_slices"
+MAX_RECORDS_KEY = "max_records"
+
+
+@dataclass
+class TestReadLimits:
+    max_records: int = field(default=DEFAULT_MAXIMUM_RECORDS)
+    max_pages_per_slice: int = field(default=DEFAULT_MAXIMUM_NUMBER_OF_PAGES_PER_SLICE)
+    max_slices: int = field(default=DEFAULT_MAXIMUM_NUMBER_OF_SLICES)
+
+
+def get_limits(config: Mapping[str, Any]) -> TestReadLimits:
+    command_config = config.get("__test_read_config", {})
+    max_pages_per_slice = (
+        command_config.get(MAX_PAGES_PER_SLICE_KEY) or DEFAULT_MAXIMUM_NUMBER_OF_PAGES_PER_SLICE
+    )
+    max_slices = command_config.get(MAX_SLICES_KEY) or DEFAULT_MAXIMUM_NUMBER_OF_SLICES
+    max_records = command_config.get(MAX_RECORDS_KEY) or DEFAULT_MAXIMUM_RECORDS
+    return TestReadLimits(max_records, max_pages_per_slice, max_slices)
+
+
+def create_source(config: Mapping[str, Any], limits: TestReadLimits) -> ManifestDeclarativeSource:
+    manifest = config["__injected_declarative_manifest"]
+    return ManifestDeclarativeSource(
+        config=config,
+        emit_connector_builder_messages=True,
+        source_config=manifest,
+        component_factory=ModelToComponentFactory(
+            emit_connector_builder_messages=True,
+            limit_pages_fetched_per_slice=limits.max_pages_per_slice,
+            limit_slices_fetched=limits.max_slices,
+            disable_retries=True,
+            disable_cache=True,
+        ),
+    )
+
+
+def read_stream(
+    source: DeclarativeSource,
+    config: Mapping[str, Any],
+    configured_catalog: ConfiguredAirbyteCatalog,
+    state: List[AirbyteStateMessage],
+    limits: TestReadLimits,
+) -> AirbyteMessage:
+    try:
+        test_read_handler = TestReader(
+            limits.max_pages_per_slice, limits.max_slices, limits.max_records
+        )
+        # The connector builder only supports a single stream
+        stream_name = configured_catalog.streams[0].stream.name
+
+        stream_read = test_read_handler.run_test_read(
+            source, config, configured_catalog, state, limits.max_records
+        )
+
+        return AirbyteMessage(
+            type=MessageType.RECORD,
+            record=AirbyteRecordMessage(
+                data=asdict(stream_read), stream=stream_name, emitted_at=_emitted_at()
+            ),
+        )
+    except Exception as exc:
+        error = AirbyteTracedException.from_exception(
+            exc,
+            message=filter_secrets(
+                f"Error reading stream with config={config} and catalog={configured_catalog}: {str(exc)}"
+            ),
+        )
+        return error.as_airbyte_message()
+
+
+def resolve_manifest(source: ManifestDeclarativeSource) -> AirbyteMessage:
+    try:
+        return AirbyteMessage(
+            type=Type.RECORD,
+            record=AirbyteRecordMessage(
+                data={"manifest": source.resolved_manifest},
+                emitted_at=_emitted_at(),
+                stream="resolve_manifest",
+            ),
+        )
+    except Exception as exc:
+        error = AirbyteTracedException.from_exception(
+            exc, message=f"Error resolving manifest: {str(exc)}"
+        )
+        return error.as_airbyte_message()
+
+
+def _emitted_at() -> int:
+    return ab_datetime_now().to_epoch_millis()

airbyte_cdk/connector_builder/main.py

@@ -0,0 +1,107 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+
+import sys
+from typing import Any, List, Mapping, Optional, Tuple
+
+import orjson
+
+from airbyte_cdk.connector import BaseConnector
+from airbyte_cdk.connector_builder.connector_builder_handler import (
+    TestReadLimits,
+    create_source,
+    get_limits,
+    read_stream,
+    resolve_manifest,
+)
+from airbyte_cdk.entrypoint import AirbyteEntrypoint
+from airbyte_cdk.models import (
+    AirbyteMessage,
+    AirbyteMessageSerializer,
+    AirbyteStateMessage,
+    ConfiguredAirbyteCatalog,
+    ConfiguredAirbyteCatalogSerializer,
+)
+from airbyte_cdk.sources.declarative.manifest_declarative_source import ManifestDeclarativeSource
+from airbyte_cdk.sources.source import Source
+from airbyte_cdk.utils.traced_exception import AirbyteTracedException
+
+
+def get_config_and_catalog_from_args(
+    args: List[str],
+) -> Tuple[str, Mapping[str, Any], Optional[ConfiguredAirbyteCatalog], Any]:
+    # TODO: Add functionality for the `debug` logger.
+    #  Currently, no one `debug` level log will be displayed during `read` a stream for a connector created through `connector-builder`.
+    parsed_args = AirbyteEntrypoint.parse_args(args)
+    config_path, catalog_path, state_path = (
+        parsed_args.config,
+        parsed_args.catalog,
+        parsed_args.state,
+    )
+    if parsed_args.command != "read":
+        raise ValueError("Only read commands are allowed for Connector Builder requests.")
+
+    config = BaseConnector.read_config(config_path)
+
+    if "__command" not in config:
+        raise ValueError(
+            f"Invalid config: `__command` should be provided at the root of the config but config only has keys {list(config.keys())}"
+        )
+
+    command = config["__command"]
+    if command == "test_read":
+        catalog = ConfiguredAirbyteCatalogSerializer.load(BaseConnector.read_config(catalog_path))
+        state = Source.read_state(state_path)
+    else:
+        catalog = None
+        state = []
+
+    if "__injected_declarative_manifest" not in config:
+        raise ValueError(
+            f"Invalid config: `__injected_declarative_manifest` should be provided at the root of the config but config only has keys {list(config.keys())}"
+        )
+
+    return command, config, catalog, state
+
+
+def handle_connector_builder_request(
+    source: ManifestDeclarativeSource,
+    command: str,
+    config: Mapping[str, Any],
+    catalog: Optional[ConfiguredAirbyteCatalog],
+    state: List[AirbyteStateMessage],
+    limits: TestReadLimits,
+) -> AirbyteMessage:
+    if command == "resolve_manifest":
+        return resolve_manifest(source)
+    elif command == "test_read":
+        assert (
+            catalog is not None
+        ), "`test_read` requires a valid `ConfiguredAirbyteCatalog`, got None."
+        return read_stream(source, config, catalog, state, limits)
+    else:
+        raise ValueError(f"Unrecognized command {command}.")
+
+
+def handle_request(args: List[str]) -> str:
+    command, config, catalog, state = get_config_and_catalog_from_args(args)
+    limits = get_limits(config)
+    source = create_source(config, limits)
+    return orjson.dumps(
+        AirbyteMessageSerializer.dump(
+            handle_connector_builder_request(source, command, config, catalog, state, limits)
+        )
+    ).decode()  # type: ignore[no-any-return] # Serializer.dump() always returns AirbyteMessage
+
+
+if __name__ == "__main__":
+    try:
+        print(handle_request(sys.argv[1:]))
+    except Exception as exc:
+        error = AirbyteTracedException.from_exception(
+            exc, message=f"Error handling request: {str(exc)}"
+        )
+        m = error.as_airbyte_message()
+        print(orjson.dumps(AirbyteMessageSerializer.dump(m)).decode())

airbyte_cdk/connector_builder/models.py

@@ -0,0 +1,73 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+from dataclasses import dataclass
+from typing import Any, Dict, List, Optional
+
+
+@dataclass
+class HttpResponse:
+    status: int
+    body: Optional[str] = None
+    headers: Optional[Dict[str, Any]] = None
+
+
+@dataclass
+class HttpRequest:
+    url: str
+    headers: Optional[Dict[str, Any]]
+    http_method: str
+    body: Optional[str] = None
+
+
+@dataclass
+class LogMessage:
+    message: str
+    level: str
+    internal_message: Optional[str] = None
+    stacktrace: Optional[str] = None
+
+
+@dataclass
+class AuxiliaryRequest:
+    title: str
+    type: str
+    description: str
+    request: HttpRequest
+    response: HttpResponse
+
+
+@dataclass
+class StreamReadPages:
+    records: List[object]
+    request: Optional[HttpRequest] = None
+    response: Optional[HttpResponse] = None
+
+
+@dataclass
+class StreamReadSlices:
+    pages: List[StreamReadPages]
+    slice_descriptor: Optional[Dict[str, Any]]
+    state: Optional[List[Dict[str, Any]]] = None
+    auxiliary_requests: Optional[List[AuxiliaryRequest]] = None
+
+
+@dataclass
+class StreamRead(object):
+    logs: List[LogMessage]
+    slices: List[StreamReadSlices]
+    test_read_limit_reached: bool
+    auxiliary_requests: List[AuxiliaryRequest]
+    inferred_schema: Optional[Dict[str, Any]]
+    inferred_datetime_formats: Optional[Dict[str, str]]
+    latest_config_update: Optional[Dict[str, Any]]
+
+
+@dataclass
+class StreamReadRequestBody:
+    manifest: Dict[str, Any]
+    stream: str
+    config: Dict[str, Any]
+    state: Optional[Dict[str, Any]]
+    record_limit: Optional[int]

airbyte_cdk/connector_builder/test_reader/__init__.py

@@ -0,0 +1,7 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+from .reader import TestReader
+
+__all__ = ("TestReader",)