airbyte-cdk 0.0.0.dev0__py3-none-any.whl

airbyte_cdk/__init__.py

@@ -0,0 +1,358 @@
+# Copyright (c) 2021 Airbyte, Inc., all rights reserved.
+"""
+# Welcome to the Airbyte Python CDK!
+
+The Airbyte Python CDK is a Python library that provides a set of tools to help you build
+connectors for the Airbyte platform.
+
+## Building Source Connectors
+
+To build a source connector, you will want to refer to
+the following classes and modules:
+
+- `airbyte_cdk.sources`
+- `airbyte_cdk.sources.concurrent_source`
+- `airbyte_cdk.sources.config`
+- `airbyte_cdk.sources.file_based`
+- `airbyte_cdk.sources.streams`
+
+## Building Destination Connectors
+
+To build a destination connector, you will want to refer to
+the following classes and modules:
+
+- `airbyte_cdk.destinations`
+- `airbyte_cdk.destinations.Destination`
+- `airbyte_cdk.destinations.vector_db_based`
+
+## Working with Airbyte Protocol Models
+
+The Airbyte CDK provides a set of classes that help you work with the Airbyte protocol models:
+
+- `airbyte_cdk.models.airbyte_protocol`
+- `airbyte_cdk.models.airbyte_protocol_serializers`
+
+---
+
+API Reference
+
+---
+
+"""
+
+# Warning: The below imports are not stable and will cause circular
+# dependencies if auto-sorted with isort. Please keep them in the same order.
+# TODO: Submodules should import from lower-level modules, rather than importing from here.
+# Imports should also be placed in `if TYPE_CHECKING` blocks if they are only used as type
+# hints - again, to avoid circular dependencies.
+# Once those issues are resolved, the below can be sorted with isort.
+import dunamai as _dunamai
+
+from .config_observation import (
+    create_connector_config_control_message,
+    emit_configuration_as_airbyte_control_message,
+)
+from .connector import BaseConnector, Connector
+from .destinations import Destination
+from .entrypoint import AirbyteEntrypoint, launch
+from .logger import AirbyteLogFormatter, init_logger
+from .models import (
+    AdvancedAuth,
+    AirbyteConnectionStatus,
+    AirbyteLogMessage,
+    AirbyteMessage,
+    AirbyteRecordMessage,
+    AirbyteStream,
+    ConfiguredAirbyteCatalog,
+    ConfiguredAirbyteStream,
+    ConnectorSpecification,
+    DestinationSyncMode,
+    FailureType,
+    Level,
+    OAuthConfigSpecification,
+    OrchestratorType,
+    Status,
+    SyncMode,
+    Type,
+)
+from .sources import AbstractSource, Source
+from .sources.concurrent_source.concurrent_source import ConcurrentSource
+from .sources.concurrent_source.concurrent_source_adapter import ConcurrentSourceAdapter
+from .sources.config import BaseConfig
+from .sources.connector_state_manager import ConnectorStateManager
+from .sources.declarative.auth import DeclarativeOauth2Authenticator
+from .sources.declarative.auth.declarative_authenticator import DeclarativeAuthenticator, NoAuth
+from .sources.declarative.auth.oauth import DeclarativeSingleUseRefreshTokenOauth2Authenticator
+from .sources.declarative.auth.token import (
+    ApiKeyAuthenticator,
+    BasicHttpAuthenticator,
+    BearerAuthenticator,
+)
+from .sources.declarative.datetime.min_max_datetime import MinMaxDatetime
+from .sources.declarative.declarative_stream import DeclarativeStream
+from .sources.declarative.decoders import Decoder, JsonDecoder
+from .sources.declarative.exceptions import ReadException
+from .sources.declarative.extractors import DpathExtractor, RecordSelector
+from .sources.declarative.extractors.record_extractor import RecordExtractor
+from .sources.declarative.extractors.record_filter import RecordFilter
+from .sources.declarative.incremental import DatetimeBasedCursor
+from .sources.declarative.interpolation import InterpolatedBoolean, InterpolatedString
+from .sources.declarative.manifest_declarative_source import ManifestDeclarativeSource
+from .sources.declarative.migrations.legacy_to_per_partition_state_migration import (
+    LegacyToPerPartitionStateMigration,
+)
+from .sources.declarative.partition_routers import (
+    CartesianProductStreamSlicer,
+    SinglePartitionRouter,
+    SubstreamPartitionRouter,
+)
+from .sources.declarative.partition_routers.substream_partition_router import ParentStreamConfig
+from .sources.declarative.requesters import HttpRequester, Requester
+from .sources.declarative.requesters.error_handlers import BackoffStrategy
+from .sources.declarative.requesters.paginators import DefaultPaginator, PaginationStrategy
+from .sources.declarative.requesters.paginators.strategies import (
+    CursorPaginationStrategy,
+    OffsetIncrement,
+    PageIncrement,
+    StopConditionPaginationStrategyDecorator,
+)
+from .sources.declarative.requesters.request_option import RequestOption, RequestOptionType
+from .sources.declarative.requesters.request_options.default_request_options_provider import (
+    DefaultRequestOptionsProvider,
+)
+from .sources.declarative.requesters.request_options.interpolated_request_input_provider import (
+    InterpolatedRequestInputProvider,
+)
+from .sources.declarative.requesters.requester import HttpMethod
+from .sources.declarative.retrievers import SimpleRetriever
+from .sources.declarative.schema import JsonFileSchemaLoader
+from .sources.declarative.transformations.add_fields import AddedFieldDefinition, AddFields
+from .sources.declarative.transformations.transformation import RecordTransformation
+from .sources.declarative.types import FieldPointer
+from .sources.declarative.yaml_declarative_source import YamlDeclarativeSource
+from .sources.message import InMemoryMessageRepository, MessageRepository
+from .sources.source import TState
+from .sources.streams.availability_strategy import AvailabilityStrategy
+from .sources.streams.call_rate import (
+    AbstractAPIBudget,
+    CachedLimiterSession,
+    HttpAPIBudget,
+    HttpRequestMatcher,
+    LimiterSession,
+    MovingWindowCallRatePolicy,
+    Rate,
+)
+from .sources.streams.checkpoint import Cursor as LegacyCursor
+from .sources.streams.checkpoint import ResumableFullRefreshCursor
+from .sources.streams.concurrent.adapters import StreamFacade
+from .sources.streams.concurrent.cursor import (
+    ConcurrentCursor,
+    Cursor,
+    CursorField,
+    FinalStateCursor,
+)
+from .sources.streams.concurrent.state_converters.datetime_stream_state_converter import (
+    EpochValueConcurrentStreamStateConverter,
+    IsoMillisConcurrentStreamStateConverter,
+)
+from .sources.streams.core import IncrementalMixin, Stream, package_name_from_class
+from .sources.streams.http import HttpStream, HttpSubStream
+from .sources.streams.http.availability_strategy import HttpAvailabilityStrategy
+from .sources.streams.http.exceptions import (
+    BaseBackoffException,
+    DefaultBackoffException,
+    UserDefinedBackoffException,
+)
+from .sources.streams.http.rate_limiting import default_backoff_handler
+from .sources.streams.http.requests_native_auth import (
+    Oauth2Authenticator,
+    SingleUseRefreshTokenOauth2Authenticator,
+    TokenAuthenticator,
+)
+from .sources.streams.http.requests_native_auth.abstract_token import AbstractHeaderAuthenticator
+from .sources.types import Config, Record, StreamSlice
+from .sources.utils import casing
+from .sources.utils.schema_helpers import (
+    InternalConfig,
+    ResourceSchemaLoader,
+    check_config_against_spec_or_exit,
+    expand_refs,
+    split_config,
+)
+from .sources.utils.transform import TransformConfig, TypeTransformer
+from .utils import AirbyteTracedException, is_cloud_environment
+from .utils.constants import ENV_REQUEST_CACHE_PATH
+from .utils.event_timing import create_timer
+from .utils.oneof_option_config import OneOfOptionConfig
+from .utils.spec_schema_transformations import resolve_refs
+from .utils.stream_status_utils import as_airbyte_message
+
+__all__ = [
+    # Availability strategy
+    "AvailabilityStrategy",
+    "HttpAvailabilityStrategy",
+    # Checkpoint
+    "LegacyCursor",
+    "ResumableFullRefreshCursor",
+    # Concurrent
+    "ConcurrentCursor",
+    "ConcurrentSource",
+    "ConcurrentSourceAdapter",
+    "Cursor",
+    "CursorField",
+    "DEFAULT_CONCURRENCY",
+    "EpochValueConcurrentStreamStateConverter",
+    "FinalStateCursor",
+    "IsoMillisConcurrentStreamStateConverter",
+    "StreamFacade",
+    # Config observation
+    "create_connector_config_control_message",
+    "emit_configuration_as_airbyte_control_message",
+    # Connector
+    "AbstractSource",
+    "BaseConfig",
+    "BaseConnector",
+    "Connector",
+    "Destination",
+    "Source",
+    "TState",
+    # Declarative
+    "AddFields",
+    "AddedFieldDefinition",
+    "ApiKeyAuthenticator",
+    "BackoffStrategy",
+    "BasicHttpAuthenticator",
+    "BearerAuthenticator",
+    "CartesianProductStreamSlicer",
+    "CursorPaginationStrategy",
+    "DatetimeBasedCursor",
+    "DeclarativeAuthenticator",
+    "DeclarativeOauth2Authenticator",
+    "DeclarativeSingleUseRefreshTokenOauth2Authenticator",
+    "DeclarativeStream",
+    "Decoder",
+    "DefaultPaginator",
+    "DefaultRequestOptionsProvider",
+    "DpathExtractor",
+    "FieldPointer",
+    "HttpMethod",
+    "HttpRequester",
+    "InterpolatedBoolean",
+    "InterpolatedRequestInputProvider",
+    "InterpolatedString",
+    "JsonDecoder",
+    "JsonFileSchemaLoader",
+    "LegacyToPerPartitionStateMigration",
+    "ManifestDeclarativeSource",
+    "MinMaxDatetime",
+    "NoAuth",
+    "OffsetIncrement",
+    "PageIncrement",
+    "PaginationStrategy",
+    "ParentStreamConfig",
+    "ReadException",
+    "RecordExtractor",
+    "RecordFilter",
+    "RecordSelector",
+    "RecordTransformation",
+    "RequestOption",
+    "RequestOptionType",
+    "Requester",
+    "ResponseStatus",
+    "SimpleRetriever",
+    "SinglePartitionRouter",
+    "StopConditionPaginationStrategyDecorator",
+    "StreamSlice",
+    "SubstreamPartitionRouter",
+    "YamlDeclarativeSource",
+    # Entrypoint
+    "launch",
+    "AirbyteEntrypoint",
+    # HTTP
+    "AbstractAPIBudget",
+    "AbstractHeaderAuthenticator",
+    "BaseBackoffException",
+    "CachedLimiterSession",
+    "DefaultBackoffException",
+    "default_backoff_handler",
+    "HttpAPIBudget",
+    "HttpAuthenticator",
+    "HttpRequestMatcher",
+    "HttpStream",
+    "HttpSubStream",
+    "LimiterSession",
+    "MovingWindowCallRatePolicy",
+    "MultipleTokenAuthenticator",
+    "Oauth2Authenticator",
+    "Rate",
+    "SingleUseRefreshTokenOauth2Authenticator",
+    "TokenAuthenticator",
+    "UserDefinedBackoffException",
+    # Logger
+    "AirbyteLogFormatter",
+    "init_logger",
+    # Protocol classes
+    "AirbyteStream",
+    "AirbyteConnectionStatus",
+    "AirbyteMessage",
+    "ConfiguredAirbyteCatalog",
+    "Status",
+    "Type",
+    "OrchestratorType",
+    "ConfiguredAirbyteStream",
+    "DestinationSyncMode",
+    "SyncMode",
+    "FailureType",
+    "AdvancedAuth",
+    "AirbyteLogMessage",
+    "OAuthConfigSpecification",
+    "ConnectorSpecification",
+    "Level",
+    "AirbyteRecordMessage",
+    # Repository
+    "InMemoryMessageRepository",
+    "MessageRepository",
+    # State management
+    "ConnectorStateManager",
+    # Stream
+    "IncrementalMixin",
+    "Stream",
+    "StreamData",
+    "package_name_from_class",
+    # Utils
+    "AirbyteTracedException",
+    "is_cloud_environment",
+    "casing",
+    "InternalConfig",
+    "ResourceSchemaLoader",
+    "check_config_against_spec_or_exit",
+    "split_config",
+    "TransformConfig",
+    "TypeTransformer",
+    "ENV_REQUEST_CACHE_PATH",
+    "create_timer",
+    "OneOfOptionConfig",
+    "resolve_refs",
+    "as_airbyte_message",
+    # Types
+    "Config",
+    "Record",
+    "Source",
+    "StreamSlice",
+]
+
+__version__: str
+"""Version generated by poetry dynamic versioning during publish.
+
+When running in development, dunamai will calculate a new prerelease version
+from existing git release tag info.
+"""
+
+try:
+    __version__ = _dunamai.get_version(
+        "airbyte-cdk",
+        third_choice=_dunamai.Version.from_any_vcs,
+        fallback=_dunamai.Version("0.0.0+dev"),
+    ).serialize()
+except:
+    __version__ = "0.0.0+dev"

airbyte_cdk/cli/__init__.py

@@ -0,0 +1 @@
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.

airbyte_cdk/cli/source_declarative_manifest/__init__.py

@@ -0,0 +1,5 @@
+from airbyte_cdk.cli.source_declarative_manifest._run import run
+
+__all__ = [
+    "run",
+]

airbyte_cdk/cli/source_declarative_manifest/_run.py

@@ -0,0 +1,236 @@
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+"""Defines the `source-declarative-manifest` connector, which installs alongside CDK.
+
+This file was originally imported from the dedicated connector directory, under the
+`airbyte` monorepo.
+
+Usage:
+
+```
+pipx install airbyte-cdk
+source-declarative-manifest --help
+source-declarative-manifest spec
+...
+```
+"""
+
+from __future__ import annotations
+
+import json
+import pkgutil
+import sys
+import traceback
+from collections.abc import Mapping
+from pathlib import Path
+from typing import Any, cast
+
+import orjson
+
+from airbyte_cdk.entrypoint import AirbyteEntrypoint, launch
+from airbyte_cdk.models import (
+    AirbyteErrorTraceMessage,
+    AirbyteMessage,
+    AirbyteMessageSerializer,
+    AirbyteStateMessage,
+    AirbyteTraceMessage,
+    ConfiguredAirbyteCatalog,
+    ConnectorSpecificationSerializer,
+    TraceType,
+    Type,
+)
+from airbyte_cdk.sources.declarative.concurrent_declarative_source import (
+    ConcurrentDeclarativeSource,
+)
+from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource
+from airbyte_cdk.sources.source import TState
+from airbyte_cdk.utils.datetime_helpers import ab_datetime_now
+
+
+class SourceLocalYaml(YamlDeclarativeSource):
+    """
+    Declarative source defined by a yaml file in the local filesystem
+    """
+
+    def __init__(
+        self,
+        catalog: ConfiguredAirbyteCatalog | None,
+        config: Mapping[str, Any] | None,
+        state: TState,
+        **kwargs: Any,
+    ) -> None:
+        """
+        HACK!
+            Problem: YamlDeclarativeSource relies on the calling module name/path to find the yaml file.
+            Implication: If you call YamlDeclarativeSource directly it will look for the yaml file in the wrong place. (e.g. the airbyte-cdk package)
+            Solution: Subclass YamlDeclarativeSource from the same location as the manifest to load.
+
+            When can we remove this?
+                When the airbyte-cdk is updated to not rely on the calling module name/path to find the yaml file.
+                When all manifest connectors are updated to use the new airbyte-cdk.
+                When all manifest connectors are updated to use the source-declarative-manifest as the base image.
+        """
+        super().__init__(
+            catalog=catalog,
+            config=config,
+            state=state,  # type: ignore [arg-type]
+            path_to_yaml="manifest.yaml",
+        )
+
+
+def _is_local_manifest_command(args: list[str]) -> bool:
+    # Check for a local manifest.yaml file
+    return Path("/airbyte/integration_code/source_declarative_manifest/manifest.yaml").exists()
+
+
+def handle_command(args: list[str]) -> None:
+    if _is_local_manifest_command(args):
+        handle_local_manifest_command(args)
+    else:
+        handle_remote_manifest_command(args)
+
+
+def _get_local_yaml_source(args: list[str]) -> SourceLocalYaml:
+    try:
+        config, catalog, state = _parse_inputs_into_config_catalog_state(args)
+        return SourceLocalYaml(config=config, catalog=catalog, state=state)
+    except Exception as error:
+        print(
+            orjson.dumps(
+                AirbyteMessageSerializer.dump(
+                    AirbyteMessage(
+                        type=Type.TRACE,
+                        trace=AirbyteTraceMessage(
+                            type=TraceType.ERROR,
+                            emitted_at=ab_datetime_now().to_epoch_millis(),
+                            error=AirbyteErrorTraceMessage(
+                                message=f"Error starting the sync. This could be due to an invalid configuration or catalog. Please contact Support for assistance. Error: {error}",
+                                stack_trace=traceback.format_exc(),
+                            ),
+                        ),
+                    )
+                )
+            ).decode()
+        )
+        raise error
+
+
+def handle_local_manifest_command(args: list[str]) -> None:
+    source = _get_local_yaml_source(args)
+    launch(
+        source=source,
+        args=args,
+    )
+
+
+def handle_remote_manifest_command(args: list[str]) -> None:
+    """Overrides the spec command to return the generalized spec for the declarative manifest source.
+
+    This is different from a typical low-code, but built and published separately source built as a ManifestDeclarativeSource,
+    because that will have a spec method that returns the spec for that specific source. Other than spec,
+    the generalized connector behaves the same as any other, since the manifest is provided in the config.
+    """
+    if args[0] == "spec":
+        json_spec = pkgutil.get_data(
+            "airbyte_cdk.cli.source_declarative_manifest",
+            "spec.json",
+        )
+        if json_spec is None:
+            raise FileNotFoundError(
+                "Could not find `spec.json` file for source-declarative-manifest"
+            )
+
+        spec_obj = json.loads(json_spec)
+        spec = ConnectorSpecificationSerializer.load(spec_obj)
+
+        message = AirbyteMessage(type=Type.SPEC, spec=spec)
+        print(AirbyteEntrypoint.airbyte_message_to_string(message))
+    else:
+        source = create_declarative_source(args)
+        launch(
+            source=source,
+            args=args,
+        )
+
+
+def create_declarative_source(
+    args: list[str],
+) -> ConcurrentDeclarativeSource:  # type: ignore [type-arg]
+    """Creates the source with the injected config.
+
+    This essentially does what other low-code sources do at build time, but at runtime,
+    with a user-provided manifest in the config. This better reflects what happens in the
+    connector builder.
+    """
+    try:
+        config: Mapping[str, Any] | None
+        catalog: ConfiguredAirbyteCatalog | None
+        state: list[AirbyteStateMessage]
+        config, catalog, state = _parse_inputs_into_config_catalog_state(args)
+        if config is None or "__injected_declarative_manifest" not in config:
+            raise ValueError(
+                "Invalid config: `__injected_declarative_manifest` should be provided at the root "
+                f"of the config but config only has keys: {list(config.keys() if config else [])}"
+            )
+        if not isinstance(config["__injected_declarative_manifest"], dict):
+            raise ValueError(
+                "Invalid config: `__injected_declarative_manifest` should be a dictionary, "
+                f"but got type: {type(config['__injected_declarative_manifest'])}"
+            )
+
+        return ConcurrentDeclarativeSource(
+            config=config,
+            catalog=catalog,
+            state=state,
+            source_config=cast(dict[str, Any], config["__injected_declarative_manifest"]),
+        )
+    except Exception as error:
+        print(
+            orjson.dumps(
+                AirbyteMessageSerializer.dump(
+                    AirbyteMessage(
+                        type=Type.TRACE,
+                        trace=AirbyteTraceMessage(
+                            type=TraceType.ERROR,
+                            emitted_at=ab_datetime_now().to_epoch_millis(),
+                            error=AirbyteErrorTraceMessage(
+                                message=f"Error starting the sync. This could be due to an invalid configuration or catalog. Please contact Support for assistance. Error: {error}",
+                                stack_trace=traceback.format_exc(),
+                            ),
+                        ),
+                    )
+                )
+            ).decode()
+        )
+        raise error
+
+
+def _parse_inputs_into_config_catalog_state(
+    args: list[str],
+) -> tuple[
+    Mapping[str, Any] | None,
+    ConfiguredAirbyteCatalog | None,
+    list[AirbyteStateMessage],
+]:
+    parsed_args = AirbyteEntrypoint.parse_args(args)
+    config = (
+        ConcurrentDeclarativeSource.read_config(parsed_args.config)
+        if hasattr(parsed_args, "config")
+        else None
+    )
+    catalog = (
+        ConcurrentDeclarativeSource.read_catalog(parsed_args.catalog)
+        if hasattr(parsed_args, "catalog")
+        else None
+    )
+    state = (
+        ConcurrentDeclarativeSource.read_state(parsed_args.state)
+        if hasattr(parsed_args, "state")
+        else []
+    )
+
+    return config, catalog, state
+
+
+def run() -> None:
+    args: list[str] = sys.argv[1:]
+    handle_command(args)

airbyte_cdk/cli/source_declarative_manifest/spec.json

@@ -0,0 +1,17 @@
+{
+  "documentationUrl": "https://docs.airbyte.com/integrations/sources/low-code",
+  "connectionSpecification": {
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "title": "Low-code source spec",
+    "type": "object",
+    "required": ["__injected_declarative_manifest"],
+    "additionalProperties": true,
+    "properties": {
+      "__injected_declarative_manifest": {
+        "title": "Low-code manifest",
+        "type": "object",
+        "description": "The low-code manifest that defines the components of the source."
+      }
+    }
+  }
+}