airbyte-cdk 0.72.1__tar.gz → 6.13.1.dev4106__tar.gz

airbyte_cdk-6.13.1.dev4106/PKG-INFO

@@ -0,0 +1,109 @@
+Metadata-Version: 2.3
+Name: airbyte-cdk
+Version: 6.13.1.dev4106
+Summary: A framework for writing Airbyte Connectors.
+License: MIT
+Keywords: airbyte,connector-development-kit,cdk
+Author: Airbyte
+Author-email: contact@airbyte.io
+Requires-Python: >=3.10,<3.13
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Provides-Extra: file-based
+Provides-Extra: sql
+Provides-Extra: vector-db-based
+Requires-Dist: Jinja2 (>=3.1.2,<3.2.0)
+Requires-Dist: PyYAML (>=6.0.1,<7.0.0)
+Requires-Dist: Unidecode (>=1.3,<2.0)
+Requires-Dist: airbyte-protocol-models-dataclasses (>=0.14,<0.15)
+Requires-Dist: avro (>=1.11.2,<1.12.0) ; extra == "file-based"
+Requires-Dist: backoff
+Requires-Dist: cachetools
+Requires-Dist: cohere (==4.21) ; extra == "vector-db-based"
+Requires-Dist: cryptography (>=42.0.5,<44.0.0)
+Requires-Dist: dpath (>=2.1.6,<3.0.0)
+Requires-Dist: dunamai (>=1.22.0,<2.0.0)
+Requires-Dist: fastavro (>=1.8.0,<1.9.0) ; extra == "file-based"
+Requires-Dist: genson (==1.3.0)
+Requires-Dist: isodate (>=0.6.1,<0.7.0)
+Requires-Dist: jsonref (>=0.2,<0.3)
+Requires-Dist: jsonschema (>=4.17.3,<4.18.0)
+Requires-Dist: langchain (==0.1.16) ; extra == "vector-db-based"
+Requires-Dist: langchain_core (==0.1.42)
+Requires-Dist: markdown ; extra == "file-based"
+Requires-Dist: nltk (==3.9.1)
+Requires-Dist: numpy (<2)
+Requires-Dist: openai[embeddings] (==0.27.9) ; extra == "vector-db-based"
+Requires-Dist: orjson (>=3.10.7,<4.0.0)
+Requires-Dist: pandas (==2.2.2)
+Requires-Dist: pdf2image (==1.16.3) ; extra == "file-based"
+Requires-Dist: pdfminer.six (==20221105) ; extra == "file-based"
+Requires-Dist: pendulum (<3.0.0)
+Requires-Dist: psutil (==6.1.0)
+Requires-Dist: pyarrow (>=15.0.0,<15.1.0) ; extra == "file-based"
+Requires-Dist: pydantic (>=2.7,<3.0)
+Requires-Dist: pyjwt (>=2.8.0,<3.0.0)
+Requires-Dist: pyrate-limiter (>=3.1.0,<3.2.0)
+Requires-Dist: pytesseract (==0.3.10) ; extra == "file-based"
+Requires-Dist: python-calamine (==0.2.3) ; extra == "file-based"
+Requires-Dist: python-dateutil
+Requires-Dist: python-snappy (==0.7.3) ; extra == "file-based"
+Requires-Dist: python-ulid (>=3.0.0,<4.0.0)
+Requires-Dist: pytz (==2024.2)
+Requires-Dist: rapidfuzz (>=3.10.1,<4.0.0)
+Requires-Dist: requests
+Requires-Dist: requests_cache
+Requires-Dist: serpyco-rs (>=1.10.2,<2.0.0)
+Requires-Dist: sqlalchemy (>=2.0,<3.0,!=2.0.36) ; extra == "sql"
+Requires-Dist: tiktoken (==0.8.0) ; extra == "vector-db-based"
+Requires-Dist: unstructured.pytesseract (>=0.3.12) ; extra == "file-based"
+Requires-Dist: unstructured[docx,pptx] (==0.10.27) ; extra == "file-based"
+Requires-Dist: wcmatch (==10.0)
+Requires-Dist: xmltodict (>=0.13.0,<0.14.0)
+Project-URL: Documentation, https://docs.airbyte.io/
+Project-URL: Homepage, https://airbyte.com
+Project-URL: Repository, https://github.com/airbytehq/airbyte-python-cdk
+Description-Content-Type: text/markdown
+
+# Airbyte Python CDK and Low-Code CDK
+
+Airbyte Python CDK is a framework for building Airbyte API Source Connectors. It provides a set of
+classes and helpers that make it easy to build a connector against an HTTP API (REST, GraphQL, etc),
+or a generic Python source connector.
+
+## Building Connectors with the CDK
+
+If you're looking to build a connector, we highly recommend that you first
+[start with the Connector Builder](https://docs.airbyte.com/connector-development/connector-builder-ui/overview).
+It should be enough for 90% connectors out there. For more flexible and complex connectors, use the
+[low-code CDK and `SourceDeclarativeManifest`](https://docs.airbyte.com/connector-development/config-based/low-code-cdk-overview).
+
+For more information on building connectors, please see the [Connector Development](https://docs.airbyte.com/connector-development/) guide on [docs.airbyte.com](https://docs.airbyte.com).
+
+## Python CDK Overview
+
+Airbyte CDK code is within `airbyte_cdk` directory. Here's a high level overview of what's inside:
+
+- `airbyte_cdk/connector_builder`. Internal wrapper that helps the Connector Builder platform run a declarative manifest (low-code connector). You should not use this code directly. If you need to run a `SourceDeclarativeManifest`, take a look at [`source-declarative-manifest`](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-declarative-manifest) connector implementation instead.
+- `airbyte_cdk/cli/source_declarative_manifest`. This module defines the `source-declarative-manifest` (aka "SDM") connector execution logic and associated CLI.
+- `airbyte_cdk/destinations`. Basic Destination connector support! If you're building a Destination connector in Python, try that. Some of our vector DB destinations like `destination-pinecone` are using that code.
+- `airbyte_cdk/models` expose `airbyte_protocol.models` as a part of `airbyte_cdk` package.
+- `airbyte_cdk/sources/concurrent_source` is the Concurrent CDK implementation. It supports reading data from streams concurrently per slice / partition, useful for connectors with high throughput and high number of records.
+- `airbyte_cdk/sources/declarative` is the low-code CDK. It works on top of Airbyte Python CDK, but provides a declarative manifest language to define streams, operations, etc. This makes it easier to build connectors without writing Python code.
+- `airbyte_cdk/sources/file_based` is the CDK for file-based sources. Examples include S3, Azure, GCS, etc.
+
+## Contributing
+
+For instructions on how to contribute, please see our [Contributing Guide](docs/CONTRIBUTING.md).
+
+## Release Management
+
+Please see the [Release Management](docs/RELEASES.md) guide for information on how to perform releases and pre-releases.
+

airbyte_cdk-6.13.1.dev4106/README.md

@@ -0,0 +1,34 @@
+# Airbyte Python CDK and Low-Code CDK
+
+Airbyte Python CDK is a framework for building Airbyte API Source Connectors. It provides a set of
+classes and helpers that make it easy to build a connector against an HTTP API (REST, GraphQL, etc),
+or a generic Python source connector.
+
+## Building Connectors with the CDK
+
+If you're looking to build a connector, we highly recommend that you first
+[start with the Connector Builder](https://docs.airbyte.com/connector-development/connector-builder-ui/overview).
+It should be enough for 90% connectors out there. For more flexible and complex connectors, use the
+[low-code CDK and `SourceDeclarativeManifest`](https://docs.airbyte.com/connector-development/config-based/low-code-cdk-overview).
+
+For more information on building connectors, please see the [Connector Development](https://docs.airbyte.com/connector-development/) guide on [docs.airbyte.com](https://docs.airbyte.com).
+
+## Python CDK Overview
+
+Airbyte CDK code is within `airbyte_cdk` directory. Here's a high level overview of what's inside:
+
+- `airbyte_cdk/connector_builder`. Internal wrapper that helps the Connector Builder platform run a declarative manifest (low-code connector). You should not use this code directly. If you need to run a `SourceDeclarativeManifest`, take a look at [`source-declarative-manifest`](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-declarative-manifest) connector implementation instead.
+- `airbyte_cdk/cli/source_declarative_manifest`. This module defines the `source-declarative-manifest` (aka "SDM") connector execution logic and associated CLI.
+- `airbyte_cdk/destinations`. Basic Destination connector support! If you're building a Destination connector in Python, try that. Some of our vector DB destinations like `destination-pinecone` are using that code.
+- `airbyte_cdk/models` expose `airbyte_protocol.models` as a part of `airbyte_cdk` package.
+- `airbyte_cdk/sources/concurrent_source` is the Concurrent CDK implementation. It supports reading data from streams concurrently per slice / partition, useful for connectors with high throughput and high number of records.
+- `airbyte_cdk/sources/declarative` is the low-code CDK. It works on top of Airbyte Python CDK, but provides a declarative manifest language to define streams, operations, etc. This makes it easier to build connectors without writing Python code.
+- `airbyte_cdk/sources/file_based` is the CDK for file-based sources. Examples include S3, Azure, GCS, etc.
+
+## Contributing
+
+For instructions on how to contribute, please see our [Contributing Guide](docs/CONTRIBUTING.md).
+
+## Release Management
+
+Please see the [Release Management](docs/RELEASES.md) guide for information on how to perform releases and pre-releases.

airbyte_cdk-6.13.1.dev4106/airbyte_cdk/__init__.py

@@ -0,0 +1,358 @@
+# Copyright (c) 2021 Airbyte, Inc., all rights reserved.
+"""
+# Welcome to the Airbyte Python CDK!
+
+The Airbyte Python CDK is a Python library that provides a set of tools to help you build
+connectors for the Airbyte platform.
+
+## Building Source Connectors
+
+To build a source connector, you will want to refer to
+the following classes and modules:
+
+- `airbyte_cdk.sources`
+- `airbyte_cdk.sources.concurrent_source`
+- `airbyte_cdk.sources.config`
+- `airbyte_cdk.sources.file_based`
+- `airbyte_cdk.sources.streams`
+
+## Building Destination Connectors
+
+To build a destination connector, you will want to refer to
+the following classes and modules:
+
+- `airbyte_cdk.destinations`
+- `airbyte_cdk.destinations.Destination`
+- `airbyte_cdk.destinations.vector_db_based`
+
+## Working with Airbyte Protocol Models
+
+The Airbyte CDK provides a set of classes that help you work with the Airbyte protocol models:
+
+- `airbyte_cdk.models.airbyte_protocol`
+- `airbyte_cdk.models.airbyte_protocol_serializers`
+
+---
+
+API Reference
+
+---
+
+"""
+
+# Warning: The below imports are not stable and will cause circular
+# dependencies if auto-sorted with isort. Please keep them in the same order.
+# TODO: Submodules should import from lower-level modules, rather than importing from here.
+# Imports should also be placed in `if TYPE_CHECKING` blocks if they are only used as type
+# hints - again, to avoid circular dependencies.
+# Once those issues are resolved, the below can be sorted with isort.
+import dunamai as _dunamai
+
+from .config_observation import (
+    create_connector_config_control_message,
+    emit_configuration_as_airbyte_control_message,
+)
+from .connector import BaseConnector, Connector
+from .destinations import Destination
+from .entrypoint import AirbyteEntrypoint, launch
+from .logger import AirbyteLogFormatter, init_logger
+from .models import (
+    AdvancedAuth,
+    AirbyteConnectionStatus,
+    AirbyteLogMessage,
+    AirbyteMessage,
+    AirbyteRecordMessage,
+    AirbyteStream,
+    ConfiguredAirbyteCatalog,
+    ConfiguredAirbyteStream,
+    ConnectorSpecification,
+    DestinationSyncMode,
+    FailureType,
+    Level,
+    OAuthConfigSpecification,
+    OrchestratorType,
+    Status,
+    SyncMode,
+    Type,
+)
+from .sources import AbstractSource, Source
+from .sources.concurrent_source.concurrent_source import ConcurrentSource
+from .sources.concurrent_source.concurrent_source_adapter import ConcurrentSourceAdapter
+from .sources.config import BaseConfig
+from .sources.connector_state_manager import ConnectorStateManager
+from .sources.declarative.auth import DeclarativeOauth2Authenticator
+from .sources.declarative.auth.declarative_authenticator import DeclarativeAuthenticator, NoAuth
+from .sources.declarative.auth.oauth import DeclarativeSingleUseRefreshTokenOauth2Authenticator
+from .sources.declarative.auth.token import (
+    ApiKeyAuthenticator,
+    BasicHttpAuthenticator,
+    BearerAuthenticator,
+)
+from .sources.declarative.datetime.min_max_datetime import MinMaxDatetime
+from .sources.declarative.declarative_stream import DeclarativeStream
+from .sources.declarative.decoders import Decoder, JsonDecoder
+from .sources.declarative.exceptions import ReadException
+from .sources.declarative.extractors import DpathExtractor, RecordSelector
+from .sources.declarative.extractors.record_extractor import RecordExtractor
+from .sources.declarative.extractors.record_filter import RecordFilter
+from .sources.declarative.incremental import DatetimeBasedCursor
+from .sources.declarative.interpolation import InterpolatedBoolean, InterpolatedString
+from .sources.declarative.manifest_declarative_source import ManifestDeclarativeSource
+from .sources.declarative.migrations.legacy_to_per_partition_state_migration import (
+    LegacyToPerPartitionStateMigration,
+)
+from .sources.declarative.partition_routers import (
+    CartesianProductStreamSlicer,
+    SinglePartitionRouter,
+    SubstreamPartitionRouter,
+)
+from .sources.declarative.partition_routers.substream_partition_router import ParentStreamConfig
+from .sources.declarative.requesters import HttpRequester, Requester
+from .sources.declarative.requesters.error_handlers import BackoffStrategy
+from .sources.declarative.requesters.paginators import DefaultPaginator, PaginationStrategy
+from .sources.declarative.requesters.paginators.strategies import (
+    CursorPaginationStrategy,
+    OffsetIncrement,
+    PageIncrement,
+    StopConditionPaginationStrategyDecorator,
+)
+from .sources.declarative.requesters.request_option import RequestOption, RequestOptionType
+from .sources.declarative.requesters.request_options.default_request_options_provider import (
+    DefaultRequestOptionsProvider,
+)
+from .sources.declarative.requesters.request_options.interpolated_request_input_provider import (
+    InterpolatedRequestInputProvider,
+)
+from .sources.declarative.requesters.requester import HttpMethod
+from .sources.declarative.retrievers import SimpleRetriever
+from .sources.declarative.schema import JsonFileSchemaLoader
+from .sources.declarative.transformations.add_fields import AddedFieldDefinition, AddFields
+from .sources.declarative.transformations.transformation import RecordTransformation
+from .sources.declarative.types import FieldPointer
+from .sources.declarative.yaml_declarative_source import YamlDeclarativeSource
+from .sources.message import InMemoryMessageRepository, MessageRepository
+from .sources.source import TState
+from .sources.streams.availability_strategy import AvailabilityStrategy
+from .sources.streams.call_rate import (
+    AbstractAPIBudget,
+    CachedLimiterSession,
+    HttpAPIBudget,
+    HttpRequestMatcher,
+    LimiterSession,
+    MovingWindowCallRatePolicy,
+    Rate,
+)
+from .sources.streams.checkpoint import Cursor as LegacyCursor
+from .sources.streams.checkpoint import ResumableFullRefreshCursor
+from .sources.streams.concurrent.adapters import StreamFacade
+from .sources.streams.concurrent.cursor import (
+    ConcurrentCursor,
+    Cursor,
+    CursorField,
+    FinalStateCursor,
+)
+from .sources.streams.concurrent.state_converters.datetime_stream_state_converter import (
+    EpochValueConcurrentStreamStateConverter,
+    IsoMillisConcurrentStreamStateConverter,
+)
+from .sources.streams.core import IncrementalMixin, Stream, package_name_from_class
+from .sources.streams.http import HttpStream, HttpSubStream
+from .sources.streams.http.availability_strategy import HttpAvailabilityStrategy
+from .sources.streams.http.exceptions import (
+    BaseBackoffException,
+    DefaultBackoffException,
+    UserDefinedBackoffException,
+)
+from .sources.streams.http.rate_limiting import default_backoff_handler
+from .sources.streams.http.requests_native_auth import (
+    Oauth2Authenticator,
+    SingleUseRefreshTokenOauth2Authenticator,
+    TokenAuthenticator,
+)
+from .sources.streams.http.requests_native_auth.abstract_token import AbstractHeaderAuthenticator
+from .sources.types import Config, Record, StreamSlice
+from .sources.utils import casing
+from .sources.utils.schema_helpers import (
+    InternalConfig,
+    ResourceSchemaLoader,
+    check_config_against_spec_or_exit,
+    expand_refs,
+    split_config,
+)
+from .sources.utils.transform import TransformConfig, TypeTransformer
+from .utils import AirbyteTracedException, is_cloud_environment
+from .utils.constants import ENV_REQUEST_CACHE_PATH
+from .utils.event_timing import create_timer
+from .utils.oneof_option_config import OneOfOptionConfig
+from .utils.spec_schema_transformations import resolve_refs
+from .utils.stream_status_utils import as_airbyte_message
+
+__all__ = [
+    # Availability strategy
+    "AvailabilityStrategy",
+    "HttpAvailabilityStrategy",
+    # Checkpoint
+    "LegacyCursor",
+    "ResumableFullRefreshCursor",
+    # Concurrent
+    "ConcurrentCursor",
+    "ConcurrentSource",
+    "ConcurrentSourceAdapter",
+    "Cursor",
+    "CursorField",
+    "DEFAULT_CONCURRENCY",
+    "EpochValueConcurrentStreamStateConverter",
+    "FinalStateCursor",
+    "IsoMillisConcurrentStreamStateConverter",
+    "StreamFacade",
+    # Config observation
+    "create_connector_config_control_message",
+    "emit_configuration_as_airbyte_control_message",
+    # Connector
+    "AbstractSource",
+    "BaseConfig",
+    "BaseConnector",
+    "Connector",
+    "Destination",
+    "Source",
+    "TState",
+    # Declarative
+    "AddFields",
+    "AddedFieldDefinition",
+    "ApiKeyAuthenticator",
+    "BackoffStrategy",
+    "BasicHttpAuthenticator",
+    "BearerAuthenticator",
+    "CartesianProductStreamSlicer",
+    "CursorPaginationStrategy",
+    "DatetimeBasedCursor",
+    "DeclarativeAuthenticator",
+    "DeclarativeOauth2Authenticator",
+    "DeclarativeSingleUseRefreshTokenOauth2Authenticator",
+    "DeclarativeStream",
+    "Decoder",
+    "DefaultPaginator",
+    "DefaultRequestOptionsProvider",
+    "DpathExtractor",
+    "FieldPointer",
+    "HttpMethod",
+    "HttpRequester",
+    "InterpolatedBoolean",
+    "InterpolatedRequestInputProvider",
+    "InterpolatedString",
+    "JsonDecoder",
+    "JsonFileSchemaLoader",
+    "LegacyToPerPartitionStateMigration",
+    "ManifestDeclarativeSource",
+    "MinMaxDatetime",
+    "NoAuth",
+    "OffsetIncrement",
+    "PageIncrement",
+    "PaginationStrategy",
+    "ParentStreamConfig",
+    "ReadException",
+    "RecordExtractor",
+    "RecordFilter",
+    "RecordSelector",
+    "RecordTransformation",
+    "RequestOption",
+    "RequestOptionType",
+    "Requester",
+    "ResponseStatus",
+    "SimpleRetriever",
+    "SinglePartitionRouter",
+    "StopConditionPaginationStrategyDecorator",
+    "StreamSlice",
+    "SubstreamPartitionRouter",
+    "YamlDeclarativeSource",
+    # Entrypoint
+    "launch",
+    "AirbyteEntrypoint",
+    # HTTP
+    "AbstractAPIBudget",
+    "AbstractHeaderAuthenticator",
+    "BaseBackoffException",
+    "CachedLimiterSession",
+    "DefaultBackoffException",
+    "default_backoff_handler",
+    "HttpAPIBudget",
+    "HttpAuthenticator",
+    "HttpRequestMatcher",
+    "HttpStream",
+    "HttpSubStream",
+    "LimiterSession",
+    "MovingWindowCallRatePolicy",
+    "MultipleTokenAuthenticator",
+    "Oauth2Authenticator",
+    "Rate",
+    "SingleUseRefreshTokenOauth2Authenticator",
+    "TokenAuthenticator",
+    "UserDefinedBackoffException",
+    # Logger
+    "AirbyteLogFormatter",
+    "init_logger",
+    # Protocol classes
+    "AirbyteStream",
+    "AirbyteConnectionStatus",
+    "AirbyteMessage",
+    "ConfiguredAirbyteCatalog",
+    "Status",
+    "Type",
+    "OrchestratorType",
+    "ConfiguredAirbyteStream",
+    "DestinationSyncMode",
+    "SyncMode",
+    "FailureType",
+    "AdvancedAuth",
+    "AirbyteLogMessage",
+    "OAuthConfigSpecification",
+    "ConnectorSpecification",
+    "Level",
+    "AirbyteRecordMessage",
+    # Repository
+    "InMemoryMessageRepository",
+    "MessageRepository",
+    # State management
+    "ConnectorStateManager",
+    # Stream
+    "IncrementalMixin",
+    "Stream",
+    "StreamData",
+    "package_name_from_class",
+    # Utils
+    "AirbyteTracedException",
+    "is_cloud_environment",
+    "casing",
+    "InternalConfig",
+    "ResourceSchemaLoader",
+    "check_config_against_spec_or_exit",
+    "split_config",
+    "TransformConfig",
+    "TypeTransformer",
+    "ENV_REQUEST_CACHE_PATH",
+    "create_timer",
+    "OneOfOptionConfig",
+    "resolve_refs",
+    "as_airbyte_message",
+    # Types
+    "Config",
+    "Record",
+    "Source",
+    "StreamSlice",
+]
+
+__version__: str
+"""Version generated by poetry dynamic versioning during publish.
+
+When running in development, dunamai will calculate a new prerelease version
+from existing git release tag info.
+"""
+
+try:
+    __version__ = _dunamai.get_version(
+        "airbyte-cdk",
+        third_choice=_dunamai.Version.from_any_vcs,
+        fallback=_dunamai.Version("0.0.0+dev"),
+    ).serialize()
+except:
+    __version__ = "0.0.0+dev"

airbyte_cdk-6.13.1.dev4106/airbyte_cdk/cli/__init__.py

@@ -0,0 +1 @@
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.

airbyte_cdk-6.13.1.dev4106/airbyte_cdk/cli/source_declarative_manifest/__init__.py

@@ -0,0 +1,5 @@
+from airbyte_cdk.cli.source_declarative_manifest._run import run
+
+__all__ = [
+    "run",
+]