airbyte-cdk 0.0.0.dev0__py3-none-any.whl

airbyte_cdk/utils/traced_exception.py

@@ -0,0 +1,145 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+import time
+import traceback
+from typing import Any, Optional
+
+import orjson
+
+from airbyte_cdk.models import (
+    AirbyteConnectionStatus,
+    AirbyteErrorTraceMessage,
+    AirbyteMessage,
+    AirbyteMessageSerializer,
+    AirbyteTraceMessage,
+    FailureType,
+    Status,
+    StreamDescriptor,
+    TraceType,
+)
+from airbyte_cdk.models import Type as MessageType
+from airbyte_cdk.utils.airbyte_secrets_utils import filter_secrets
+
+
+class AirbyteTracedException(Exception):
+    """
+    An exception that should be emitted as an AirbyteTraceMessage
+    """
+
+    def __init__(
+        self,
+        internal_message: Optional[str] = None,
+        message: Optional[str] = None,
+        failure_type: FailureType = FailureType.system_error,
+        exception: Optional[BaseException] = None,
+        stream_descriptor: Optional[StreamDescriptor] = None,
+    ):
+        """
+        :param internal_message: the internal error that caused the failure
+        :param message: a user-friendly message that indicates the cause of the error
+        :param failure_type: the type of error
+        :param exception: the exception that caused the error, from which the stack trace should be retrieved
+        :param stream_descriptor: describe the stream from which the exception comes from
+        """
+        self.internal_message = internal_message
+        self.message = message
+        self.failure_type = failure_type
+        self._exception = exception
+        self._stream_descriptor = stream_descriptor
+        super().__init__(internal_message)
+
+    def as_airbyte_message(
+        self, stream_descriptor: Optional[StreamDescriptor] = None
+    ) -> AirbyteMessage:
+        """
+        Builds an AirbyteTraceMessage from the exception
+
+        :param stream_descriptor is deprecated, please use the stream_description in `__init__ or `from_exception`. If many
+          stream_descriptors are defined, the one from `as_airbyte_message` will be discarded.
+        """
+        now_millis = time.time_ns() // 1_000_000
+
+        trace_exc = self._exception or self
+        stack_trace_str = "".join(traceback.TracebackException.from_exception(trace_exc).format())
+
+        trace_message = AirbyteTraceMessage(
+            type=TraceType.ERROR,
+            emitted_at=now_millis,
+            error=AirbyteErrorTraceMessage(
+                message=self.message
+                or "Something went wrong in the connector. See the logs for more details.",
+                internal_message=self.internal_message,
+                failure_type=self.failure_type,
+                stack_trace=stack_trace_str,
+                stream_descriptor=self._stream_descriptor
+                if self._stream_descriptor is not None
+                else stream_descriptor,
+            ),
+        )
+
+        return AirbyteMessage(type=MessageType.TRACE, trace=trace_message)
+
+    def as_connection_status_message(self) -> Optional[AirbyteMessage]:
+        if self.failure_type == FailureType.config_error:
+            return AirbyteMessage(
+                type=MessageType.CONNECTION_STATUS,
+                connectionStatus=AirbyteConnectionStatus(
+                    status=Status.FAILED, message=self.message
+                ),
+            )
+        return None
+
+    def emit_message(self) -> None:
+        """
+        Prints the exception as an AirbyteTraceMessage.
+        Note that this will be called automatically on uncaught exceptions when using the airbyte_cdk entrypoint.
+        """
+        message = orjson.dumps(AirbyteMessageSerializer.dump(self.as_airbyte_message())).decode()
+        filtered_message = filter_secrets(message)
+        print(filtered_message)
+
+    @classmethod
+    def from_exception(
+        cls,
+        exc: BaseException,
+        stream_descriptor: Optional[StreamDescriptor] = None,
+        *args: Any,
+        **kwargs: Any,
+    ) -> "AirbyteTracedException":
+        """
+        Helper to create an AirbyteTracedException from an existing exception
+        :param exc: the exception that caused the error
+        :param stream_descriptor: describe the stream from which the exception comes from
+        """
+        return cls(
+            internal_message=str(exc),
+            exception=exc,
+            stream_descriptor=stream_descriptor,
+            *args,
+            **kwargs,
+        )  # type: ignore  # ignoring because of args and kwargs
+
+    def as_sanitized_airbyte_message(
+        self, stream_descriptor: Optional[StreamDescriptor] = None
+    ) -> AirbyteMessage:
+        """
+        Builds an AirbyteTraceMessage from the exception and sanitizes any secrets from the message body
+
+        :param stream_descriptor is deprecated, please use the stream_description in `__init__ or `from_exception`. If many
+          stream_descriptors are defined, the one from `as_sanitized_airbyte_message` will be discarded.
+        """
+        error_message = self.as_airbyte_message(stream_descriptor=stream_descriptor)
+        if error_message.trace.error.message:  # type: ignore[union-attr] # AirbyteMessage with MessageType.TRACE has AirbyteTraceMessage
+            error_message.trace.error.message = filter_secrets(  # type: ignore[union-attr]
+                error_message.trace.error.message,  # type: ignore[union-attr]
+            )
+        if error_message.trace.error.internal_message:  # type: ignore[union-attr] # AirbyteMessage with MessageType.TRACE has AirbyteTraceMessage
+            error_message.trace.error.internal_message = filter_secrets(  # type: ignore[union-attr] # AirbyteMessage with MessageType.TRACE has AirbyteTraceMessage
+                error_message.trace.error.internal_message  # type: ignore[union-attr] # AirbyteMessage with MessageType.TRACE has AirbyteTraceMessage
+            )
+        if error_message.trace.error.stack_trace:  # type: ignore[union-attr] # AirbyteMessage with MessageType.TRACE has AirbyteTraceMessage
+            error_message.trace.error.stack_trace = filter_secrets(  # type: ignore[union-attr] # AirbyteMessage with MessageType.TRACE has AirbyteTraceMessage
+                error_message.trace.error.stack_trace  # type: ignore[union-attr] # AirbyteMessage with MessageType.TRACE has AirbyteTraceMessage
+            )
+        return error_message

airbyte_cdk-0.0.0.dev0.dist-info/LICENSE.txt

@@ -0,0 +1,19 @@
+Copyright (c) 2020 Airbyte
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

airbyte_cdk-0.0.0.dev0.dist-info/LICENSE_SHORT

@@ -0,0 +1 @@
+Copyright (c) 2025 Airbyte, Inc., all rights reserved.

airbyte_cdk-0.0.0.dev0.dist-info/METADATA

@@ -0,0 +1,111 @@
+Metadata-Version: 2.1
+Name: airbyte-cdk
+Version: 0.0.0.dev0
+Summary: A framework for writing Airbyte Connectors.
+Home-page: https://airbyte.com
+License: MIT
+Keywords: airbyte,connector-development-kit,cdk
+Author: Airbyte
+Author-email: contact@airbyte.io
+Requires-Python: >=3.10,<3.13
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Provides-Extra: file-based
+Provides-Extra: sql
+Provides-Extra: vector-db-based
+Requires-Dist: Jinja2 (>=3.1.2,<3.2.0)
+Requires-Dist: PyYAML (>=6.0.1,<7.0.0)
+Requires-Dist: airbyte-protocol-models-dataclasses (>=0.15,<0.16)
+Requires-Dist: anyascii (>=0.3.2,<0.4.0)
+Requires-Dist: avro (>=1.11.2,<1.13.0) ; extra == "file-based"
+Requires-Dist: backoff
+Requires-Dist: cachetools
+Requires-Dist: cohere (==4.21) ; extra == "vector-db-based"
+Requires-Dist: cryptography (>=44.0.0,<45.0.0)
+Requires-Dist: dpath (>=2.1.6,<3.0.0)
+Requires-Dist: dunamai (>=1.22.0,<2.0.0)
+Requires-Dist: fastavro (>=1.8.0,<1.9.0) ; extra == "file-based"
+Requires-Dist: genson (==1.3.0)
+Requires-Dist: isodate (>=0.6.1,<0.7.0)
+Requires-Dist: jsonref (>=0.2,<0.3)
+Requires-Dist: jsonschema (>=4.17.3,<4.18.0)
+Requires-Dist: langchain (==0.1.16) ; extra == "vector-db-based"
+Requires-Dist: langchain_core (==0.1.42)
+Requires-Dist: markdown ; extra == "file-based"
+Requires-Dist: nltk (==3.9.1)
+Requires-Dist: numpy (<2)
+Requires-Dist: openai[embeddings] (==0.27.9) ; extra == "vector-db-based"
+Requires-Dist: orjson (>=3.10.7,<4.0.0)
+Requires-Dist: packaging
+Requires-Dist: pandas (==2.2.2)
+Requires-Dist: pdf2image (==1.16.3) ; extra == "file-based"
+Requires-Dist: pdfminer.six (==20221105) ; extra == "file-based"
+Requires-Dist: psutil (==6.1.0)
+Requires-Dist: pyarrow (>=19.0.0,<20.0.0) ; extra == "file-based"
+Requires-Dist: pydantic (>=2.7,<3.0)
+Requires-Dist: pyjwt (>=2.8.0,<3.0.0)
+Requires-Dist: pyrate-limiter (>=3.1.0,<3.2.0)
+Requires-Dist: pytesseract (==0.3.10) ; extra == "file-based"
+Requires-Dist: python-calamine (==0.2.3) ; extra == "file-based"
+Requires-Dist: python-dateutil (>=2.9.0,<3.0.0)
+Requires-Dist: python-snappy (==0.7.3) ; extra == "file-based"
+Requires-Dist: python-ulid (>=3.0.0,<4.0.0)
+Requires-Dist: pytz (==2024.2)
+Requires-Dist: rapidfuzz (>=3.10.1,<4.0.0)
+Requires-Dist: requests
+Requires-Dist: requests_cache
+Requires-Dist: serpyco-rs (>=1.10.2,<2.0.0)
+Requires-Dist: sqlalchemy (>=2.0,<3.0,!=2.0.36) ; extra == "sql"
+Requires-Dist: tiktoken (==0.8.0) ; extra == "vector-db-based"
+Requires-Dist: typing-extensions
+Requires-Dist: unstructured.pytesseract (>=0.3.12) ; extra == "file-based"
+Requires-Dist: unstructured[docx,pptx] (==0.10.27) ; extra == "file-based"
+Requires-Dist: wcmatch (==10.0)
+Requires-Dist: whenever (>=0.6.16,<0.7.0)
+Requires-Dist: xmltodict (>=0.13,<0.15)
+Project-URL: Documentation, https://docs.airbyte.io/
+Project-URL: Repository, https://github.com/airbytehq/airbyte-python-cdk
+Description-Content-Type: text/markdown
+
+# Airbyte Python CDK and Low-Code CDK
+
+Airbyte Python CDK is a framework for building Airbyte API Source Connectors. It provides a set of
+classes and helpers that make it easy to build a connector against an HTTP API (REST, GraphQL, etc),
+or a generic Python source connector.
+
+## Building Connectors with the CDK
+
+If you're looking to build a connector, we highly recommend that you first
+[start with the Connector Builder](https://docs.airbyte.com/connector-development/connector-builder-ui/overview).
+It should be enough for 90% connectors out there. For more flexible and complex connectors, use the
+[low-code CDK and `SourceDeclarativeManifest`](https://docs.airbyte.com/connector-development/config-based/low-code-cdk-overview).
+
+For more information on building connectors, please see the [Connector Development](https://docs.airbyte.com/connector-development/) guide on [docs.airbyte.com](https://docs.airbyte.com).
+
+## Python CDK Overview
+
+Airbyte CDK code is within `airbyte_cdk` directory. Here's a high level overview of what's inside:
+
+- `airbyte_cdk/connector_builder`. Internal wrapper that helps the Connector Builder platform run a declarative manifest (low-code connector). You should not use this code directly. If you need to run a `SourceDeclarativeManifest`, take a look at [`source-declarative-manifest`](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-declarative-manifest) connector implementation instead.
+- `airbyte_cdk/cli/source_declarative_manifest`. This module defines the `source-declarative-manifest` (aka "SDM") connector execution logic and associated CLI.
+- `airbyte_cdk/destinations`. Basic Destination connector support! If you're building a Destination connector in Python, try that. Some of our vector DB destinations like `destination-pinecone` are using that code.
+- `airbyte_cdk/models` expose `airbyte_protocol.models` as a part of `airbyte_cdk` package.
+- `airbyte_cdk/sources/concurrent_source` is the Concurrent CDK implementation. It supports reading data from streams concurrently per slice / partition, useful for connectors with high throughput and high number of records.
+- `airbyte_cdk/sources/declarative` is the low-code CDK. It works on top of Airbyte Python CDK, but provides a declarative manifest language to define streams, operations, etc. This makes it easier to build connectors without writing Python code.
+- `airbyte_cdk/sources/file_based` is the CDK for file-based sources. Examples include S3, Azure, GCS, etc.
+
+## Contributing
+
+For instructions on how to contribute, please see our [Contributing Guide](docs/CONTRIBUTING.md).
+
+## Release Management
+
+Please see the [Release Management](docs/RELEASES.md) guide for information on how to perform releases and pre-releases.
+