airbyte-cdk 6.8.1__py3-none-any.whl → 6.8.1rc1__py3-none-any.whl

airbyte_cdk/test/mock_http/response_builder.py

@@ -183,7 +183,7 @@ class HttpResponseBuilder:
 
 def _get_unit_test_folder(execution_folder: str) -> FilePath:
     # FIXME: This function should be removed after the next CDK release to avoid breaking amazon-seller-partner test code.
-    return get_unit_test_folder(execution_folder)
+    return get_unit_test_folder(execution_folder)  # type: ignore # get_unit_test_folder is known to return a FilePath
 
 
 def find_template(resource: str, execution_folder: str) -> Dict[str, Any]:

airbyte_cdk/utils/airbyte_secrets_utils.py

@@ -47,7 +47,7 @@ def get_secrets(
     result = []
     for path in secret_paths:
         try:
-            result.append(dpath.get(config, path))  # type: ignore  # dpath expect MutableMapping but doesn't need it
+            result.append(dpath.get(config, path))
         except KeyError:
             # Since we try to get paths to all known secrets in the spec, in the case of oneOfs, some secret fields may not be present
             # In that case, a KeyError is thrown. This is expected behavior.

airbyte_cdk/utils/event_timing.py

@@ -7,7 +7,7 @@ import logging
 import time
 from contextlib import contextmanager
 from dataclasses import dataclass, field
-from typing import Any, Generator, Literal, Optional
+from typing import Optional
 
 logger = logging.getLogger("airbyte")
 
@@ -18,13 +18,13 @@ class EventTimer:
        Event nesting follows a LIFO pattern, so finish will apply to the last started event.
     """
 
-    def __init__(self, name: str) -> None:
+    def __init__(self, name):
         self.name = name
-        self.events: dict[str, Any] = {}
+        self.events = {}
         self.count = 0
-        self.stack: list[Any] = []
+        self.stack = []
 
-    def start_event(self, name: str) -> None:
+    def start_event(self, name):
         """
         Start a new event and push it to the stack.
         """
@@ -32,7 +32,7 @@ class EventTimer:
         self.count += 1
         self.stack.insert(0, self.events[name])
 
-    def finish_event(self) -> None:
+    def finish_event(self):
         """
         Finish the current event and pop it from the stack.
         """
@@ -43,7 +43,7 @@ class EventTimer:
         else:
             logger.warning(f"{self.name} finish_event called without start_event")
 
-    def report(self, order_by: Literal["name", "duration"] = "name") -> str:
+    def report(self, order_by="name"):
         """
         :param order_by: 'name' or 'duration'
         """
@@ -69,15 +69,15 @@ class Event:
             return (self.end - self.start) / 1e9
         return float("+inf")
 
-    def __str__(self) -> str:
+    def __str__(self):
         return f"{self.name} {datetime.timedelta(seconds=self.duration)}"
 
-    def finish(self) -> None:
+    def finish(self):
         self.end = time.perf_counter_ns()
 
 
 @contextmanager
-def create_timer(name: str) -> Generator[EventTimer, Any, None]:
+def create_timer(name):
     """
     Creates a new EventTimer as a context manager to improve code readability.
     """

airbyte_cdk/utils/message_utils.py

@@ -8,16 +8,15 @@ def get_stream_descriptor(message: AirbyteMessage) -> HashableStreamDescriptor:
     match message.type:
         case Type.RECORD:
             return HashableStreamDescriptor(
-                name=message.record.stream,  # type: ignore[union-attr] # record has `stream`
-                namespace=message.record.namespace,  # type: ignore[union-attr] # record has `namespace`
-            )
+                name=message.record.stream, namespace=message.record.namespace
+            )  # type: ignore[union-attr] # record has `stream` and `namespace`
         case Type.STATE:
             if not message.state.stream or not message.state.stream.stream_descriptor:  # type: ignore[union-attr] # state has `stream`
                 raise ValueError(
                     "State message was not in per-stream state format, which is required for record counts."
                 )
             return HashableStreamDescriptor(
-                name=message.state.stream.stream_descriptor.name,  # type: ignore[union-attr] # state has `stream`
+                name=message.state.stream.stream_descriptor.name,
                 namespace=message.state.stream.stream_descriptor.namespace,  # type: ignore[union-attr] # state has `stream`
             )
         case _:

airbyte_cdk/utils/spec_schema_transformations.py

@@ -4,12 +4,11 @@
 
 import json
 import re
-from typing import Any
 
 from jsonschema import RefResolver
 
 
-def resolve_refs(schema: dict[str, Any]) -> dict[str, Any]:
+def resolve_refs(schema: dict) -> dict:
     """
     For spec schemas generated using Pydantic models, the resulting JSON schema can contain refs between object
     relationships.
@@ -21,6 +20,6 @@ def resolve_refs(schema: dict[str, Any]) -> dict[str, Any]:
         str_schema = str_schema.replace(
             ref_block, json.dumps(json_schema_ref_resolver.resolve(ref)[1])
         )
-    pyschema: dict[str, Any] = json.loads(str_schema)
+    pyschema: dict = json.loads(str_schema)
     del pyschema["definitions"]
     return pyschema

airbyte_cdk/utils/traced_exception.py

@@ -3,9 +3,9 @@
 #
 import time
 import traceback
-from typing import Any, Optional
+from typing import Optional
 
-import orjson
+from orjson import orjson
 
 from airbyte_cdk.models import (
     AirbyteConnectionStatus,
@@ -104,9 +104,9 @@ class AirbyteTracedException(Exception):
         cls,
         exc: BaseException,
         stream_descriptor: Optional[StreamDescriptor] = None,
-        *args: Any,
-        **kwargs: Any,
-    ) -> "AirbyteTracedException":
+        *args,
+        **kwargs,
+    ) -> "AirbyteTracedException":  # type: ignore  # ignoring because of args and kwargs
         """
         Helper to create an AirbyteTracedException from an existing exception
         :param exc: the exception that caused the error
@@ -131,15 +131,13 @@ class AirbyteTracedException(Exception):
         """
         error_message = self.as_airbyte_message(stream_descriptor=stream_descriptor)
         if error_message.trace.error.message:  # type: ignore[union-attr] # AirbyteMessage with MessageType.TRACE has AirbyteTraceMessage
-            error_message.trace.error.message = filter_secrets(  # type: ignore[union-attr]
-                error_message.trace.error.message,  # type: ignore[union-attr]
-            )
+            error_message.trace.error.message = filter_secrets(error_message.trace.error.message)  # type: ignore[union-attr] # AirbyteMessage with MessageType.TRACE has AirbyteTraceMessage
         if error_message.trace.error.internal_message:  # type: ignore[union-attr] # AirbyteMessage with MessageType.TRACE has AirbyteTraceMessage
-            error_message.trace.error.internal_message = filter_secrets(  # type: ignore[union-attr] # AirbyteMessage with MessageType.TRACE has AirbyteTraceMessage
-                error_message.trace.error.internal_message  # type: ignore[union-attr] # AirbyteMessage with MessageType.TRACE has AirbyteTraceMessage
-            )
+            error_message.trace.error.internal_message = filter_secrets(
+                error_message.trace.error.internal_message
+            )  # type: ignore[union-attr] # AirbyteMessage with MessageType.TRACE has AirbyteTraceMessage
         if error_message.trace.error.stack_trace:  # type: ignore[union-attr] # AirbyteMessage with MessageType.TRACE has AirbyteTraceMessage
-            error_message.trace.error.stack_trace = filter_secrets(  # type: ignore[union-attr] # AirbyteMessage with MessageType.TRACE has AirbyteTraceMessage
-                error_message.trace.error.stack_trace  # type: ignore[union-attr] # AirbyteMessage with MessageType.TRACE has AirbyteTraceMessage
-            )
+            error_message.trace.error.stack_trace = filter_secrets(
+                error_message.trace.error.stack_trace
+            )  # type: ignore[union-attr] # AirbyteMessage with MessageType.TRACE has AirbyteTraceMessage
         return error_message

airbyte_cdk-6.8.1rc1.dist-info/METADATA

@@ -0,0 +1,307 @@
+Metadata-Version: 2.1
+Name: airbyte-cdk
+Version: 6.8.1rc1
+Summary: A framework for writing Airbyte Connectors.
+Home-page: https://airbyte.com
+License: MIT
+Keywords: airbyte,connector-development-kit,cdk
+Author: Airbyte
+Author-email: contact@airbyte.io
+Requires-Python: >=3.10,<3.13
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Provides-Extra: file-based
+Provides-Extra: sphinx-docs
+Provides-Extra: sql
+Provides-Extra: vector-db-based
+Requires-Dist: Deprecated (>=1.2,<1.3)
+Requires-Dist: Jinja2 (>=3.1.2,<3.2.0)
+Requires-Dist: PyYAML (>=6.0.1,<7.0.0)
+Requires-Dist: Sphinx (>=4.2,<4.3) ; extra == "sphinx-docs"
+Requires-Dist: airbyte-protocol-models-dataclasses (>=0.14,<0.15)
+Requires-Dist: avro (>=1.11.2,<1.12.0) ; extra == "file-based"
+Requires-Dist: backoff
+Requires-Dist: cachetools
+Requires-Dist: cohere (==4.21) ; extra == "vector-db-based"
+Requires-Dist: cryptography (>=42.0.5,<44.0.0)
+Requires-Dist: dpath (>=2.1.6,<3.0.0)
+Requires-Dist: dunamai (>=1.22.0,<2.0.0)
+Requires-Dist: fastavro (>=1.8.0,<1.9.0) ; extra == "file-based"
+Requires-Dist: genson (==1.3.0)
+Requires-Dist: isodate (>=0.6.1,<0.7.0)
+Requires-Dist: jsonref (>=0.2,<0.3)
+Requires-Dist: jsonschema (>=4.17.3,<4.18.0)
+Requires-Dist: langchain (==0.1.16) ; extra == "vector-db-based"
+Requires-Dist: langchain_core (==0.1.42)
+Requires-Dist: markdown ; extra == "file-based"
+Requires-Dist: nltk (==3.9.1)
+Requires-Dist: numpy (<2)
+Requires-Dist: openai[embeddings] (==0.27.9) ; extra == "vector-db-based"
+Requires-Dist: orjson (>=3.10.7,<4.0.0)
+Requires-Dist: pandas (==2.2.2)
+Requires-Dist: pdf2image (==1.16.3) ; extra == "file-based"
+Requires-Dist: pdfminer.six (==20221105) ; extra == "file-based"
+Requires-Dist: pendulum (<3.0.0)
+Requires-Dist: psutil (==6.1.0)
+Requires-Dist: pyarrow (>=15.0.0,<15.1.0) ; extra == "file-based"
+Requires-Dist: pydantic (>=2.7,<3.0)
+Requires-Dist: pyjwt (>=2.8.0,<3.0.0)
+Requires-Dist: pyrate-limiter (>=3.1.0,<3.2.0)
+Requires-Dist: pytesseract (==0.3.10) ; extra == "file-based"
+Requires-Dist: python-calamine (==0.2.3) ; extra == "file-based"
+Requires-Dist: python-dateutil
+Requires-Dist: python-snappy (==0.7.3) ; extra == "file-based"
+Requires-Dist: python-ulid (>=3.0.0,<4.0.0)
+Requires-Dist: pytz (==2024.1)
+Requires-Dist: rapidfuzz (>=3.10.1,<4.0.0)
+Requires-Dist: requests
+Requires-Dist: requests_cache
+Requires-Dist: serpyco-rs (>=1.10.2,<2.0.0)
+Requires-Dist: sphinx-rtd-theme (>=1.0,<1.1) ; extra == "sphinx-docs"
+Requires-Dist: sqlalchemy (>=2.0,<3.0,!=2.0.36) ; extra == "sql"
+Requires-Dist: tiktoken (==0.8.0) ; extra == "vector-db-based"
+Requires-Dist: unstructured.pytesseract (>=0.3.12) ; extra == "file-based"
+Requires-Dist: unstructured[docx,pptx] (==0.10.27) ; extra == "file-based"
+Requires-Dist: wcmatch (==10.0)
+Requires-Dist: xmltodict (>=0.13.0,<0.14.0)
+Project-URL: Documentation, https://docs.airbyte.io/
+Project-URL: Repository, https://github.com/airbytehq/airbyte-python-cdk
+Description-Content-Type: text/markdown
+
+# Airbyte Python CDK and Low-Code CDK
+
+Airbyte Python CDK is a framework for building Airbyte API Source Connectors. It provides a set of
+classes and helpers that make it easy to build a connector against an HTTP API (REST, GraphQL, etc),
+or a generic Python source connector.
+
+## Usage
+
+If you're looking to build a connector, we highly recommend that you
+[start with the Connector Builder](https://docs.airbyte.com/connector-development/connector-builder-ui/overview).
+It should be enough for 90% connectors out there. For more flexible and complex connectors, use the
+[low-code CDK and `SourceDeclarativeManifest`](https://docs.airbyte.com/connector-development/config-based/low-code-cdk-overview).
+
+If that doesn't work, then consider building on top of the
+[lower-level Python CDK itself](https://docs.airbyte.com/connector-development/cdk-python/).
+
+### Quick Start
+
+To get started on a Python CDK based connector or a low-code connector, you can generate a connector
+project from a template:
+
+```bash
+# from the repo root
+cd airbyte-integrations/connector-templates/generator
+./generate.sh
+```
+
+### Example Connectors
+
+**HTTP Connectors**:
+
+- [Stripe](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-stripe/)
+- [Salesforce](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-salesforce/)
+
+**Python connectors using the bare-bones `Source` abstraction**:
+
+- [Google Sheets](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-google-sheets/google_sheets_source/google_sheets_source.py)
+
+This will generate a project with a type and a name of your choice and put it in
+`airbyte-integrations/connectors`. Open the directory with your connector in an editor and follow
+the `TODO` items.
+
+## Python CDK Overview
+
+Airbyte CDK code is within `airbyte_cdk` directory. Here's a high level overview of what's inside:
+
+- `connector_builder`. Internal wrapper that helps the Connector Builder platform run a declarative
+  manifest (low-code connector). You should not use this code directly. If you need to run a
+  `SourceDeclarativeManifest`, take a look at
+  [`source-declarative-manifest`](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-declarative-manifest)
+  connector implementation instead.
+- `destinations`. Basic Destination connector support! If you're building a Destination connector in
+  Python, try that. Some of our vector DB destinations like `destination-pinecone` are using that
+  code.
+- `models` expose `airbyte_protocol.models` as a part of `airbyte_cdk` package.
+- `sources/concurrent_source` is the Concurrent CDK implementation. It supports reading data from
+  streams concurrently per slice / partition, useful for connectors with high throughput and high
+  number of records.
+- `sources/declarative` is the low-code CDK. It works on top of Airbyte Python CDK, but provides a
+  declarative manifest language to define streams, operations, etc. This makes it easier to build
+  connectors without writing Python code.
+- `sources/file_based` is the CDK for file-based sources. Examples include S3, Azure, GCS, etc.
+
+## Contributing
+
+Thank you for being interested in contributing to Airbyte Python CDK! Here are some guidelines to
+get you started:
+
+- We adhere to the [code of conduct](/CODE_OF_CONDUCT.md).
+- You can contribute by reporting bugs, posting github discussions, opening issues, improving
+  [documentation](/docs/), and submitting pull requests with bugfixes and new features alike.
+- If you're changing the code, please add unit tests for your change.
+- When submitting issues or PRs, please add a small reproduction project. Using the changes in your
+  connector and providing that connector code as an example (or a satellite PR) helps!
+
+### First time setup
+
+Install the project dependencies and development tools:
+
+```bash
+poetry install --all-extras
+```
+
+Installing all extras is required to run the full suite of unit tests.
+
+#### Running tests locally
+
+- Iterate on the CDK code locally
+- Run tests via `poetry run poe unit-test-with-cov`, or `python -m pytest -s unit_tests` if you want
+  to pass pytest options.
+- Run `poetry run poe check-local` to lint all code, type-check modified code, and run unit tests
+  with coverage in one command.
+
+To see all available scripts, run `poetry run poe`.
+
+#### Formatting the code
+
+- Iterate on the CDK code locally
+- Run `poetry run ruff format` to format your changes.
+
+To see all available `ruff` options, run `poetry run ruff`.
+
+##### Autogenerated files
+
+Low-code CDK models are generated from `sources/declarative/declarative_component_schema.yaml`. If
+the iteration you are working on includes changes to the models or the connector generator, you
+might want to regenerate them. In order to do that, you can run:
+
+```bash
+poetry run poe build
+```
+
+This will generate the code generator docker image and the component manifest files based on the
+schemas and templates.
+
+#### Testing
+
+All tests are located in the `unit_tests` directory. Run `poetry run poe unit-test-with-cov` to run
+them. This also presents a test coverage report. For faster iteration with no coverage report and
+more options, `python -m pytest -s unit_tests` is a good place to start.
+
+#### Building and testing a connector with your local CDK
+
+When developing a new feature in the CDK, you may find it helpful to run a connector that uses that
+new feature. You can test this in one of two ways:
+
+- Running a connector locally
+- Building and running a source via Docker
+
+##### Installing your local CDK into a local Python connector
+
+Open the connector's `pyproject.toml` file and replace the line with `airbyte_cdk` with the
+following:
+
+```toml
+airbyte_cdk = { path = "../../../airbyte-cdk/python/airbyte_cdk", develop = true }
+```
+
+Then, running `poetry update` should reinstall `airbyte_cdk` from your local working directory.
+
+##### Building a Python connector in Docker with your local CDK installed
+
+_Pre-requisite: Install the
+[`airbyte-ci` CLI](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md)_
+
+You can build your connector image with the local CDK using
+
+```bash
+# from the airbytehq/airbyte base directory
+airbyte-ci connectors --use-local-cdk --name=<CONNECTOR> build
+```
+
+Note that the local CDK is injected at build time, so if you make changes, you will have to run the
+build command again to see them reflected.
+
+##### Running Connector Acceptance Tests for a single connector in Docker with your local CDK installed
+
+_Pre-requisite: Install the
+[`airbyte-ci` CLI](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md)_
+
+To run acceptance tests for a single connectors using the local CDK, from the connector directory,
+run
+
+```bash
+airbyte-ci connectors --use-local-cdk --name=<CONNECTOR> test
+```
+
+#### When you don't have access to the API
+
+There may be a time when you do not have access to the API (either because you don't have the
+credentials, network access, etc...) You will probably still want to do end-to-end testing at least
+once. In order to do so, you can emulate the server you would be reaching using a server stubbing
+tool.
+
+For example, using [mockserver](https://www.mock-server.com/), you can set up an expectation file
+like this:
+
+```json
+{
+  "httpRequest": {
+    "method": "GET",
+    "path": "/data"
+  },
+  "httpResponse": {
+    "body": "{\"data\": [{\"record_key\": 1}, {\"record_key\": 2}]}"
+  }
+}
+```
+
+Assuming this file has been created at `secrets/mock_server_config/expectations.json`, running the
+following command will allow to match any requests on path `/data` to return the response defined in
+the expectation file:
+
+```bash
+docker run -d --rm -v $(pwd)/secrets/mock_server_config:/config -p 8113:8113 --env MOCKSERVER_LOG_LEVEL=TRACE --env MOCKSERVER_SERVER_PORT=8113 --env MOCKSERVER_WATCH_INITIALIZATION_JSON=true --env MOCKSERVER_PERSISTED_EXPECTATIONS_PATH=/config/expectations.json --env MOCKSERVER_INITIALIZATION_JSON_PATH=/config/expectations.json mockserver/mockserver:5.15.0
+```
+
+HTTP requests to `localhost:8113/data` should now return the body defined in the expectations file.
+To test this, the implementer either has to change the code which defines the base URL for Python
+source or update the `url_base` from low-code. With the Connector Builder running in docker, you
+will have to use domain `host.docker.internal` instead of `localhost` as the requests are executed
+within docker.
+
+#### Publishing a new version to PyPi
+
+Python CDK has a
+[GitHub workflow](https://github.com/airbytehq/airbyte/actions/workflows/publish-cdk-command-manually.yml)
+that manages the CDK changelog, making a new release for `airbyte_cdk`, publishing it to PyPI, and
+then making a commit to update (and subsequently auto-release)
+[`source-declarative-manifest`](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-declarative-manifest)
+and Connector Builder (in the platform repository).
+
+> [!Note]: The workflow will handle the `CHANGELOG.md` entry for you. You should not add changelog
+> lines in your PRs to the CDK itself.
+
+> [!Warning]: The workflow bumps version on it's own, please don't change the CDK version in
+> `pyproject.toml` manually.
+
+1. You only trigger the release workflow once all the PRs that you want to be included are already
+   merged into the `master` branch.
+2. The
+   [`Publish CDK Manually`](https://github.com/airbytehq/airbyte/actions/workflows/publish-cdk-command-manually.yml)
+   workflow from master using `release-type=major|manor|patch` and setting the changelog message.
+3. When the workflow runs, it will commit a new version directly to master branch.
+4. The workflow will bump the version of `source-declarative-manifest` according to the
+   `release-type` of the CDK, then commit these changes back to master. The commit to master will
+   kick off a publish of the new version of `source-declarative-manifest`.
+5. The workflow will also add a pull request to `airbyte-platform-internal` repo to bump the
+   dependency in Connector Builder.
+