airbyte-cdk 0.72.1__py3-none-any.whl → 6.17.1.dev0__py3-none-any.whl

airbyte_cdk/sources/streams/http/auth/token.py

@@ -1,47 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-
-import base64
-from itertools import cycle
-from typing import Any, List, Mapping
-
-from deprecated import deprecated
-
-from .core import HttpAuthenticator
-
-
-@deprecated(version="0.1.20", reason="Use airbyte_cdk.sources.streams.http.requests_native_auth.TokenAuthenticator instead")
-class TokenAuthenticator(HttpAuthenticator):
-    def __init__(self, token: str, auth_method: str = "Bearer", auth_header: str = "Authorization"):
-        self.auth_method = auth_method
-        self.auth_header = auth_header
-        self._token = token
-
-    def get_auth_header(self) -> Mapping[str, Any]:
-        return {self.auth_header: f"{self.auth_method} {self._token}"}
-
-
-@deprecated(version="0.1.20", reason="Use airbyte_cdk.sources.streams.http.requests_native_auth.MultipleTokenAuthenticator instead")
-class MultipleTokenAuthenticator(HttpAuthenticator):
-    def __init__(self, tokens: List[str], auth_method: str = "Bearer", auth_header: str = "Authorization"):
-        self.auth_method = auth_method
-        self.auth_header = auth_header
-        self._tokens = tokens
-        self._tokens_iter = cycle(self._tokens)
-
-    def get_auth_header(self) -> Mapping[str, Any]:
-        return {self.auth_header: f"{self.auth_method} {next(self._tokens_iter)}"}
-
-
-class BasicHttpAuthenticator(TokenAuthenticator):
-    """
-    Builds auth based off the basic authentication scheme as defined by RFC 7617, which transmits credentials as USER ID/password pairs, encoded using bas64
-    https://developer.mozilla.org/en-US/docs/Web/HTTP/Authentication#basic_authentication_scheme
-    """
-
-    def __init__(self, username: str, password: str, auth_method: str = "Basic", auth_header: str = "Authorization"):
-        auth_string = f"{username}:{password}".encode("utf8")
-        b64_encoded = base64.b64encode(auth_string).decode("utf8")
-        super().__init__(b64_encoded, auth_method, auth_header)

airbyte_cdk/sources/streams/utils/stream_helper.py

@@ -1,40 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-from typing import Any, Mapping, Optional
-
-from airbyte_cdk.models import SyncMode
-from airbyte_cdk.sources.streams.core import Stream, StreamData
-
-
-def get_first_stream_slice(stream) -> Optional[Mapping[str, Any]]:
-    """
-    Gets the first stream_slice from a given stream's stream_slices.
-    :param stream: stream
-    :raises StopIteration: if there is no first slice to return (the stream_slices generator is empty)
-    :return: first stream slice from 'stream_slices' generator (`None` is a valid stream slice)
-    """
-    # We wrap the return output of stream_slices() because some implementations return types that are iterable,
-    # but not iterators such as lists or tuples
-    slices = iter(
-        stream.stream_slices(
-            cursor_field=stream.cursor_field,
-            sync_mode=SyncMode.full_refresh,
-        )
-    )
-    return next(slices)
-
-
-def get_first_record_for_slice(stream: Stream, stream_slice: Optional[Mapping[str, Any]]) -> StreamData:
-    """
-    Gets the first record for a stream_slice of a stream.
-    :param stream: stream
-    :param stream_slice: stream_slice
-    :raises StopIteration: if there is no first record to return (the read_records generator is empty)
-    :return: StreamData containing the first record in the slice
-    """
-    # We wrap the return output of read_records() because some implementations return types that are iterable,
-    # but not iterators such as lists or tuples
-    records_for_slice = iter(stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=stream_slice))
-    return next(records_for_slice)

airbyte_cdk/sources/utils/catalog_helpers.py

@@ -1,22 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-
-from airbyte_cdk.models import AirbyteCatalog, SyncMode
-
-
-class CatalogHelper:
-    @staticmethod
-    def coerce_catalog_as_full_refresh(catalog: AirbyteCatalog) -> AirbyteCatalog:
-        """
-        Updates the sync mode on all streams in this catalog to be full refresh
-        """
-        coerced_catalog = catalog.copy()
-        for stream in catalog.streams:
-            stream.source_defined_cursor = False
-            stream.supported_sync_modes = [SyncMode.full_refresh]
-            stream.default_cursor_field = None
-
-        # remove nulls
-        return AirbyteCatalog.parse_raw(coerced_catalog.json(exclude_unset=True, exclude_none=True))

airbyte_cdk/sources/utils/schema_models.py

@@ -1,84 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-from typing import Any, Dict, Optional, Type
-
-from airbyte_cdk.sources.utils.schema_helpers import expand_refs
-from pydantic import BaseModel, Extra
-from pydantic.main import ModelMetaclass
-from pydantic.typing import resolve_annotations
-
-
-class AllOptional(ModelMetaclass):
-    """
-    Metaclass for marking all Pydantic model fields as Optional
-    Here is example of declaring model using this metaclass like:
-    '''
-            class MyModel(BaseModel, metaclass=AllOptional):
-                a: str
-                b: str
-    '''
-    it is an equivalent of:
-    '''
-            class MyModel(BaseModel):
-                a: Optional[str]
-                b: Optional[str]
-    '''
-    It would make code more clear and eliminate a lot of manual work.
-    """
-
-    def __new__(mcs, name, bases, namespaces, **kwargs):
-        """
-        Iterate through fields and wrap then with typing.Optional type.
-        """
-        annotations = resolve_annotations(namespaces.get("__annotations__", {}), namespaces.get("__module__", None))
-        for base in bases:
-            annotations = {**annotations, **getattr(base, "__annotations__", {})}
-        for field in annotations:
-            if not field.startswith("__"):
-                annotations[field] = Optional[annotations[field]]
-        namespaces["__annotations__"] = annotations
-        return super().__new__(mcs, name, bases, namespaces, **kwargs)
-
-
-class BaseSchemaModel(BaseModel):
-    """
-    Base class for all schema models. It has some extra schema postprocessing.
-    Can be used in combination with AllOptional metaclass
-    """
-
-    class Config:
-        extra = Extra.allow
-
-        @classmethod
-        def schema_extra(cls, schema: Dict[str, Any], model: Type[BaseModel]) -> None:
-            """Modify generated jsonschema, remove "title", "description" and "required" fields.
-
-            Pydantic doesn't treat Union[None, Any] type correctly when generate jsonschema,
-            so we can't set field as nullable (i.e. field that can have either null and non-null values),
-            We generate this jsonschema value manually.
-
-            :param schema: generated jsonschema
-            :param model:
-            """
-            schema.pop("title", None)
-            schema.pop("description", None)
-            schema.pop("required", None)
-            for name, prop in schema.get("properties", {}).items():
-                prop.pop("title", None)
-                prop.pop("description", None)
-                allow_none = model.__fields__[name].allow_none
-                if allow_none:
-                    if "type" in prop:
-                        prop["type"] = ["null", prop["type"]]
-                    elif "$ref" in prop:
-                        ref = prop.pop("$ref")
-                        prop["oneOf"] = [{"type": "null"}, {"$ref": ref}]
-
-    @classmethod
-    def schema(cls, *args, **kwargs) -> Dict[str, Any]:
-        """We're overriding the schema classmethod to enable some post-processing"""
-        schema = super().schema(*args, **kwargs)
-        expand_refs(schema)
-        return schema

airbyte_cdk-0.72.1.dist-info/METADATA

@@ -1,243 +0,0 @@
-Metadata-Version: 2.1
-Name: airbyte-cdk
-Version: 0.72.1
-Summary: A framework for writing Airbyte Connectors.
-Home-page: https://github.com/airbytehq/airbyte
-Author: Airbyte
-Author-email: contact@airbyte.io
-License: MIT
-Project-URL: Documentation, https://docs.airbyte.io/
-Project-URL: Source, https://github.com/airbytehq/airbyte
-Project-URL: Tracker, https://github.com/airbytehq/airbyte/issues
-Keywords: airbyte connector-development-kit cdk
-Classifier: Development Status :: 3 - Alpha
-Classifier: Intended Audience :: Developers
-Classifier: Topic :: Scientific/Engineering
-Classifier: Topic :: Software Development :: Libraries :: Python Modules
-Classifier: License :: OSI Approved :: MIT License
-Classifier: Programming Language :: Python :: 3.8
-Requires-Python: >=3.8
-Description-Content-Type: text/markdown
-License-File: LICENSE.txt
-Requires-Dist: airbyte-protocol-models ==0.5.1
-Requires-Dist: backoff
-Requires-Dist: dpath ~=2.0.1
-Requires-Dist: isodate ~=0.6.1
-Requires-Dist: jsonschema ~=3.2.0
-Requires-Dist: jsonref ~=0.2
-Requires-Dist: pendulum <3.0.0
-Requires-Dist: genson ==1.2.2
-Requires-Dist: pydantic <2.0.0,>=1.10.8
-Requires-Dist: pyrate-limiter ~=3.1.0
-Requires-Dist: python-dateutil
-Requires-Dist: PyYAML >=6.0.1
-Requires-Dist: requests
-Requires-Dist: requests-cache
-Requires-Dist: Deprecated ~=1.2
-Requires-Dist: Jinja2 ~=3.1.2
-Requires-Dist: cachetools
-Requires-Dist: wcmatch ==8.4
-Provides-Extra: dev
-Requires-Dist: avro ~=1.11.2 ; extra == 'dev'
-Requires-Dist: fastavro ~=1.8.0 ; extra == 'dev'
-Requires-Dist: freezegun ; extra == 'dev'
-Requires-Dist: mypy ; extra == 'dev'
-Requires-Dist: pytest ; extra == 'dev'
-Requires-Dist: pytest-cov ; extra == 'dev'
-Requires-Dist: pytest-mock ; extra == 'dev'
-Requires-Dist: requests-mock ; extra == 'dev'
-Requires-Dist: pytest-httpserver ; extra == 'dev'
-Requires-Dist: pandas ==2.0.3 ; extra == 'dev'
-Requires-Dist: pyarrow ~=15.0.0 ; extra == 'dev'
-Requires-Dist: langchain ==0.0.271 ; extra == 'dev'
-Requires-Dist: openai[embeddings] ==0.27.9 ; extra == 'dev'
-Requires-Dist: cohere ==4.21 ; extra == 'dev'
-Requires-Dist: tiktoken ==0.4.0 ; extra == 'dev'
-Requires-Dist: unstructured ==0.10.27 ; extra == 'dev'
-Requires-Dist: unstructured[docx,pptx] ==0.10.27 ; extra == 'dev'
-Requires-Dist: pdf2image ==1.16.3 ; extra == 'dev'
-Requires-Dist: pdfminer.six ==20221105 ; extra == 'dev'
-Requires-Dist: unstructured.pytesseract >=0.3.12 ; extra == 'dev'
-Requires-Dist: pytesseract ==0.3.10 ; extra == 'dev'
-Requires-Dist: markdown ; extra == 'dev'
-Provides-Extra: file-based
-Requires-Dist: avro ~=1.11.2 ; extra == 'file-based'
-Requires-Dist: fastavro ~=1.8.0 ; extra == 'file-based'
-Requires-Dist: pyarrow ~=15.0.0 ; extra == 'file-based'
-Requires-Dist: unstructured ==0.10.27 ; extra == 'file-based'
-Requires-Dist: unstructured[docx,pptx] ==0.10.27 ; extra == 'file-based'
-Requires-Dist: pdf2image ==1.16.3 ; extra == 'file-based'
-Requires-Dist: pdfminer.six ==20221105 ; extra == 'file-based'
-Requires-Dist: unstructured.pytesseract >=0.3.12 ; extra == 'file-based'
-Requires-Dist: pytesseract ==0.3.10 ; extra == 'file-based'
-Requires-Dist: markdown ; extra == 'file-based'
-Provides-Extra: sphinx-docs
-Requires-Dist: Sphinx ~=4.2 ; extra == 'sphinx-docs'
-Requires-Dist: sphinx-rtd-theme ~=1.0 ; extra == 'sphinx-docs'
-Provides-Extra: vector-db-based
-Requires-Dist: langchain ==0.0.271 ; extra == 'vector-db-based'
-Requires-Dist: openai[embeddings] ==0.27.9 ; extra == 'vector-db-based'
-Requires-Dist: cohere ==4.21 ; extra == 'vector-db-based'
-Requires-Dist: tiktoken ==0.4.0 ; extra == 'vector-db-based'
-
-# Connector Development Kit \(Python\)
-
-The Airbyte Python CDK is a framework for rapidly developing production-grade Airbyte connectors.The CDK currently offers helpers specific for creating Airbyte source connectors for:
-
-- HTTP APIs \(REST APIs, GraphQL, etc..\)
-- Generic Python sources \(anything not covered by the above\)
-
-The CDK provides an improved developer experience by providing basic implementation structure and abstracting away low-level glue boilerplate.
-
-This document is a general introduction to the CDK. Readers should have basic familiarity with the [Airbyte Specification](https://docs.airbyte.com/understanding-airbyte/airbyte-protocol/) before proceeding.
-
-## Getting Started
-
-Generate an empty connector using the code generator. First clone the Airbyte repository then from the repository root run
-
-```bash
-cd airbyte-integrations/connector-templates/generator
-./generate.sh
-```
-
-then follow the interactive prompt. Next, find all `TODO`s in the generated project directory -- they're accompanied by lots of comments explaining what you'll need to do in order to implement your connector. Upon completing all TODOs properly, you should have a functioning connector.
-
-Additionally, you can follow [this tutorial](https://docs.airbyte.com/connector-development/cdk-python/) for a complete walkthrough of creating an HTTP connector using the Airbyte CDK.
-
-### Concepts & Documentation
-
-See the [concepts docs](docs/concepts/) for a tour through what the API offers.
-
-### Example Connectors
-
-**HTTP Connectors**:
-
-- [Stripe](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-stripe/source_stripe/source.py)
-- [Slack](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-slack/source_slack/source.py)
-
-**Simple Python connectors using the barebones `Source` abstraction**:
-
-- [Google Sheets](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-google-sheets/google_sheets_source/google_sheets_source.py)
-- [Mailchimp](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/source.py)
-
-## Contributing
-
-### First time setup
-
-We assume `python` points to Python 3.9 or higher.
-
-Setup a virtual env:
-
-```bash
-python -m venv .venv
-source .venv/bin/activate
-pip install -e ".[dev]" # [dev] installs development-only dependencies
-```
-
-#### Iteration
-
-- Iterate on the code locally
-- Run tests via `python -m pytest -s unit_tests`
-- Perform static type checks using `mypy airbyte_cdk`. `MyPy` configuration is in `mypy.ini`.
-- Run `mypy <files to check>` to only check specific files. This is useful as the CDK still contains code that is not compliant.
-- The `type_check_and_test.sh` script bundles both type checking and testing in one convenient command. Feel free to use it!
-
-##### Autogenerated files
-
-If the iteration you are working on includes changes to the models, you might want to regenerate them. In order to do that, you can run:
-
-```bash
-cd airbyte-cdk/python
-./gradlew build
-```
-
-This will generate the files based on the schemas, add the license information and format the code. If you want to only do the former and rely on
-pre-commit to the others, you can run the appropriate generation command i.e. `./gradlew generateComponentManifestClassFiles`.
-
-#### Testing
-
-All tests are located in the `unit_tests` directory. Run `python -m pytest --cov=airbyte_cdk unit_tests/` to run them. This also presents a test coverage report.
-
-#### Building and testing a connector with your local CDK
-
-When developing a new feature in the CDK, you may find it helpful to run a connector that uses that new feature. You can test this in one of two ways:
-
-- Running a connector locally
-- Building and running a source via Docker
-
-##### Installing your local CDK into a local Python connector
-
-In order to get a local Python connector running your local CDK, do the following.
-
-First, make sure you have your connector's virtual environment active:
-
-```bash
-# from the `airbyte/airbyte-integrations/connectors/<connector-directory>` directory
-source .venv/bin/activate
-
-# if you haven't installed dependencies for your connector already
-pip install -e .
-```
-
-Then, navigate to the CDK and install it in editable mode:
-
-```bash
-cd ../../../airbyte-cdk/python
-pip install -e .
-```
-
-You should see that `pip` has uninstalled the version of `airbyte-cdk` defined by your connector's `setup.py` and installed your local CDK. Any changes you make will be immediately reflected in your editor, so long as your editor's interpreter is set to your connector's virtual environment.
-
-##### Building a Python connector in Docker with your local CDK installed
-
-_Pre-requisite: Install the [`airbyte-ci` CLI](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md)_
-
-You can build your connector image with the local CDK using
-
-```bash
-# from the airbytehq/airbyte base directory
-airbyte-ci connectors --use-local-cdk --name=<CONNECTOR> build
-```
-
-Note that the local CDK is injected at build time, so if you make changes, you will have to run the build command again to see them reflected.
-
-##### Running Connector Acceptance Tests for a single connector in Docker with your local CDK installed
-
-_Pre-requisite: Install the [`airbyte-ci` CLI](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md)_
-
-To run acceptance tests for a single connectors using the local CDK, from the connector directory, run
-
-```bash
-airbyte-ci connectors --use-local-cdk --name=<CONNECTOR> test
-```
-
-#### When you don't have access to the API
-
-There can be some time where you do not have access to the API (either because you don't have the credentials, network access, etc...) You will probably still want to do end-to-end testing at least once. In order to do so, you can emulate the server you would be reaching using a server stubbing tool.
-
-For example, using [mockserver](https://www.mock-server.com/), you can set up an expectation file like this:
-
-```json
-{
-  "httpRequest": {
-    "method": "GET",
-    "path": "/data"
-  },
-  "httpResponse": {
-    "body": "{\"data\": [{\"record_key\": 1}, {\"record_key\": 2}]}"
-  }
-}
-```
-
-Assuming this file has been created at `secrets/mock_server_config/expectations.json`, running the following command will allow to match any requests on path `/data` to return the response defined in the expectation file:
-
-```bash
-docker run -d --rm -v $(pwd)/secrets/mock_server_config:/config -p 8113:8113 --env MOCKSERVER_LOG_LEVEL=TRACE --env MOCKSERVER_SERVER_PORT=8113 --env MOCKSERVER_WATCH_INITIALIZATION_JSON=true --env MOCKSERVER_PERSISTED_EXPECTATIONS_PATH=/config/expectations.json --env MOCKSERVER_INITIALIZATION_JSON_PATH=/config/expectations.json mockserver/mockserver:5.15.0
-```
-
-HTTP requests to `localhost:8113/data` should now return the body defined in the expectations file. To test this, the implementer either has to change the code which defines the base URL for Python source or update the `url_base` from low-code. With the Connector Builder running in docker, you will have to use domain `host.docker.internal` instead of `localhost` as the requests are executed within docker.
-
-#### Publishing a new version to PyPi
-
-1. Open a PR
-2. Once it is approved and **merged**, an Airbyte member must run the `Publish CDK Manually` workflow from master using `release-type=major|manor|patch` and setting the changelog message.