airbyte-cdk 6.8.2.dev1__tar.gz → 6.8.3rc1__tar.gz

airbyte_cdk-6.8.3rc1/PKG-INFO

@@ -0,0 +1,306 @@
+Metadata-Version: 2.1
+Name: airbyte-cdk
+Version: 6.8.3rc1
+Summary: A framework for writing Airbyte Connectors.
+Home-page: https://airbyte.com
+License: MIT
+Keywords: airbyte,connector-development-kit,cdk
+Author: Airbyte
+Author-email: contact@airbyte.io
+Requires-Python: >=3.10,<3.13
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Provides-Extra: file-based
+Provides-Extra: sphinx-docs
+Provides-Extra: sql
+Provides-Extra: vector-db-based
+Requires-Dist: Jinja2 (>=3.1.2,<3.2.0)
+Requires-Dist: PyYAML (>=6.0.1,<7.0.0)
+Requires-Dist: Sphinx (>=4.2,<4.3) ; extra == "sphinx-docs"
+Requires-Dist: airbyte-protocol-models-dataclasses (>=0.14,<0.15)
+Requires-Dist: avro (>=1.11.2,<1.12.0) ; extra == "file-based"
+Requires-Dist: backoff
+Requires-Dist: cachetools
+Requires-Dist: cohere (==4.21) ; extra == "vector-db-based"
+Requires-Dist: cryptography (>=42.0.5,<44.0.0)
+Requires-Dist: dpath (>=2.1.6,<3.0.0)
+Requires-Dist: dunamai (>=1.22.0,<2.0.0)
+Requires-Dist: fastavro (>=1.8.0,<1.9.0) ; extra == "file-based"
+Requires-Dist: genson (==1.3.0)
+Requires-Dist: isodate (>=0.6.1,<0.7.0)
+Requires-Dist: jsonref (>=0.2,<0.3)
+Requires-Dist: jsonschema (>=4.17.3,<4.18.0)
+Requires-Dist: langchain (==0.1.16) ; extra == "vector-db-based"
+Requires-Dist: langchain_core (==0.1.42)
+Requires-Dist: markdown ; extra == "file-based"
+Requires-Dist: nltk (==3.9.1)
+Requires-Dist: numpy (<2)
+Requires-Dist: openai[embeddings] (==0.27.9) ; extra == "vector-db-based"
+Requires-Dist: orjson (>=3.10.7,<4.0.0)
+Requires-Dist: pandas (==2.2.2)
+Requires-Dist: pdf2image (==1.16.3) ; extra == "file-based"
+Requires-Dist: pdfminer.six (==20221105) ; extra == "file-based"
+Requires-Dist: pendulum (<3.0.0)
+Requires-Dist: psutil (==6.1.0)
+Requires-Dist: pyarrow (>=15.0.0,<15.1.0) ; extra == "file-based"
+Requires-Dist: pydantic (>=2.7,<3.0)
+Requires-Dist: pyjwt (>=2.8.0,<3.0.0)
+Requires-Dist: pyrate-limiter (>=3.1.0,<3.2.0)
+Requires-Dist: pytesseract (==0.3.10) ; extra == "file-based"
+Requires-Dist: python-calamine (==0.2.3) ; extra == "file-based"
+Requires-Dist: python-dateutil
+Requires-Dist: python-snappy (==0.7.3) ; extra == "file-based"
+Requires-Dist: python-ulid (>=3.0.0,<4.0.0)
+Requires-Dist: pytz (==2024.1)
+Requires-Dist: rapidfuzz (>=3.10.1,<4.0.0)
+Requires-Dist: requests
+Requires-Dist: requests_cache
+Requires-Dist: serpyco-rs (>=1.10.2,<2.0.0)
+Requires-Dist: sphinx-rtd-theme (>=1.0,<1.1) ; extra == "sphinx-docs"
+Requires-Dist: sqlalchemy (>=2.0,<3.0,!=2.0.36) ; extra == "sql"
+Requires-Dist: tiktoken (==0.8.0) ; extra == "vector-db-based"
+Requires-Dist: unstructured.pytesseract (>=0.3.12) ; extra == "file-based"
+Requires-Dist: unstructured[docx,pptx] (==0.10.27) ; extra == "file-based"
+Requires-Dist: wcmatch (==10.0)
+Requires-Dist: xmltodict (>=0.13.0,<0.14.0)
+Project-URL: Documentation, https://docs.airbyte.io/
+Project-URL: Repository, https://github.com/airbytehq/airbyte-python-cdk
+Description-Content-Type: text/markdown
+
+# Airbyte Python CDK and Low-Code CDK
+
+Airbyte Python CDK is a framework for building Airbyte API Source Connectors. It provides a set of
+classes and helpers that make it easy to build a connector against an HTTP API (REST, GraphQL, etc),
+or a generic Python source connector.
+
+## Usage
+
+If you're looking to build a connector, we highly recommend that you
+[start with the Connector Builder](https://docs.airbyte.com/connector-development/connector-builder-ui/overview).
+It should be enough for 90% connectors out there. For more flexible and complex connectors, use the
+[low-code CDK and `SourceDeclarativeManifest`](https://docs.airbyte.com/connector-development/config-based/low-code-cdk-overview).
+
+If that doesn't work, then consider building on top of the
+[lower-level Python CDK itself](https://docs.airbyte.com/connector-development/cdk-python/).
+
+### Quick Start
+
+To get started on a Python CDK based connector or a low-code connector, you can generate a connector
+project from a template:
+
+```bash
+# from the repo root
+cd airbyte-integrations/connector-templates/generator
+./generate.sh
+```
+
+### Example Connectors
+
+**HTTP Connectors**:
+
+- [Stripe](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-stripe/)
+- [Salesforce](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-salesforce/)
+
+**Python connectors using the bare-bones `Source` abstraction**:
+
+- [Google Sheets](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-google-sheets/google_sheets_source/google_sheets_source.py)
+
+This will generate a project with a type and a name of your choice and put it in
+`airbyte-integrations/connectors`. Open the directory with your connector in an editor and follow
+the `TODO` items.
+
+## Python CDK Overview
+
+Airbyte CDK code is within `airbyte_cdk` directory. Here's a high level overview of what's inside:
+
+- `connector_builder`. Internal wrapper that helps the Connector Builder platform run a declarative
+  manifest (low-code connector). You should not use this code directly. If you need to run a
+  `SourceDeclarativeManifest`, take a look at
+  [`source-declarative-manifest`](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-declarative-manifest)
+  connector implementation instead.
+- `destinations`. Basic Destination connector support! If you're building a Destination connector in
+  Python, try that. Some of our vector DB destinations like `destination-pinecone` are using that
+  code.
+- `models` expose `airbyte_protocol.models` as a part of `airbyte_cdk` package.
+- `sources/concurrent_source` is the Concurrent CDK implementation. It supports reading data from
+  streams concurrently per slice / partition, useful for connectors with high throughput and high
+  number of records.
+- `sources/declarative` is the low-code CDK. It works on top of Airbyte Python CDK, but provides a
+  declarative manifest language to define streams, operations, etc. This makes it easier to build
+  connectors without writing Python code.
+- `sources/file_based` is the CDK for file-based sources. Examples include S3, Azure, GCS, etc.
+
+## Contributing
+
+Thank you for being interested in contributing to Airbyte Python CDK! Here are some guidelines to
+get you started:
+
+- We adhere to the [code of conduct](/CODE_OF_CONDUCT.md).
+- You can contribute by reporting bugs, posting github discussions, opening issues, improving
+  [documentation](/docs/), and submitting pull requests with bugfixes and new features alike.
+- If you're changing the code, please add unit tests for your change.
+- When submitting issues or PRs, please add a small reproduction project. Using the changes in your
+  connector and providing that connector code as an example (or a satellite PR) helps!
+
+### First time setup
+
+Install the project dependencies and development tools:
+
+```bash
+poetry install --all-extras
+```
+
+Installing all extras is required to run the full suite of unit tests.
+
+#### Running tests locally
+
+- Iterate on the CDK code locally
+- Run tests via `poetry run poe unit-test-with-cov`, or `python -m pytest -s unit_tests` if you want
+  to pass pytest options.
+- Run `poetry run poe check-local` to lint all code, type-check modified code, and run unit tests
+  with coverage in one command.
+
+To see all available scripts, run `poetry run poe`.
+
+#### Formatting the code
+
+- Iterate on the CDK code locally
+- Run `poetry run ruff format` to format your changes.
+
+To see all available `ruff` options, run `poetry run ruff`.
+
+##### Autogenerated files
+
+Low-code CDK models are generated from `sources/declarative/declarative_component_schema.yaml`. If
+the iteration you are working on includes changes to the models or the connector generator, you
+might want to regenerate them. In order to do that, you can run:
+
+```bash
+poetry run poe build
+```
+
+This will generate the code generator docker image and the component manifest files based on the
+schemas and templates.
+
+#### Testing
+
+All tests are located in the `unit_tests` directory. Run `poetry run poe unit-test-with-cov` to run
+them. This also presents a test coverage report. For faster iteration with no coverage report and
+more options, `python -m pytest -s unit_tests` is a good place to start.
+
+#### Building and testing a connector with your local CDK
+
+When developing a new feature in the CDK, you may find it helpful to run a connector that uses that
+new feature. You can test this in one of two ways:
+
+- Running a connector locally
+- Building and running a source via Docker
+
+##### Installing your local CDK into a local Python connector
+
+Open the connector's `pyproject.toml` file and replace the line with `airbyte_cdk` with the
+following:
+
+```toml
+airbyte_cdk = { path = "../../../airbyte-cdk/python/airbyte_cdk", develop = true }
+```
+
+Then, running `poetry update` should reinstall `airbyte_cdk` from your local working directory.
+
+##### Building a Python connector in Docker with your local CDK installed
+
+_Pre-requisite: Install the
+[`airbyte-ci` CLI](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md)_
+
+You can build your connector image with the local CDK using
+
+```bash
+# from the airbytehq/airbyte base directory
+airbyte-ci connectors --use-local-cdk --name=<CONNECTOR> build
+```
+
+Note that the local CDK is injected at build time, so if you make changes, you will have to run the
+build command again to see them reflected.
+
+##### Running Connector Acceptance Tests for a single connector in Docker with your local CDK installed
+
+_Pre-requisite: Install the
+[`airbyte-ci` CLI](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md)_
+
+To run acceptance tests for a single connectors using the local CDK, from the connector directory,
+run
+
+```bash
+airbyte-ci connectors --use-local-cdk --name=<CONNECTOR> test
+```
+
+#### When you don't have access to the API
+
+There may be a time when you do not have access to the API (either because you don't have the
+credentials, network access, etc...) You will probably still want to do end-to-end testing at least
+once. In order to do so, you can emulate the server you would be reaching using a server stubbing
+tool.
+
+For example, using [mockserver](https://www.mock-server.com/), you can set up an expectation file
+like this:
+
+```json
+{
+  "httpRequest": {
+    "method": "GET",
+    "path": "/data"
+  },
+  "httpResponse": {
+    "body": "{\"data\": [{\"record_key\": 1}, {\"record_key\": 2}]}"
+  }
+}
+```
+
+Assuming this file has been created at `secrets/mock_server_config/expectations.json`, running the
+following command will allow to match any requests on path `/data` to return the response defined in
+the expectation file:
+
+```bash
+docker run -d --rm -v $(pwd)/secrets/mock_server_config:/config -p 8113:8113 --env MOCKSERVER_LOG_LEVEL=TRACE --env MOCKSERVER_SERVER_PORT=8113 --env MOCKSERVER_WATCH_INITIALIZATION_JSON=true --env MOCKSERVER_PERSISTED_EXPECTATIONS_PATH=/config/expectations.json --env MOCKSERVER_INITIALIZATION_JSON_PATH=/config/expectations.json mockserver/mockserver:5.15.0
+```
+
+HTTP requests to `localhost:8113/data` should now return the body defined in the expectations file.
+To test this, the implementer either has to change the code which defines the base URL for Python
+source or update the `url_base` from low-code. With the Connector Builder running in docker, you
+will have to use domain `host.docker.internal` instead of `localhost` as the requests are executed
+within docker.
+
+#### Publishing a new version to PyPi
+
+Python CDK has a
+[GitHub workflow](https://github.com/airbytehq/airbyte/actions/workflows/publish-cdk-command-manually.yml)
+that manages the CDK changelog, making a new release for `airbyte_cdk`, publishing it to PyPI, and
+then making a commit to update (and subsequently auto-release)
+[`source-declarative-manifest`](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-declarative-manifest)
+and Connector Builder (in the platform repository).
+
+> [!Note]: The workflow will handle the `CHANGELOG.md` entry for you. You should not add changelog
+> lines in your PRs to the CDK itself.
+
+> [!Warning]: The workflow bumps version on it's own, please don't change the CDK version in
+> `pyproject.toml` manually.
+
+1. You only trigger the release workflow once all the PRs that you want to be included are already
+   merged into the `master` branch.
+2. The
+   [`Publish CDK Manually`](https://github.com/airbytehq/airbyte/actions/workflows/publish-cdk-command-manually.yml)
+   workflow from master using `release-type=major|manor|patch` and setting the changelog message.
+3. When the workflow runs, it will commit a new version directly to master branch.
+4. The workflow will bump the version of `source-declarative-manifest` according to the
+   `release-type` of the CDK, then commit these changes back to master. The commit to master will
+   kick off a publish of the new version of `source-declarative-manifest`.
+5. The workflow will also add a pull request to `airbyte-platform-internal` repo to bump the
+   dependency in Connector Builder.
+

airbyte_cdk-6.8.3rc1/README.md

@@ -0,0 +1,229 @@
+# Airbyte Python CDK and Low-Code CDK
+
+Airbyte Python CDK is a framework for building Airbyte API Source Connectors. It provides a set of
+classes and helpers that make it easy to build a connector against an HTTP API (REST, GraphQL, etc),
+or a generic Python source connector.
+
+## Usage
+
+If you're looking to build a connector, we highly recommend that you
+[start with the Connector Builder](https://docs.airbyte.com/connector-development/connector-builder-ui/overview).
+It should be enough for 90% connectors out there. For more flexible and complex connectors, use the
+[low-code CDK and `SourceDeclarativeManifest`](https://docs.airbyte.com/connector-development/config-based/low-code-cdk-overview).
+
+If that doesn't work, then consider building on top of the
+[lower-level Python CDK itself](https://docs.airbyte.com/connector-development/cdk-python/).
+
+### Quick Start
+
+To get started on a Python CDK based connector or a low-code connector, you can generate a connector
+project from a template:
+
+```bash
+# from the repo root
+cd airbyte-integrations/connector-templates/generator
+./generate.sh
+```
+
+### Example Connectors
+
+**HTTP Connectors**:
+
+- [Stripe](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-stripe/)
+- [Salesforce](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-salesforce/)
+
+**Python connectors using the bare-bones `Source` abstraction**:
+
+- [Google Sheets](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-google-sheets/google_sheets_source/google_sheets_source.py)
+
+This will generate a project with a type and a name of your choice and put it in
+`airbyte-integrations/connectors`. Open the directory with your connector in an editor and follow
+the `TODO` items.
+
+## Python CDK Overview
+
+Airbyte CDK code is within `airbyte_cdk` directory. Here's a high level overview of what's inside:
+
+- `connector_builder`. Internal wrapper that helps the Connector Builder platform run a declarative
+  manifest (low-code connector). You should not use this code directly. If you need to run a
+  `SourceDeclarativeManifest`, take a look at
+  [`source-declarative-manifest`](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-declarative-manifest)
+  connector implementation instead.
+- `destinations`. Basic Destination connector support! If you're building a Destination connector in
+  Python, try that. Some of our vector DB destinations like `destination-pinecone` are using that
+  code.
+- `models` expose `airbyte_protocol.models` as a part of `airbyte_cdk` package.
+- `sources/concurrent_source` is the Concurrent CDK implementation. It supports reading data from
+  streams concurrently per slice / partition, useful for connectors with high throughput and high
+  number of records.
+- `sources/declarative` is the low-code CDK. It works on top of Airbyte Python CDK, but provides a
+  declarative manifest language to define streams, operations, etc. This makes it easier to build
+  connectors without writing Python code.
+- `sources/file_based` is the CDK for file-based sources. Examples include S3, Azure, GCS, etc.
+
+## Contributing
+
+Thank you for being interested in contributing to Airbyte Python CDK! Here are some guidelines to
+get you started:
+
+- We adhere to the [code of conduct](/CODE_OF_CONDUCT.md).
+- You can contribute by reporting bugs, posting github discussions, opening issues, improving
+  [documentation](/docs/), and submitting pull requests with bugfixes and new features alike.
+- If you're changing the code, please add unit tests for your change.
+- When submitting issues or PRs, please add a small reproduction project. Using the changes in your
+  connector and providing that connector code as an example (or a satellite PR) helps!
+
+### First time setup
+
+Install the project dependencies and development tools:
+
+```bash
+poetry install --all-extras
+```
+
+Installing all extras is required to run the full suite of unit tests.
+
+#### Running tests locally
+
+- Iterate on the CDK code locally
+- Run tests via `poetry run poe unit-test-with-cov`, or `python -m pytest -s unit_tests` if you want
+  to pass pytest options.
+- Run `poetry run poe check-local` to lint all code, type-check modified code, and run unit tests
+  with coverage in one command.
+
+To see all available scripts, run `poetry run poe`.
+
+#### Formatting the code
+
+- Iterate on the CDK code locally
+- Run `poetry run ruff format` to format your changes.
+
+To see all available `ruff` options, run `poetry run ruff`.
+
+##### Autogenerated files
+
+Low-code CDK models are generated from `sources/declarative/declarative_component_schema.yaml`. If
+the iteration you are working on includes changes to the models or the connector generator, you
+might want to regenerate them. In order to do that, you can run:
+
+```bash
+poetry run poe build
+```
+
+This will generate the code generator docker image and the component manifest files based on the
+schemas and templates.
+
+#### Testing
+
+All tests are located in the `unit_tests` directory. Run `poetry run poe unit-test-with-cov` to run
+them. This also presents a test coverage report. For faster iteration with no coverage report and
+more options, `python -m pytest -s unit_tests` is a good place to start.
+
+#### Building and testing a connector with your local CDK
+
+When developing a new feature in the CDK, you may find it helpful to run a connector that uses that
+new feature. You can test this in one of two ways:
+
+- Running a connector locally
+- Building and running a source via Docker
+
+##### Installing your local CDK into a local Python connector
+
+Open the connector's `pyproject.toml` file and replace the line with `airbyte_cdk` with the
+following:
+
+```toml
+airbyte_cdk = { path = "../../../airbyte-cdk/python/airbyte_cdk", develop = true }
+```
+
+Then, running `poetry update` should reinstall `airbyte_cdk` from your local working directory.
+
+##### Building a Python connector in Docker with your local CDK installed
+
+_Pre-requisite: Install the
+[`airbyte-ci` CLI](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md)_
+
+You can build your connector image with the local CDK using
+
+```bash
+# from the airbytehq/airbyte base directory
+airbyte-ci connectors --use-local-cdk --name=<CONNECTOR> build
+```
+
+Note that the local CDK is injected at build time, so if you make changes, you will have to run the
+build command again to see them reflected.
+
+##### Running Connector Acceptance Tests for a single connector in Docker with your local CDK installed
+
+_Pre-requisite: Install the
+[`airbyte-ci` CLI](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md)_
+
+To run acceptance tests for a single connectors using the local CDK, from the connector directory,
+run
+
+```bash
+airbyte-ci connectors --use-local-cdk --name=<CONNECTOR> test
+```
+
+#### When you don't have access to the API
+
+There may be a time when you do not have access to the API (either because you don't have the
+credentials, network access, etc...) You will probably still want to do end-to-end testing at least
+once. In order to do so, you can emulate the server you would be reaching using a server stubbing
+tool.
+
+For example, using [mockserver](https://www.mock-server.com/), you can set up an expectation file
+like this:
+
+```json
+{
+  "httpRequest": {
+    "method": "GET",
+    "path": "/data"
+  },
+  "httpResponse": {
+    "body": "{\"data\": [{\"record_key\": 1}, {\"record_key\": 2}]}"
+  }
+}
+```
+
+Assuming this file has been created at `secrets/mock_server_config/expectations.json`, running the
+following command will allow to match any requests on path `/data` to return the response defined in
+the expectation file:
+
+```bash
+docker run -d --rm -v $(pwd)/secrets/mock_server_config:/config -p 8113:8113 --env MOCKSERVER_LOG_LEVEL=TRACE --env MOCKSERVER_SERVER_PORT=8113 --env MOCKSERVER_WATCH_INITIALIZATION_JSON=true --env MOCKSERVER_PERSISTED_EXPECTATIONS_PATH=/config/expectations.json --env MOCKSERVER_INITIALIZATION_JSON_PATH=/config/expectations.json mockserver/mockserver:5.15.0
+```
+
+HTTP requests to `localhost:8113/data` should now return the body defined in the expectations file.
+To test this, the implementer either has to change the code which defines the base URL for Python
+source or update the `url_base` from low-code. With the Connector Builder running in docker, you
+will have to use domain `host.docker.internal` instead of `localhost` as the requests are executed
+within docker.
+
+#### Publishing a new version to PyPi
+
+Python CDK has a
+[GitHub workflow](https://github.com/airbytehq/airbyte/actions/workflows/publish-cdk-command-manually.yml)
+that manages the CDK changelog, making a new release for `airbyte_cdk`, publishing it to PyPI, and
+then making a commit to update (and subsequently auto-release)
+[`source-declarative-manifest`](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-declarative-manifest)
+and Connector Builder (in the platform repository).
+
+> [!Note]: The workflow will handle the `CHANGELOG.md` entry for you. You should not add changelog
+> lines in your PRs to the CDK itself.
+
+> [!Warning]: The workflow bumps version on it's own, please don't change the CDK version in
+> `pyproject.toml` manually.
+
+1. You only trigger the release workflow once all the PRs that you want to be included are already
+   merged into the `master` branch.
+2. The
+   [`Publish CDK Manually`](https://github.com/airbytehq/airbyte/actions/workflows/publish-cdk-command-manually.yml)
+   workflow from master using `release-type=major|manor|patch` and setting the changelog message.
+3. When the workflow runs, it will commit a new version directly to master branch.
+4. The workflow will bump the version of `source-declarative-manifest` according to the
+   `release-type` of the CDK, then commit these changes back to master. The commit to master will
+   kick off a publish of the new version of `source-declarative-manifest`.
+5. The workflow will also add a pull request to `airbyte-platform-internal` repo to bump the
+   dependency in Connector Builder.

{airbyte_cdk-6.8.2.dev1 → airbyte_cdk-6.8.3rc1}/airbyte_cdk/sources/declarative/concurrent_declarative_source.py

@@ -20,9 +20,6 @@ from airbyte_cdk.sources.declarative.extractors.record_filter import (
     ClientSideIncrementalRecordFilterDecorator,
 )
 from airbyte_cdk.sources.declarative.incremental.datetime_based_cursor import DatetimeBasedCursor
-from airbyte_cdk.sources.declarative.incremental.per_partition_with_global import (
-    PerPartitionWithGlobalCursor,
-)
 from airbyte_cdk.sources.declarative.interpolation import InterpolatedString
 from airbyte_cdk.sources.declarative.manifest_declarative_source import ManifestDeclarativeSource
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
@@ -309,59 +306,6 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
                             cursor=final_state_cursor,
                         )
                     )
-                elif (
-                    incremental_sync_component_definition
-                    and incremental_sync_component_definition.get("type", "")
-                    == DatetimeBasedCursorModel.__name__
-                    and self._stream_supports_concurrent_partition_processing(
-                        declarative_stream=declarative_stream
-                    )
-                    and hasattr(declarative_stream.retriever, "stream_slicer")
-                    and isinstance(declarative_stream.retriever.stream_slicer, PerPartitionWithGlobalCursor)
-                ):
-                    stream_state = state_manager.get_stream_state(
-                        stream_name=declarative_stream.name, namespace=declarative_stream.namespace
-                    )
-                    partition_router = declarative_stream.retriever.stream_slicer._partition_router
-
-                    cursor = self._constructor.create_concurrent_cursor_from_perpartition_cursor(
-                            state_manager=state_manager,
-                            model_type=DatetimeBasedCursorModel,
-                            component_definition=incremental_sync_component_definition,
-                            stream_name=declarative_stream.name,
-                            stream_namespace=declarative_stream.namespace,
-                            config=config or {},
-                            stream_state=stream_state,
-                            partition_router=partition_router,
-                        )
-
-
-                    partition_generator = StreamSlicerPartitionGenerator(
-                        DeclarativePartitionFactory(
-                            declarative_stream.name,
-                            declarative_stream.get_json_schema(),
-                            self._retriever_factory(
-                                name_to_stream_mapping[declarative_stream.name],
-                                config,
-                                stream_state,
-                            ),
-                            self.message_repository,
-                        ),
-                        cursor,
-                    )
-
-                    concurrent_streams.append(
-                        DefaultStream(
-                            partition_generator=partition_generator,
-                            name=declarative_stream.name,
-                            json_schema=declarative_stream.get_json_schema(),
-                            availability_strategy=AlwaysAvailableAvailabilityStrategy(),
-                            primary_key=get_primary_key_from_stream(declarative_stream.primary_key),
-                            cursor_field=cursor.cursor_field.cursor_field_key,
-                            logger=self.logger,
-                            cursor=cursor,
-                        )
-                    )
                 else:
                     synchronous_streams.append(declarative_stream)
             else:

{airbyte_cdk-6.8.2.dev1 → airbyte_cdk-6.8.3rc1}/airbyte_cdk/sources/declarative/extractors/record_filter.py

@@ -59,11 +59,13 @@ class ClientSideIncrementalRecordFilterDecorator(RecordFilter):
 
     def __init__(
         self,
-        cursor: Union[DatetimeBasedCursor, PerPartitionWithGlobalCursor, GlobalSubstreamCursor],
+        date_time_based_cursor: DatetimeBasedCursor,
+        substream_cursor: Optional[Union[PerPartitionWithGlobalCursor, GlobalSubstreamCursor]],
         **kwargs: Any,
     ):
         super().__init__(**kwargs)
-        self._cursor = cursor
+        self._date_time_based_cursor = date_time_based_cursor
+        self._substream_cursor = substream_cursor
 
     def filter_records(
         self,
@@ -75,7 +77,7 @@ class ClientSideIncrementalRecordFilterDecorator(RecordFilter):
         records = (
             record
             for record in records
-            if self._cursor.should_be_synced(
+            if (self._substream_cursor or self._date_time_based_cursor).should_be_synced(
                 # Record is created on the fly to align with cursors interface; stream name is ignored as we don't need it here
                 # Record stream name is empty cause it is not used durig the filtering
                 Record(data=record, associated_slice=stream_slice, stream_name="")

{airbyte_cdk-6.8.2.dev1 → airbyte_cdk-6.8.3rc1}/airbyte_cdk/sources/declarative/incremental/__init__.py

@@ -2,7 +2,6 @@
 # Copyright (c) 2022 Airbyte, Inc., all rights reserved.
 #
 
-from airbyte_cdk.sources.declarative.incremental.concurrent_partition_cursor import ConcurrentCursorFactory, ConcurrentPerPartitionCursor
 from airbyte_cdk.sources.declarative.incremental.datetime_based_cursor import DatetimeBasedCursor
 from airbyte_cdk.sources.declarative.incremental.declarative_cursor import DeclarativeCursor
 from airbyte_cdk.sources.declarative.incremental.global_substream_cursor import GlobalSubstreamCursor
@@ -15,8 +14,6 @@ from airbyte_cdk.sources.declarative.incremental.resumable_full_refresh_cursor i
 
 __all__ = [
     "CursorFactory",
-    "ConcurrentCursorFactory"
-    "ConcurrentPerPartitionCursor",
     "DatetimeBasedCursor",
     "DeclarativeCursor",
     "GlobalSubstreamCursor",

{airbyte_cdk-6.8.2.dev1 → airbyte_cdk-6.8.3rc1}/airbyte_cdk/sources/declarative/incremental/per_partition_cursor.py

@@ -303,15 +303,6 @@ class PerPartitionCursor(DeclarativeCursor):
             raise ValueError("A partition needs to be provided in order to get request body json")
 
     def should_be_synced(self, record: Record) -> bool:
-        if self._to_partition_key(record.associated_slice.partition) not in self._cursor_per_partition:
-            partition_state = (
-                self._state_to_migrate_from
-                if self._state_to_migrate_from
-                else self._NO_CURSOR_STATE
-            )
-            cursor = self._create_cursor(partition_state)
-
-            self._cursor_per_partition[self._to_partition_key(record.associated_slice.partition)] = cursor
         return self._get_cursor(record).should_be_synced(
             self._convert_record_to_cursor_record(record)
         )