confluent-sql 0.6.0__tar.gz → 0.8.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- confluent_sql-0.8.0/.github/CODEOWNERS +1 -0
- {confluent_sql-0.6.0 → confluent_sql-0.8.0}/CHANGELOG.md +36 -1
- {confluent_sql-0.6.0 → confluent_sql-0.8.0}/CLAUDE.md +12 -0
- {confluent_sql-0.6.0 → confluent_sql-0.8.0}/DBAPI_EXTENSIONS.md +86 -1
- {confluent_sql-0.6.0 → confluent_sql-0.8.0}/PKG-INFO +50 -3
- {confluent_sql-0.6.0 → confluent_sql-0.8.0}/README.md +48 -2
- confluent_sql-0.8.0/examples/artifact_udf/pyproject.toml +12 -0
- confluent_sql-0.8.0/examples/artifact_udf/src/example_udf/__init__.py +2 -0
- confluent_sql-0.8.0/examples/artifact_udf/src/example_udf/py.typed +0 -0
- confluent_sql-0.8.0/examples/artifact_udf/src/example_udf/tshirt_sizing.py +54 -0
- confluent_sql-0.8.0/examples/artifact_upload_example.py +107 -0
- confluent_sql-0.8.0/examples/interactive_oauth_login.py +43 -0
- {confluent_sql-0.6.0 → confluent_sql-0.8.0}/pyproject.toml +14 -1
- {confluent_sql-0.6.0 → confluent_sql-0.8.0}/src/confluent_sql/__init__.py +30 -0
- {confluent_sql-0.6.0 → confluent_sql-0.8.0}/src/confluent_sql/__version__.py +1 -1
- confluent_sql-0.8.0/src/confluent_sql/artifacts.py +702 -0
- {confluent_sql-0.6.0 → confluent_sql-0.8.0}/src/confluent_sql/auth.py +4 -1
- {confluent_sql-0.6.0 → confluent_sql-0.8.0}/src/confluent_sql/connection.py +716 -216
- {confluent_sql-0.6.0 → confluent_sql-0.8.0}/src/confluent_sql/connectors.py +15 -18
- {confluent_sql-0.6.0 → confluent_sql-0.8.0}/src/confluent_sql/cursor.py +14 -4
- {confluent_sql-0.6.0 → confluent_sql-0.8.0}/src/confluent_sql/exceptions.py +142 -0
- confluent_sql-0.8.0/src/confluent_sql/oauth/__init__.py +56 -0
- confluent_sql-0.8.0/src/confluent_sql/oauth/callback_server.py +533 -0
- confluent_sql-0.8.0/src/confluent_sql/oauth/config.py +63 -0
- confluent_sql-0.8.0/src/confluent_sql/oauth/holder.py +398 -0
- confluent_sql-0.8.0/src/confluent_sql/oauth/http_json.py +108 -0
- confluent_sql-0.8.0/src/confluent_sql/oauth/pkce.py +38 -0
- confluent_sql-0.8.0/src/confluent_sql/oauth/provider.py +1015 -0
- confluent_sql-0.8.0/src/confluent_sql/oauth/token_chain.py +360 -0
- confluent_sql-0.8.0/src/confluent_sql/oauth/token_set.py +56 -0
- {confluent_sql-0.6.0 → confluent_sql-0.8.0}/src/confluent_sql/retry.py +3 -3
- confluent_sql-0.8.0/src/confluent_sql/service_api.py +33 -0
- {confluent_sql-0.6.0 → confluent_sql-0.8.0}/src/confluent_sql/statement.py +13 -6
- {confluent_sql-0.6.0 → confluent_sql-0.8.0}/src/confluent_sql/tableflow.py +2 -5
- confluent_sql-0.8.0/src/confluent_sql/utils.py +90 -0
- {confluent_sql-0.6.0 → confluent_sql-0.8.0}/tests/integration/conftest.py +52 -11
- confluent_sql-0.8.0/tests/integration/test_artifact.py +236 -0
- {confluent_sql-0.6.0 → confluent_sql-0.8.0}/tests/integration/test_connection.py +64 -1
- {confluent_sql-0.6.0 → confluent_sql-0.8.0}/tests/integration/test_connector.py +2 -2
- {confluent_sql-0.6.0 → confluent_sql-0.8.0}/tests/integration/test_cursor.py +45 -18
- {confluent_sql-0.6.0 → confluent_sql-0.8.0}/tests/integration/test_fetch.py +13 -4
- confluent_sql-0.8.0/tests/integration/test_oauth.py +110 -0
- confluent_sql-0.8.0/tests/integration/test_oauth_rejected_org.py +48 -0
- confluent_sql-0.8.0/tests/integration/test_oauth_user_identity.py +79 -0
- confluent_sql-0.8.0/tests/integration/test_oauth_user_mismatch.py +91 -0
- {confluent_sql-0.6.0 → confluent_sql-0.8.0}/tests/integration/test_tableflow.py +3 -2
- {confluent_sql-0.6.0 → confluent_sql-0.8.0}/tests/unit/conftest.py +26 -1
- confluent_sql-0.8.0/tests/unit/oauth/test_callback_server_unit.py +762 -0
- confluent_sql-0.8.0/tests/unit/oauth/test_config_unit.py +75 -0
- confluent_sql-0.8.0/tests/unit/oauth/test_holder_unit.py +809 -0
- confluent_sql-0.8.0/tests/unit/oauth/test_http_json_unit.py +133 -0
- confluent_sql-0.8.0/tests/unit/oauth/test_pkce_unit.py +42 -0
- confluent_sql-0.8.0/tests/unit/oauth/test_provider_unit.py +1728 -0
- confluent_sql-0.8.0/tests/unit/oauth/test_token_chain_unit.py +610 -0
- confluent_sql-0.8.0/tests/unit/oauth/test_token_set_unit.py +108 -0
- confluent_sql-0.8.0/tests/unit/test_artifacts_connection_unit.py +231 -0
- confluent_sql-0.8.0/tests/unit/test_artifacts_unit.py +690 -0
- {confluent_sql-0.6.0 → confluent_sql-0.8.0}/tests/unit/test_connection_byoidc_unit.py +2 -0
- confluent_sql-0.8.0/tests/unit/test_connection_close_unit.py +144 -0
- {confluent_sql-0.6.0 → confluent_sql-0.8.0}/tests/unit/test_connection_connector_unit.py +1 -21
- confluent_sql-0.8.0/tests/unit/test_connection_credentials_unit.py +55 -0
- confluent_sql-0.8.0/tests/unit/test_connection_oauth_unit.py +563 -0
- {confluent_sql-0.6.0 → confluent_sql-0.8.0}/tests/unit/test_connection_unit.py +205 -7
- {confluent_sql-0.6.0 → confluent_sql-0.8.0}/tests/unit/test_connectors_connection_unit.py +18 -0
- {confluent_sql-0.6.0 → confluent_sql-0.8.0}/tests/unit/test_cursor_unit.py +22 -1
- confluent_sql-0.8.0/tests/unit/test_package_unit.py +18 -0
- {confluent_sql-0.6.0 → confluent_sql-0.8.0}/tests/unit/test_statement_unit.py +43 -1
- {confluent_sql-0.6.0 → confluent_sql-0.8.0}/tests/unit/test_tableflow_connection_unit.py +3 -24
- {confluent_sql-0.6.0 → confluent_sql-0.8.0}/tests/unit/test_types_unit.py +3 -1
- confluent_sql-0.8.0/tests/unit/test_utils_unit.py +35 -0
- confluent_sql-0.8.0/uv.lock +510 -0
- confluent_sql-0.6.0/.github/CODEOWNERS +0 -2
- confluent_sql-0.6.0/src/confluent_sql/utils.py +0 -35
- confluent_sql-0.6.0/tests/unit/test_package_unit.py +0 -10
- confluent_sql-0.6.0/uv.lock +0 -466
- {confluent_sql-0.6.0 → confluent_sql-0.8.0}/.claude/rules/software-engineering.md +0 -0
- {confluent_sql-0.6.0 → confluent_sql-0.8.0}/.gitignore +0 -0
- {confluent_sql-0.6.0 → confluent_sql-0.8.0}/.semaphore/publish_to_pypi.yml +0 -0
- {confluent_sql-0.6.0 → confluent_sql-0.8.0}/.semaphore/semaphore.yml +0 -0
- {confluent_sql-0.6.0 → confluent_sql-0.8.0}/ARCHITECTURE.md +0 -0
- {confluent_sql-0.6.0 → confluent_sql-0.8.0}/LICENSE.txt +0 -0
- {confluent_sql-0.6.0 → confluent_sql-0.8.0}/Makefile +0 -0
- {confluent_sql-0.6.0 → confluent_sql-0.8.0}/STREAMING.md +0 -0
- {confluent_sql-0.6.0 → confluent_sql-0.8.0}/TYPES.md +0 -0
- {confluent_sql-0.6.0 → confluent_sql-0.8.0}/examples/byoidc_bearer_token_example.py +0 -0
- {confluent_sql-0.6.0 → confluent_sql-0.8.0}/examples/errors.py +0 -0
- {confluent_sql-0.6.0 → confluent_sql-0.8.0}/examples/simple_append_only_streaming_query_example.py +0 -0
- {confluent_sql-0.6.0 → confluent_sql-0.8.0}/examples/snapshot_mode_tuple_cursor_simple_example.py +0 -0
- {confluent_sql-0.6.0 → confluent_sql-0.8.0}/examples/statement_properties_example.py +0 -0
- {confluent_sql-0.6.0 → confluent_sql-0.8.0}/examples/tableflow_gcs_example.py +0 -0
- {confluent_sql-0.6.0 → confluent_sql-0.8.0}/examples/tableflow_lifecycle_example.py +0 -0
- {confluent_sql-0.6.0 → confluent_sql-0.8.0}/service.yml +0 -0
- {confluent_sql-0.6.0 → confluent_sql-0.8.0}/src/confluent_sql/changelog_compressor.py +0 -0
- {confluent_sql-0.6.0 → confluent_sql-0.8.0}/src/confluent_sql/execution_mode.py +0 -0
- {confluent_sql-0.6.0 → confluent_sql-0.8.0}/src/confluent_sql/polling.py +0 -0
- {confluent_sql-0.6.0 → confluent_sql-0.8.0}/src/confluent_sql/result_readers.py +0 -0
- {confluent_sql-0.6.0 → confluent_sql-0.8.0}/src/confluent_sql/statement_properties.py +0 -0
- {confluent_sql-0.6.0 → confluent_sql-0.8.0}/src/confluent_sql/types.py +0 -0
- {confluent_sql-0.6.0 → confluent_sql-0.8.0}/src/confluent_sql/variant.py +0 -0
- {confluent_sql-0.6.0 → confluent_sql-0.8.0}/tests/__init__.py +0 -0
- {confluent_sql-0.6.0 → confluent_sql-0.8.0}/tests/conftest.py +0 -0
- {confluent_sql-0.6.0 → confluent_sql-0.8.0}/tests/unit/test_auth_unit.py +0 -0
- {confluent_sql-0.6.0 → confluent_sql-0.8.0}/tests/unit/test_changelog_compressor_unit.py +0 -0
- {confluent_sql-0.6.0 → confluent_sql-0.8.0}/tests/unit/test_changelog_unit.py +0 -0
- {confluent_sql-0.6.0 → confluent_sql-0.8.0}/tests/unit/test_connection_unit_properties.py +0 -0
- {confluent_sql-0.6.0 → confluent_sql-0.8.0}/tests/unit/test_connectors_unit.py +0 -0
- {confluent_sql-0.6.0 → confluent_sql-0.8.0}/tests/unit/test_execution_mode_unit.py +0 -0
- {confluent_sql-0.6.0 → confluent_sql-0.8.0}/tests/unit/test_polling_unit.py +0 -0
- {confluent_sql-0.6.0 → confluent_sql-0.8.0}/tests/unit/test_result_readers_unit.py +0 -0
- {confluent_sql-0.6.0 → confluent_sql-0.8.0}/tests/unit/test_retry_unit.py +0 -0
- {confluent_sql-0.6.0 → confluent_sql-0.8.0}/tests/unit/test_statement_properties_unit.py +0 -0
- {confluent_sql-0.6.0 → confluent_sql-0.8.0}/tests/unit/test_tableflow_unit.py +0 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
* @confluentinc/dbt-product @confluentinc/sql-clients
|
|
@@ -4,7 +4,42 @@ All notable changes to this dbapi driver will be documented in this file.
|
|
|
4
4
|
|
|
5
5
|
## Unreleased
|
|
6
6
|
|
|
7
|
-
## 0.
|
|
7
|
+
## 0.8.0, 2026-10-30
|
|
8
|
+
|
|
9
|
+
### Changed - Breaking
|
|
10
|
+
|
|
11
|
+
- `Connection.__init__`'s positional parameters are reordered: `organization_id` now comes after `cloud_provider`/`cloud_region`/`endpoint` (previously it was second, right after `environment_id`).
|
|
12
|
+
|
|
13
|
+
### Added
|
|
14
|
+
|
|
15
|
+
- Interactive browser login: `connect(auth="oauth", ...)` signs in once through your browser instead of requiring API keys, and the login is shared by every connection in the process. `organization_id` is optional. While the login is in progress the driver listens on the fixed loopback port `127.0.0.1:26642`, which must be free (otherwise `OAuthLoginError` with reason `PORT_IN_USE`). See the README and `examples/interactive_oauth_login.py` (#150).
|
|
16
|
+
- `confluent_sql.oauth_discard_login()`: discard the process's shared interactive OAuth login. Closing a `Connection` never does this; the next `connect(auth="oauth", ...)` opens the browser again. (#150)
|
|
17
|
+
- `connect(..., reauth="auto"|"raise")` (with `auth="oauth"` only): what to do once an OAuth session can no longer be refreshed. `"auto"` (default) opens the browser for a fresh login and retries the request; `"raise"` raises `ReauthenticationRequired` instead, for unattended runs. (#150)
|
|
18
|
+
- New exceptions, all subclasses of `OperationalError` and importable from `confluent_sql` (#150):
|
|
19
|
+
- `OAuthLoginError`: the interactive browser login could not complete. Its `reason` is an `OAuthLoginFailure`: `TIMED_OUT`, `PORT_IN_USE`, `AUTHORIZATION_DENIED`, `SERVER_ERROR`, or (on re-authentication) `USER_MISMATCH`.
|
|
20
|
+
- `OAuthTokenEndpointError`: the auth service's token endpoint rejected a request. Carries the OAuth `error_code` (e.g. `"invalid_grant"`) and `http_status_code`.
|
|
21
|
+
- `ReauthenticationRequired`: the session can only be recovered by a fresh login. Its `reason` is a `ReauthenticationReason`: `ABSOLUTE_EXPIRY` (the ~8h session limit) or `REFRESH_REJECTED`.
|
|
22
|
+
|
|
23
|
+
### Changed
|
|
24
|
+
|
|
25
|
+
- `organization_id` is now genuinely optional on both `connect()` and `Connection()` when it can be self-discovered (configured to use either oauth or a global API key).
|
|
26
|
+
- Direct `Connection()` construction's `organization_id` requirement (introduced in 0.6.0, #213) does not apply under `auth="oauth"`, which discovers the organization from the interactive login's session.
|
|
27
|
+
|
|
28
|
+
## 0.7.0, 2026-10-19
|
|
29
|
+
|
|
30
|
+
### Added
|
|
31
|
+
|
|
32
|
+
- `Connection.dry_run_statement(statement_text, *, mode=ExecutionMode.SNAPSHOT, compute_pool_id=None)`: validate a statement with Flink's `sql.dry-run` in a single request, without running or storing it, and return the resulting `Statement`. The returned `Statement`'s `schema` and `is_append_only` may then be of interest to the caller. Raises `OperationalError` with the server's detail if Flink rejects the statement. (#246)
|
|
33
|
+
- New property `Statement.is_dry_run`: whether the statement was submitted with the `sql.dry-run` statement property. (#246)
|
|
34
|
+
- Support for the Flink Artifact API (`/artifact/v1/flink-artifacts` and `/presigned-upload-url`), for uploading UDF packages (a JAR, or for Python a ZIP) to Confluent Cloud.
|
|
35
|
+
See `examples/artifact_upload_example.py` for a full example of packaging, uploading and calling a Python UDF artifact end to end.
|
|
36
|
+
Artifact calls authenticate with a global key, or a dedicated Cloud resource management key via the new `cloud_api_key`/`cloud_api_secret` parameters of `connect()`.
|
|
37
|
+
|
|
38
|
+
### Changed
|
|
39
|
+
|
|
40
|
+
- `Cursor.execute()` rejects `sql.dry-run` property being supplied as True with an `InterfaceError` pointing to use `Connection.dry_run_statement()` instead. Previously, valid SQL raised `StatementNotFoundError` (the cursor polled for a statement the server never stores) and invalid SQL raised `OperationalError`. (#246)
|
|
41
|
+
|
|
42
|
+
## 0.6.0, 2026-09-29
|
|
8
43
|
|
|
9
44
|
### Changed - Breaking
|
|
10
45
|
|
|
@@ -58,3 +58,15 @@ Exemplars: `Connection.stop_statement` / `Cursor.stop_statement` (`wait_for_stop
|
|
|
58
58
|
(`wait_for_removal=True`). The DDL convenience methods (`execute_snapshot_ddl`,
|
|
59
59
|
`execute_streaming_ddl`) block unconditionally and offer no opt-out — that's also acceptable when a
|
|
60
60
|
non-blocking variant would be meaningless.
|
|
61
|
+
|
|
62
|
+
## The `oauth` test marker
|
|
63
|
+
|
|
64
|
+
Every test module exercising the interactive OAuth login -- everything under `tests/unit/oauth/`,
|
|
65
|
+
`test_connection_oauth_unit.py`, and the `tests/integration/test_oauth*.py` modules -- carries
|
|
66
|
+
`pytest.mark.oauth` via module-level `pytestmark`, alongside its `unit`/`integration` marker. A new
|
|
67
|
+
OAuth test module must do the same. It is a topic axis orthogonal to unit/integration:
|
|
68
|
+
|
|
69
|
+
- `uv run pytest -m oauth` -- every OAuth test (integration ones skip without `RUN_OAUTH_TESTS`).
|
|
70
|
+
- `uv run pytest -m "oauth and unit"` -- the fast, browser-free subset.
|
|
71
|
+
- `RUN_OAUTH_TESTS=1 uv run pytest -n0 -m "oauth and integration"` -- the human-in-the-loop browser
|
|
72
|
+
run. `-n0` is mandatory: the loopback callback port is fixed, so parallel workers collide.
|
|
@@ -33,8 +33,9 @@ For comprehensive details on streaming queries, polling patterns, and changelog
|
|
|
33
33
|
|
|
34
34
|
- [Result Format Extensions](#result-format-extensions) - Dictionary rows, custom types
|
|
35
35
|
- [Streaming Query Support](#streaming-query-support) - Comprehensive streaming guide
|
|
36
|
-
- [Statement Lifecycle Management](#statement-lifecycle-management) - DDL, naming, stopping, deletion
|
|
36
|
+
- [Statement Lifecycle Management](#statement-lifecycle-management) - DDL, dry-run, naming, stopping, deletion
|
|
37
37
|
- [Tableflow Lifecycle](#tableflow-lifecycle) - Enable, read, and disable Iceberg/Delta sinks
|
|
38
|
+
- [Flink Artifacts](#flink-artifacts) - Upload and manage UDF packages
|
|
38
39
|
- [Introspection and Metadata](#introspection-and-metadata) - Properties for query state
|
|
39
40
|
- [Performance Monitoring](#performance-monitoring) - Fetch metrics
|
|
40
41
|
- [Type System Extensions](#type-system-extensions) - Flink type support
|
|
@@ -404,6 +405,36 @@ For more details on managing named and labeled statements, see the [Statement Na
|
|
|
404
405
|
|
|
405
406
|
---
|
|
406
407
|
|
|
408
|
+
### Dry-Run Validation (`dry_run_statement()`)
|
|
409
|
+
|
|
410
|
+
Validate a statement without running it, using Flink's `sql.dry-run`. Flink validates and plans
|
|
411
|
+
the statement and answers in the submission response, without storing it, so this is a single
|
|
412
|
+
request with nothing to clean up:
|
|
413
|
+
|
|
414
|
+
```python
|
|
415
|
+
statement = connection.dry_run_statement(
|
|
416
|
+
"SELECT id, price FROM orders", mode=ExecutionMode.STREAMING_QUERY
|
|
417
|
+
)
|
|
418
|
+
print(statement.schema) # the query's result columns; None for DDL
|
|
419
|
+
```
|
|
420
|
+
|
|
421
|
+
- `mode` (default `ExecutionMode.SNAPSHOT`) sets `sql.snapshot.mode` as a cursor in that mode
|
|
422
|
+
would. The result schema is the same in either mode, but the statement's traits are not:
|
|
423
|
+
`is_bounded` and `is_append_only` describe a run in the mode you validated in. To learn whether
|
|
424
|
+
a streaming submission would be append-only (e.g. an unbounded aggregation is not), pass
|
|
425
|
+
`ExecutionMode.STREAMING_QUERY`; under the default, a snapshot's bounded result always reports
|
|
426
|
+
append-only.
|
|
427
|
+
- Invalid SQL raises `OperationalError` with the server's detail.
|
|
428
|
+
- The returned `Statement` has `is_dry_run` true and `statement_id` `""`. It was never stored,
|
|
429
|
+
so `get_statement()` for it raises `StatementNotFoundError`.
|
|
430
|
+
- The statement text is sent as-is (no parameter interpolation).
|
|
431
|
+
|
|
432
|
+
`Cursor.execute()` does not accept property `sql.dry-run`: a dry-run has no rows to fetch and no
|
|
433
|
+
server-side statement to manage, so it doesn't fit the cursor lifecycle. Passing it as true raises
|
|
434
|
+
`InterfaceError`, pointing to use this `Connection`-level method instead.
|
|
435
|
+
|
|
436
|
+
---
|
|
437
|
+
|
|
407
438
|
### Statement Naming and Labeling
|
|
408
439
|
|
|
409
440
|
Control statement identity and grouping for tracking and management.
|
|
@@ -741,6 +772,59 @@ for table in ("orders", "shipments", "returns"):
|
|
|
741
772
|
|
|
742
773
|
---
|
|
743
774
|
|
|
775
|
+
## Flink Artifacts
|
|
776
|
+
|
|
777
|
+
Flink artifacts are how user code -- a UDF packaged as a JAR (Java) or a ZIP (Python) -- gets into
|
|
778
|
+
Confluent Cloud Flink. Upload one, then register a function from it with
|
|
779
|
+
`CREATE FUNCTION ... USING JAR 'confluent-artifact://<artifact id>'`. Artifacts are scoped to the
|
|
780
|
+
connection's cloud, region and environment (from `connect()`'s `cloud_provider`/`cloud_region`, or a
|
|
781
|
+
standard `https://flink.<region>.<cloud>.confluent.cloud` `endpoint`).
|
|
782
|
+
|
|
783
|
+
A runnable end-to-end example -- build a Python UDF, upload it, call it from SQL, clean up -- is in
|
|
784
|
+
[examples/artifact_upload_example.py](examples/artifact_upload_example.py).
|
|
785
|
+
|
|
786
|
+
> **Credentials.** Artifacts need a global API key, or a "Cloud resource management" key passed as
|
|
787
|
+
> `cloud_api_key`/`cloud_api_secret` (used only when no global key is given). A Tableflow or Flink
|
|
788
|
+
> key won't work, and neither will BYOIDC -- like Tableflow, artifacts are a control-plane surface.
|
|
789
|
+
|
|
790
|
+
The artifact operations live on an `ArtifactsApi` object, returned by
|
|
791
|
+
`Connection.get_artifacts_api()` (composed on first call, then cached):
|
|
792
|
+
|
|
793
|
+
```python
|
|
794
|
+
from confluent_sql import ArtifactRuntimeLanguage
|
|
795
|
+
|
|
796
|
+
artifacts = connection.get_artifacts_api()
|
|
797
|
+
|
|
798
|
+
artifact = artifacts.create(
|
|
799
|
+
"my-udf",
|
|
800
|
+
"dist/my_udf-0.1.0.zip", # path of the archive to upload
|
|
801
|
+
runtime_language=ArtifactRuntimeLanguage.PYTHON,
|
|
802
|
+
description="T-shirt size comparisons",
|
|
803
|
+
)
|
|
804
|
+
artifacts.get(artifact.id).versions # versions appear on a read, not on create/list
|
|
805
|
+
artifacts.list(runtime_language="PYTHON")
|
|
806
|
+
artifacts.delete(artifact.id) # blocks until gone; wait_for_removal=False to opt out
|
|
807
|
+
```
|
|
808
|
+
|
|
809
|
+
`create` requests a presigned URL, uploads the file to it and registers the artifact. It infers
|
|
810
|
+
`content_format` from the `.jar`/`.zip` extension when you don't pass one. Only JAVA with JAR and PYTHON
|
|
811
|
+
with ZIP are accepted; anything else raises `InterfaceError` before the upload. `runtime_language`
|
|
812
|
+
defaults to JAVA, so a ZIP needs `runtime_language=PYTHON`. The file is streamed
|
|
813
|
+
from disk.
|
|
814
|
+
|
|
815
|
+
Uploads retry failures to *connect* (nothing sent yet) three times by default, but not failures
|
|
816
|
+
mid-transfer, which would re-send the whole archive. Tune with an `UploadRetry` via
|
|
817
|
+
`create(upload_retry=...)`: `UploadRetry(connect_retries=3, transfer_retries=0)`.
|
|
818
|
+
|
|
819
|
+
The Cloud API doesn't currently support adding versions to an existing artifact: each artifact has
|
|
820
|
+
a single server-assigned version, and its `display_name` is immutable. To ship new code, create a
|
|
821
|
+
new artifact rather than updating.
|
|
822
|
+
|
|
823
|
+
Errors: `ArtifactNotFoundError` (404 on get/delete), `ArtifactAlreadyExistsError` (display
|
|
824
|
+
name taken). Python UDFs run only in streaming mode.
|
|
825
|
+
|
|
826
|
+
---
|
|
827
|
+
|
|
744
828
|
## Introspection and Metadata
|
|
745
829
|
|
|
746
830
|
### Connection Properties
|
|
@@ -820,6 +904,7 @@ The `cursor.statement` property provides detailed metadata about the executed qu
|
|
|
820
904
|
| `is_append_only` | `bool` | Query produces only inserts (vs changelog with updates/deletes) |
|
|
821
905
|
| `is_bounded` | `bool` | Query has finite result set (snapshot) vs unbounded (streaming) |
|
|
822
906
|
| `is_deletable` | `bool` | Statement can be deleted |
|
|
907
|
+
| `is_dry_run` | `bool` | Statement was submitted with `sql.dry-run` (validated, never stored) |
|
|
823
908
|
| `schema` | `Schema` | Result schema with column names and types |
|
|
824
909
|
| `sql_kind` | `str` | Query type: `SELECT`, `INSERT`, `CREATE`, etc. |
|
|
825
910
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: confluent-sql
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.8.0
|
|
4
4
|
Summary: DB-API v2 compliant driver for Confluent Cloud Flink SQL
|
|
5
5
|
Project-URL: Repository, https://github.com/confluentinc/confluent-sql
|
|
6
6
|
Project-URL: Documentation, https://github.com/confluentinc/confluent-sql?tab=readme-ov-file#confluent-sql
|
|
@@ -227,6 +227,7 @@ Requires-Dist: httpx<0.29.0,>=0.23.0
|
|
|
227
227
|
Provides-Extra: dev
|
|
228
228
|
Requires-Dist: pytest; extra == 'dev'
|
|
229
229
|
Requires-Dist: pytest-cov; extra == 'dev'
|
|
230
|
+
Requires-Dist: pytest-xdist>=3.5.0; extra == 'dev'
|
|
230
231
|
Description-Content-Type: text/markdown
|
|
231
232
|
|
|
232
233
|
# confluent-sql
|
|
@@ -252,12 +253,13 @@ The driver defaults to [snapshot mode](https://docs.confluent.io/cloud/current/f
|
|
|
252
253
|
- **Confluent Cloud account** with Flink environment
|
|
253
254
|
- **Existing Flink Database** (Confluent Cloud Kafka cluster)
|
|
254
255
|
- **API credentials**, one of:
|
|
256
|
+
- an **interactive browser login**, with `auth="oauth"` (see [Interactive login](#interactive-browser-login) below), or
|
|
255
257
|
- a **"Global" Confluent Cloud API key** and secret, passed as `global_api_key` / `global_api_secret`, or
|
|
256
258
|
- a **Flink Region API key** and secret, passed as `flink_api_key` / `flink_api_secret`, or
|
|
257
259
|
- a **BYOIDC bearer token** minted by your own OAuth/OIDC identity provider, passed as `external_access_token` / `identity_pool_id` (see [BYOIDC bearer-token authentication](#byoidc-bearer-token-authentication) below).
|
|
258
260
|
|
|
259
|
-
|
|
260
|
-
- **Organization ID** — required,
|
|
261
|
+
The interactive browser login is the simplest way to get started, at the cost of one browser sign-in per process. A Global key works against every route this driver touches. A Flink Region key only works against the core Flink SQL routes (the main DB-API functionality), not other features such as Tableflow, Connectors, or Artifacts.
|
|
262
|
+
- **Organization ID** — required with a Flink Region API key or a BYOIDC bearer token, neither of which can discover it. With a Global API key it may be omitted if the key can see exactly one organization, in which case it's inferred automatically on first use of the connection. With `auth="oauth"` it may always be omitted and defaults to your login's organization.
|
|
261
263
|
|
|
262
264
|
### How to Obtain a Flink Region API Key
|
|
263
265
|
|
|
@@ -278,6 +280,34 @@ To create or find a Flink Region API key:
|
|
|
278
280
|
|
|
279
281
|
API keys may also be generated using the Confluent CLI or by API access, outside the scope of this document.
|
|
280
282
|
|
|
283
|
+
### Interactive browser login
|
|
284
|
+
|
|
285
|
+
Instead of API keys, sign in once through your browser:
|
|
286
|
+
|
|
287
|
+
```python
|
|
288
|
+
conn = confluent_sql.connect(
|
|
289
|
+
auth="oauth",
|
|
290
|
+
environment_id="env-...",
|
|
291
|
+
cloud_provider="aws",
|
|
292
|
+
cloud_region="us-east-1",
|
|
293
|
+
)
|
|
294
|
+
```
|
|
295
|
+
|
|
296
|
+
The browser opens once per process; the login is shared by every connection the process opens. `organization_id` is optional and defaults to your login's organization. See [examples/interactive_oauth_login.py](examples/interactive_oauth_login.py). The resulting login can reach everything your user account can, so all of the driver's functionality, including Tableflow, Connectors, and Artifacts, is available.
|
|
297
|
+
|
|
298
|
+
**Callback port.** To catch the browser's redirect after you sign in, the driver listens on `127.0.0.1:26642` (path `/callback-confluent-sql-docs`) while the login is in progress. This port is fixed by the driver's registered OAuth client and cannot be changed, so it must be free; if something else holds it, `connect()` raises `OAuthLoginError` with reason `PORT_IN_USE`. Only one login can be in progress per machine at a time. If you don't finish signing in within five minutes, it raises `OAuthLoginError` with reason `TIMED_OUT`.
|
|
299
|
+
|
|
300
|
+
**Login lifetime and signing out.** The login is held for the life of the process: closing a `Connection` does not sign you out, and a later `connect(auth="oauth", ...)` in the same process reuses it without another browser bounce. To explicitly discard the login (and its tokens), for example in a long-lived notebook kernel or server, call `confluent_sql.oauth_discard_login()`. The next `connect(auth="oauth", ...)` will then open the browser again. `oauth_discard_login()` closes the shared login's HTTP client immediately, so it must only be called once nothing is using the login: stop any OAuth activity and close every OAuth `Connection` first. A connection left open will fail at its next token refresh (including one triggered by a 401), and a refresh already in progress can be interrupted, even if the refresh token is still valid.
|
|
301
|
+
|
|
302
|
+
**Session lifetime and re-authentication.** A login's refresh token has a hard ~8 hour absolute lifetime (and expires sooner if idle for too long or revoked), after which the session can no longer be silently refreshed. What happens next is controlled by the `reauth` argument to `connect()`:
|
|
303
|
+
|
|
304
|
+
- `reauth="auto"` (the default): the driver opens the browser for a fresh sign-in and retries the failed request. Every connection in the process shares the new login. This suits interactive use such as notebooks and scripts run by a person.
|
|
305
|
+
- `reauth="raise"`: the request raises `confluent_sql.ReauthenticationRequired` instead. Use this for unattended runs (a long batch job or dbt run started and left running), where nobody would be at the browser. Its `reason` is `ABSOLUTE_EXPIRY` (the ~8 hour wall) or `REFRESH_REJECTED` (idle expiry or revocation); catch it to abort or restart the job cleanly.
|
|
306
|
+
|
|
307
|
+
Re-authentication must be completed as the same user who originally signed in. If a different user signs in, even within the same organization, the driver raises `OAuthLoginError` with reason `OAuthLoginFailure.USER_MISMATCH` and the session stays unauthenticated, so you can sign in again as the original user.
|
|
308
|
+
|
|
309
|
+
This check relies on the user identity claims in the login's token, and is strict: a re-authentication whose token carries no identity comparable to the original's is refused the same way, as is one that names a user when the original login's token named none. The only case let through unchecked is a login whose token never named a user at all, where the driver logs a warning at login and has nothing to compare.
|
|
310
|
+
|
|
281
311
|
### BYOIDC bearer-token authentication
|
|
282
312
|
|
|
283
313
|
If your Confluent Cloud organization is configured with your own registered OAuth/OIDC identity provider, you can authenticate to the **Flink data plane** with a bearer token you mint yourself (an "external" token in Confluent's authorization vocabulary) rather than a Confluent API key. Pass the token and the identity-pool id that scopes it:
|
|
@@ -487,3 +517,20 @@ Run tests:
|
|
|
487
517
|
```bash
|
|
488
518
|
uv run pytest
|
|
489
519
|
```
|
|
520
|
+
|
|
521
|
+
#### OAuth tests
|
|
522
|
+
|
|
523
|
+
Every OAuth test (unit and integration) carries the `oauth` pytest marker. The OAuth *integration* tests open your browser for a real Confluent Cloud sign-in, so they are opt-in: they skip unless `RUN_OAUTH_TESTS` is set (they also need `CONFLUENT_ENV_ID`, `CONFLUENT_CLOUD_PROVIDER` and `CONFLUENT_CLOUD_REGION`; some need `CONFLUENT_TEST_DBNAME`). A plain `uv run pytest` therefore runs the OAuth unit tests only.
|
|
524
|
+
|
|
525
|
+
```bash
|
|
526
|
+
# Fast, browser-free OAuth unit tests only
|
|
527
|
+
uv run pytest -m "oauth and unit"
|
|
528
|
+
|
|
529
|
+
# The browser-driven OAuth integration tests only
|
|
530
|
+
RUN_OAUTH_TESTS=1 uv run pytest -n0 -m "oauth and integration"
|
|
531
|
+
|
|
532
|
+
# Every OAuth test, unit and integration
|
|
533
|
+
RUN_OAUTH_TESTS=1 uv run pytest -n0 -m oauth tests
|
|
534
|
+
```
|
|
535
|
+
|
|
536
|
+
`-n0` is required for any run that includes the integration tests: the loopback callback port is fixed, so parallel workers would collide (and open several browsers at once).
|
|
@@ -21,12 +21,13 @@ The driver defaults to [snapshot mode](https://docs.confluent.io/cloud/current/f
|
|
|
21
21
|
- **Confluent Cloud account** with Flink environment
|
|
22
22
|
- **Existing Flink Database** (Confluent Cloud Kafka cluster)
|
|
23
23
|
- **API credentials**, one of:
|
|
24
|
+
- an **interactive browser login**, with `auth="oauth"` (see [Interactive login](#interactive-browser-login) below), or
|
|
24
25
|
- a **"Global" Confluent Cloud API key** and secret, passed as `global_api_key` / `global_api_secret`, or
|
|
25
26
|
- a **Flink Region API key** and secret, passed as `flink_api_key` / `flink_api_secret`, or
|
|
26
27
|
- a **BYOIDC bearer token** minted by your own OAuth/OIDC identity provider, passed as `external_access_token` / `identity_pool_id` (see [BYOIDC bearer-token authentication](#byoidc-bearer-token-authentication) below).
|
|
27
28
|
|
|
28
|
-
|
|
29
|
-
- **Organization ID** — required,
|
|
29
|
+
The interactive browser login is the simplest way to get started, at the cost of one browser sign-in per process. A Global key works against every route this driver touches. A Flink Region key only works against the core Flink SQL routes (the main DB-API functionality), not other features such as Tableflow, Connectors, or Artifacts.
|
|
30
|
+
- **Organization ID** — required with a Flink Region API key or a BYOIDC bearer token, neither of which can discover it. With a Global API key it may be omitted if the key can see exactly one organization, in which case it's inferred automatically on first use of the connection. With `auth="oauth"` it may always be omitted and defaults to your login's organization.
|
|
30
31
|
|
|
31
32
|
### How to Obtain a Flink Region API Key
|
|
32
33
|
|
|
@@ -47,6 +48,34 @@ To create or find a Flink Region API key:
|
|
|
47
48
|
|
|
48
49
|
API keys may also be generated using the Confluent CLI or by API access, outside the scope of this document.
|
|
49
50
|
|
|
51
|
+
### Interactive browser login
|
|
52
|
+
|
|
53
|
+
Instead of API keys, sign in once through your browser:
|
|
54
|
+
|
|
55
|
+
```python
|
|
56
|
+
conn = confluent_sql.connect(
|
|
57
|
+
auth="oauth",
|
|
58
|
+
environment_id="env-...",
|
|
59
|
+
cloud_provider="aws",
|
|
60
|
+
cloud_region="us-east-1",
|
|
61
|
+
)
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
The browser opens once per process; the login is shared by every connection the process opens. `organization_id` is optional and defaults to your login's organization. See [examples/interactive_oauth_login.py](examples/interactive_oauth_login.py). The resulting login can reach everything your user account can, so all of the driver's functionality, including Tableflow, Connectors, and Artifacts, is available.
|
|
65
|
+
|
|
66
|
+
**Callback port.** To catch the browser's redirect after you sign in, the driver listens on `127.0.0.1:26642` (path `/callback-confluent-sql-docs`) while the login is in progress. This port is fixed by the driver's registered OAuth client and cannot be changed, so it must be free; if something else holds it, `connect()` raises `OAuthLoginError` with reason `PORT_IN_USE`. Only one login can be in progress per machine at a time. If you don't finish signing in within five minutes, it raises `OAuthLoginError` with reason `TIMED_OUT`.
|
|
67
|
+
|
|
68
|
+
**Login lifetime and signing out.** The login is held for the life of the process: closing a `Connection` does not sign you out, and a later `connect(auth="oauth", ...)` in the same process reuses it without another browser bounce. To explicitly discard the login (and its tokens), for example in a long-lived notebook kernel or server, call `confluent_sql.oauth_discard_login()`. The next `connect(auth="oauth", ...)` will then open the browser again. `oauth_discard_login()` closes the shared login's HTTP client immediately, so it must only be called once nothing is using the login: stop any OAuth activity and close every OAuth `Connection` first. A connection left open will fail at its next token refresh (including one triggered by a 401), and a refresh already in progress can be interrupted, even if the refresh token is still valid.
|
|
69
|
+
|
|
70
|
+
**Session lifetime and re-authentication.** A login's refresh token has a hard ~8 hour absolute lifetime (and expires sooner if idle for too long or revoked), after which the session can no longer be silently refreshed. What happens next is controlled by the `reauth` argument to `connect()`:
|
|
71
|
+
|
|
72
|
+
- `reauth="auto"` (the default): the driver opens the browser for a fresh sign-in and retries the failed request. Every connection in the process shares the new login. This suits interactive use such as notebooks and scripts run by a person.
|
|
73
|
+
- `reauth="raise"`: the request raises `confluent_sql.ReauthenticationRequired` instead. Use this for unattended runs (a long batch job or dbt run started and left running), where nobody would be at the browser. Its `reason` is `ABSOLUTE_EXPIRY` (the ~8 hour wall) or `REFRESH_REJECTED` (idle expiry or revocation); catch it to abort or restart the job cleanly.
|
|
74
|
+
|
|
75
|
+
Re-authentication must be completed as the same user who originally signed in. If a different user signs in, even within the same organization, the driver raises `OAuthLoginError` with reason `OAuthLoginFailure.USER_MISMATCH` and the session stays unauthenticated, so you can sign in again as the original user.
|
|
76
|
+
|
|
77
|
+
This check relies on the user identity claims in the login's token, and is strict: a re-authentication whose token carries no identity comparable to the original's is refused the same way, as is one that names a user when the original login's token named none. The only case let through unchecked is a login whose token never named a user at all, where the driver logs a warning at login and has nothing to compare.
|
|
78
|
+
|
|
50
79
|
### BYOIDC bearer-token authentication
|
|
51
80
|
|
|
52
81
|
If your Confluent Cloud organization is configured with your own registered OAuth/OIDC identity provider, you can authenticate to the **Flink data plane** with a bearer token you mint yourself (an "external" token in Confluent's authorization vocabulary) rather than a Confluent API key. Pass the token and the identity-pool id that scopes it:
|
|
@@ -256,3 +285,20 @@ Run tests:
|
|
|
256
285
|
```bash
|
|
257
286
|
uv run pytest
|
|
258
287
|
```
|
|
288
|
+
|
|
289
|
+
#### OAuth tests
|
|
290
|
+
|
|
291
|
+
Every OAuth test (unit and integration) carries the `oauth` pytest marker. The OAuth *integration* tests open your browser for a real Confluent Cloud sign-in, so they are opt-in: they skip unless `RUN_OAUTH_TESTS` is set (they also need `CONFLUENT_ENV_ID`, `CONFLUENT_CLOUD_PROVIDER` and `CONFLUENT_CLOUD_REGION`; some need `CONFLUENT_TEST_DBNAME`). A plain `uv run pytest` therefore runs the OAuth unit tests only.
|
|
292
|
+
|
|
293
|
+
```bash
|
|
294
|
+
# Fast, browser-free OAuth unit tests only
|
|
295
|
+
uv run pytest -m "oauth and unit"
|
|
296
|
+
|
|
297
|
+
# The browser-driven OAuth integration tests only
|
|
298
|
+
RUN_OAUTH_TESTS=1 uv run pytest -n0 -m "oauth and integration"
|
|
299
|
+
|
|
300
|
+
# Every OAuth test, unit and integration
|
|
301
|
+
RUN_OAUTH_TESTS=1 uv run pytest -n0 -m oauth tests
|
|
302
|
+
```
|
|
303
|
+
|
|
304
|
+
`-n0` is required for any run that includes the integration tests: the loopback callback port is fixed, so parallel workers would collide (and open several browsers at once).
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
# Minimal project used by examples/artifact_upload_example.py: `uv build --sdist` of this
|
|
2
|
+
# directory produces the tarball the example zips and uploads as a Python UDF artifact.
|
|
3
|
+
[project]
|
|
4
|
+
name = "example-udf"
|
|
5
|
+
version = "0.1.0"
|
|
6
|
+
description = "Example Python UDF packaged for Confluent Cloud Flink"
|
|
7
|
+
requires-python = "~=3.11.0"
|
|
8
|
+
dependencies = ["apache-flink==2.0.0", "setuptools<81"]
|
|
9
|
+
|
|
10
|
+
[build-system]
|
|
11
|
+
requires = ["uv_build>=0.11.30,<0.12.0"]
|
|
12
|
+
build-backend = "uv_build"
|
|
File without changes
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
# pyright: reportMissingImports=false
|
|
2
|
+
# (pyflink is only available in the Confluent Cloud UDF runtime, not this project's environment)
|
|
3
|
+
from pyflink.table import DataTypes
|
|
4
|
+
from pyflink.table.types import DataType
|
|
5
|
+
from pyflink.table.udf import udf
|
|
6
|
+
|
|
7
|
+
# Dictionary mapping both full names and abbreviations to numeric size values
|
|
8
|
+
_SIZE_MAP = {
|
|
9
|
+
"x-small": 0,
|
|
10
|
+
"xs": 0,
|
|
11
|
+
"small": 1,
|
|
12
|
+
"s": 1,
|
|
13
|
+
"medium": 2,
|
|
14
|
+
"m": 2,
|
|
15
|
+
"large": 3,
|
|
16
|
+
"l": 3,
|
|
17
|
+
"x-large": 4,
|
|
18
|
+
"xl": 4,
|
|
19
|
+
"xx-large": 5,
|
|
20
|
+
"xxl": 5,
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _get_size_value(shirt: str) -> int:
|
|
25
|
+
"""
|
|
26
|
+
Returns the numeric size value for a given shirt size string.
|
|
27
|
+
Returns -1 if the size is not found.
|
|
28
|
+
"""
|
|
29
|
+
if shirt is None:
|
|
30
|
+
return -1
|
|
31
|
+
return _SIZE_MAP.get(shirt.strip().lower(), -1)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _f_is_smaller(shirt1: str, shirt2: str) -> bool:
|
|
35
|
+
"""
|
|
36
|
+
Returns True if shirt1 is a smaller size than shirt2 based on standard T-shirt sizes.
|
|
37
|
+
If a size cannot be found, returns False.
|
|
38
|
+
"""
|
|
39
|
+
size1 = _get_size_value(shirt1)
|
|
40
|
+
size2 = _get_size_value(shirt2)
|
|
41
|
+
if size1 == -1 or size2 == -1:
|
|
42
|
+
return False
|
|
43
|
+
return size1 < size2
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
_is_smaller_inp_types: list[DataType] = [
|
|
47
|
+
DataTypes.STRING(),
|
|
48
|
+
DataTypes.STRING(),
|
|
49
|
+
]
|
|
50
|
+
is_smaller = udf(
|
|
51
|
+
_f_is_smaller,
|
|
52
|
+
input_types=_is_smaller_inp_types,
|
|
53
|
+
result_type=DataTypes.BOOLEAN(),
|
|
54
|
+
)
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
"""Example of the Flink artifact lifecycle: package a Python UDF, upload it as an artifact,
|
|
2
|
+
register it with `CREATE FUNCTION`, call it from SQL, and clean everything up.
|
|
3
|
+
|
|
4
|
+
Artifacts are how user code (a Java JAR or, as here, a Python ZIP) gets into Confluent Cloud
|
|
5
|
+
Flink. The UDF's source lives in examples/artifact_udf/ (a copy of the one built in Confluent's
|
|
6
|
+
"create a UDF" guide); this script builds its sdist with `uv build`, zips it as the API expects,
|
|
7
|
+
uploads it, and exercises it.
|
|
8
|
+
|
|
9
|
+
Artifacts are a control-plane surface and are not reachable under BYOIDC -- use an API-key
|
|
10
|
+
connection. They are scoped to the connection's cloud/region/environment.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
import os
|
|
14
|
+
import subprocess
|
|
15
|
+
import tempfile
|
|
16
|
+
import zipfile
|
|
17
|
+
from pathlib import Path
|
|
18
|
+
|
|
19
|
+
import confluent_sql
|
|
20
|
+
from confluent_sql import ArtifactRuntimeLanguage
|
|
21
|
+
|
|
22
|
+
UDF_PROJECT = Path(__file__).parent / "artifact_udf"
|
|
23
|
+
UDF_NAME = "example_udf_is_smaller"
|
|
24
|
+
ARTIFACT_NAME = "confluent_sql_artifact_example"
|
|
25
|
+
|
|
26
|
+
conn = confluent_sql.connect(
|
|
27
|
+
global_api_key=os.environ["CONFLUENT_GLOBAL_API_KEY"],
|
|
28
|
+
global_api_secret=os.environ["CONFLUENT_GLOBAL_API_SECRET"],
|
|
29
|
+
## API-specific keys can be used instead of global API keys:
|
|
30
|
+
# cloud_api_key=os.environ["CONFLUENT_CLOUD_API_KEY"],
|
|
31
|
+
# cloud_api_secret=os.environ["CONFLUENT_CLOUD_API_SECRET"],
|
|
32
|
+
# flink_api_key=os.environ["CONFLUENT_FLINK_API_KEY"],
|
|
33
|
+
# flink_api_secret=os.environ["CONFLUENT_FLINK_API_SECRET"],
|
|
34
|
+
environment_id=os.environ["CONFLUENT_ENV_ID"],
|
|
35
|
+
organization_id=os.environ["CONFLUENT_ORG_ID"],
|
|
36
|
+
cloud_provider=os.environ["CONFLUENT_CLOUD_PROVIDER"],
|
|
37
|
+
cloud_region=os.environ["CONFLUENT_CLOUD_REGION"],
|
|
38
|
+
compute_pool_id=os.getenv("CONFLUENT_COMPUTE_POOL_ID"), # optional; None -> default pool
|
|
39
|
+
database=os.getenv("CONFLUENT_DATABASE", os.getenv("CONFLUENT_TEST_DBNAME", "")),
|
|
40
|
+
)
|
|
41
|
+
artifacts = conn.get_artifacts_api()
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def build_udf_zip(workdir: Path) -> Path:
|
|
45
|
+
"""Build the UDF project's sdist and wrap it in the ZIP a Python artifact must be."""
|
|
46
|
+
subprocess.run(
|
|
47
|
+
["uv", "build", "--sdist", "--out-dir", str(workdir)], cwd=UDF_PROJECT, check=True
|
|
48
|
+
)
|
|
49
|
+
(sdist,) = workdir.glob("*.tar.gz")
|
|
50
|
+
zip_path = workdir / sdist.with_suffix("").with_suffix(".zip").name
|
|
51
|
+
with zipfile.ZipFile(zip_path, "w") as zf:
|
|
52
|
+
zf.write(sdist, sdist.name)
|
|
53
|
+
return zip_path
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
artifact = None
|
|
57
|
+
try:
|
|
58
|
+
print("building the UDF package and uploading it as an artifact...")
|
|
59
|
+
with tempfile.TemporaryDirectory() as tmp:
|
|
60
|
+
udf_zip = build_udf_zip(Path(tmp))
|
|
61
|
+
# Requests a presigned URL, uploads the file to it, then creates the artifact. The .zip
|
|
62
|
+
# extension gives content_format=ZIP; Python UDFs must say runtime_language=PYTHON.
|
|
63
|
+
artifact = artifacts.create(
|
|
64
|
+
ARTIFACT_NAME,
|
|
65
|
+
file=udf_zip,
|
|
66
|
+
runtime_language=ArtifactRuntimeLanguage.PYTHON,
|
|
67
|
+
description="Created by confluent-sql's artifact_upload_example.py",
|
|
68
|
+
)
|
|
69
|
+
print(f"created artifact {artifact.id} ({artifact.display_name})")
|
|
70
|
+
|
|
71
|
+
print("artifacts in this environment/region:")
|
|
72
|
+
for a in artifacts.list(runtime_language=ArtifactRuntimeLanguage.PYTHON):
|
|
73
|
+
print(f" {a.id} {a.display_name} {a.content_format}")
|
|
74
|
+
|
|
75
|
+
# Versions only show up on a read of the artifact (not in the create response or a list).
|
|
76
|
+
artifact = artifacts.get(artifact.id)
|
|
77
|
+
print(f"versions: {[v.version for v in artifact.versions]}")
|
|
78
|
+
|
|
79
|
+
print(f"registering function {UDF_NAME} from the artifact (can take a while)...")
|
|
80
|
+
cursor = conn.cursor()
|
|
81
|
+
cursor.execute(
|
|
82
|
+
f"CREATE FUNCTION {UDF_NAME} AS 'example_udf.tshirt_sizing.is_smaller' "
|
|
83
|
+
f"LANGUAGE PYTHON USING JAR 'confluent-artifact://{artifact.id}'"
|
|
84
|
+
)
|
|
85
|
+
print("running a streaming query that calls the UDF (Python UDF startup is slow)...")
|
|
86
|
+
try:
|
|
87
|
+
# Python UDFs only run in streaming mode. A VALUES source is bounded, so this finishes.
|
|
88
|
+
with conn.closing_streaming_cursor(as_dict=True) as streaming_cursor:
|
|
89
|
+
streaming_cursor.execute(
|
|
90
|
+
f"SELECT size_a, size_b, {UDF_NAME}(size_a, size_b) AS is_smaller "
|
|
91
|
+
"FROM (VALUES ('small', 'x-large'), ('xl', 's'), ('m', 'm')) AS t(size_a, size_b)"
|
|
92
|
+
)
|
|
93
|
+
for row in streaming_cursor:
|
|
94
|
+
assert isinstance(row, dict)
|
|
95
|
+
print(f"{UDF_NAME}({row['size_a']!r}, {row['size_b']!r}) -> {row['is_smaller']}")
|
|
96
|
+
finally:
|
|
97
|
+
print(f"dropping function {UDF_NAME}...")
|
|
98
|
+
cursor.execute(f"DROP FUNCTION {UDF_NAME}")
|
|
99
|
+
finally:
|
|
100
|
+
try:
|
|
101
|
+
# wait_for_removal=True is the default: blocks until a read 404s.
|
|
102
|
+
if artifact is not None:
|
|
103
|
+
print(f"deleting artifact {artifact.id} (waits until it's gone)...")
|
|
104
|
+
artifacts.delete(artifact.id)
|
|
105
|
+
print(f"deleted artifact {artifact.id}")
|
|
106
|
+
finally:
|
|
107
|
+
conn.close()
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
"""Example of interactive human-login OAuth: sign in once via a browser instead of
|
|
2
|
+
supplying API keys.
|
|
3
|
+
|
|
4
|
+
Running this pops a browser to Confluent Cloud for a one-time interactive sign-in (the driver
|
|
5
|
+
listens on 127.0.0.1:26642 for the redirect); once you consent, the resulting login is reused for
|
|
6
|
+
every Connection this process opens -- one browser bounce per process, even under something like
|
|
7
|
+
dbt's multi-threaded mode.
|
|
8
|
+
|
|
9
|
+
organization_id is optional under auth="oauth": omitted, it's discovered from your login session
|
|
10
|
+
(your default Confluent Cloud organization); supplied, it scopes the login to that organization
|
|
11
|
+
instead.
|
|
12
|
+
|
|
13
|
+
To sign in against a non-production Confluent Cloud environment instead, import `STAG` or `DEVEL`
|
|
14
|
+
(`from confluent_sql.oauth.config import STAG`) and pass `oauth_config=STAG` to
|
|
15
|
+
`confluent_sql.connect()`. Note that `oauth_config` only redirects the *login*: resource endpoints
|
|
16
|
+
still default to production. You must also pass the matching non-production Flink gateway as
|
|
17
|
+
`endpoint=` (https://flink.<region>.<cloud>.<environment>.cpdev.cloud, where <environment> is
|
|
18
|
+
`stag` or `devel`) and, for control-plane operations (Tableflow, Connectors, Artifacts), the
|
|
19
|
+
matching `controlplane_endpoint=` (https://api.<environment>.cpdev.cloud). Otherwise the example's
|
|
20
|
+
statements go to the production gateway with non-production credentials. Cloud provider and
|
|
21
|
+
region are not needed when `endpoint=` is given.
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
import os
|
|
25
|
+
|
|
26
|
+
import confluent_sql
|
|
27
|
+
|
|
28
|
+
conn = confluent_sql.connect(
|
|
29
|
+
auth="oauth",
|
|
30
|
+
environment_id=os.environ["CONFLUENT_ENV_ID"],
|
|
31
|
+
cloud_provider=os.environ["CONFLUENT_CLOUD_PROVIDER"],
|
|
32
|
+
cloud_region=os.environ["CONFLUENT_CLOUD_REGION"],
|
|
33
|
+
)
|
|
34
|
+
print(f"Signed in to Confluent Cloud for organization {conn.organization_id!r}")
|
|
35
|
+
|
|
36
|
+
cursor = conn.cursor()
|
|
37
|
+
try:
|
|
38
|
+
cursor.execute("SELECT 1 as test_value_1, 2 as test_value_2, 3 as test_value_3")
|
|
39
|
+
for row in cursor:
|
|
40
|
+
print(f"iterating over cursor results: {row}")
|
|
41
|
+
finally:
|
|
42
|
+
cursor.close()
|
|
43
|
+
conn.close()
|
|
@@ -67,6 +67,15 @@ verboseOutput = true
|
|
|
67
67
|
pythonVersion = "3.10"
|
|
68
68
|
pythonPlatform = "All"
|
|
69
69
|
include = ["src", "tests", "examples"]
|
|
70
|
+
# Default to requiring fully parameter type annotation (src/, examples/)
|
|
71
|
+
reportMissingParameterType = "error"
|
|
72
|
+
executionEnvironments = [
|
|
73
|
+
# But let tests/ not require parameter type annotations (at this time).
|
|
74
|
+
# (extraPaths is needed because an environment's root is also its import root: without it,
|
|
75
|
+
# tests' `from tests.conftest import ...` (needs ".") and `import confluent_sql` (needs "src")
|
|
76
|
+
# stop resolving.)
|
|
77
|
+
{ root = "tests", reportMissingParameterType = "none", extraPaths = [".", "src"] }
|
|
78
|
+
]
|
|
70
79
|
|
|
71
80
|
[tool.pytest.ini_options]
|
|
72
81
|
addopts = [
|
|
@@ -75,6 +84,8 @@ addopts = [
|
|
|
75
84
|
"--strict-markers",
|
|
76
85
|
"--cov=confluent_sql",
|
|
77
86
|
"--cov-report=html",
|
|
87
|
+
"-n=2",
|
|
88
|
+
"--dist=loadscope",
|
|
78
89
|
]
|
|
79
90
|
testpaths = "tests"
|
|
80
91
|
markers = [
|
|
@@ -82,10 +93,11 @@ markers = [
|
|
|
82
93
|
"integration: mark a test as an integration test",
|
|
83
94
|
"slow: mark a test as a slow (longer than a few seconds integration) test",
|
|
84
95
|
"typeconv: mark a test as a type conversion test",
|
|
96
|
+
"oauth: mark a test as exercising the interactive OAuth login (unit or integration)",
|
|
85
97
|
]
|
|
86
98
|
|
|
87
99
|
[project.optional-dependencies]
|
|
88
|
-
dev = ["pytest", "pytest-cov"]
|
|
100
|
+
dev = ["pytest", "pytest-cov", "pytest-xdist>=3.5.0"]
|
|
89
101
|
|
|
90
102
|
[dependency-groups]
|
|
91
103
|
dev = [
|
|
@@ -93,6 +105,7 @@ dev = [
|
|
|
93
105
|
"pytest>=7.0.0",
|
|
94
106
|
"pytest-cov>=4.0.0",
|
|
95
107
|
"pytest-mock>=3.15.1",
|
|
108
|
+
"pytest-xdist>=3.5.0",
|
|
96
109
|
"python-dotenv>=1.0.0",
|
|
97
110
|
"ruff>=0.14.3",
|
|
98
111
|
]
|